From 933e1aa97fb6999678336b30bf8df785757a6e23 Mon Sep 17 00:00:00 2001 From: visi Date: Thu, 27 Jun 2024 17:31:55 -0400 Subject: [PATCH 01/86] WIP: step1: aha:network default --- synapse/lib/aha.py | 181 ++++++++++++++++++++++------------ synapse/lib/cell.py | 37 ++++--- synapse/tests/test_lib_aha.py | 148 ++++++++++----------------- synapse/tools/aha/enroll.py | 3 +- 4 files changed, 199 insertions(+), 170 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 0e495bdd8f8..f3738eaf52e 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -380,17 +380,17 @@ async def getUserInfo(self): return { 'aha:urls': self.aha._getAhaUrls(), 'aha:user': self.userinfo.get('name'), - 'aha:network': self.aha.conf.get('aha:network'), + 'aha:network': self.aha._getAhaNetwork(), } async def getCaCert(self): - ahanetw = self.aha.conf.get('aha:network') + ahanetw = self.aha._getAhaNetwork() return self.aha.certdir.getCaCertBytes(ahanetw) async def signUserCsr(self, byts): ahauser = self.userinfo.get('name') - ahanetw = self.aha.conf.get('aha:network') + ahanetw = self.aha._getAhaNetwork() username = f'{ahauser}@{ahanetw}' @@ -416,7 +416,7 @@ async def getProvInfo(self): return self.provinfo async def getCaCert(self): - ahanetw = self.aha.conf.get('aha:network') + ahanetw = self.aha._getAhaNetwork() return self.aha.certdir.getCaCertBytes(ahanetw) async def signHostCsr(self, byts): @@ -464,6 +464,10 @@ class AhaCell(s_cell.Cell): confbase['mirror']['hidedocs'] = False # type: ignore confbase['mirror']['hidecmdl'] = False # type: ignore confdefs = { + 'dns:name': { + 'description': 'The registered DNS name used to reach the AHA service.', + 'type': ['string', 'null'], + }, 'aha:urls': { 'description': 'A list of all available AHA server URLs.', 'type': ['string', 'array'], @@ -537,41 +541,95 @@ def _initCellHttpApis(self): self.addHttpApi('/api/v1/aha/provision/service', AhaProvisionServiceV1, {'cell': self}) async def initServiceRuntime(self): + self.addActiveCoro(self._clearInactiveSessions) if self.isactive: + # bootstrap a CA for our aha:network - netw = self.conf.get('aha:network') - if netw is not None: + netw = self._getAhaNetwork() + + if self.certdir.getCaCertPath(netw) is None: + logger.info(f'Adding CA certificate for {netw}') + await self.genCaCert(netw) + + name = self.conf.get('aha:name') + + host = f'{name}.{netw}' + if self.certdir.getHostCertPath(host) is None: + logger.info(f'Adding server certificate for {host}') + await self._genHostCert(host, signas=netw) + + user = self._getAhaAdmin() + if user is not None: + if self.certdir.getUserCertPath(user) is None: + logger.info(f'Adding user certificate for {user}@{netw}') + await self._genUserCert(user, signas=netw) + + def _getDnsName(self): + # emulate the old aha name.network behavior if the + # explicit option is not set. + + hostname = self.conf.get('dns:name') + if hostname is not None: + return hostname + + name = self.conf.get('aha:name') + if name is not None: + return f'{name}.{self._getAhaNetwork()}' - if self.certdir.getCaCertPath(netw) is None: - logger.info(f'Adding CA certificate for {netw}') - await self.genCaCert(netw) + def _getProvListen(self): - name = self.conf.get('aha:name') + lisn = self.conf.get('provision:listen') + if lisn is not None: + return lisn - host = f'{name}.{netw}' - if self.certdir.getHostCertPath(host) is None: - logger.info(f'Adding server certificate for {host}') - await self._genHostCert(host, signas=netw) + # this may not use _getDnsName() in order to maintain + # backward compatibilty with aha name.network configs + # that do not intend to listen for provisioning. + hostname = self.conf.get('dns:name') + if hostname is not None: + return f'ssl://{hostname}:27272' - user = self._getAhaAdmin() - if user is not None: - if self.certdir.getUserCertPath(user) is None: - logger.info(f'Adding user certificate for {user}@{netw}') - await self._genUserCert(user, signas=netw) + def _getDmonListen(self): + + lisn = self.conf.get('dmon:listen') + if lisn is not None: + return lisn + + network = self._getAhaNetwork() + dnsname = self._getDnsName() + if dnsname is not None: + return f'ssl://0.0.0.0?hostname={dnsname}&ca={network}' + + def _reqProvListen(self): + lisn = self._getProvListen() + if lisn is not None: + return lisn + + mesg = 'The AHA server is not configured for provisioning.' + raise s_exc.NeedConfValu(mesg=mesg) async def initServiceNetwork(self): + # bootstrap CA/host certs first + network = self._getAhaNetwork() + if network is not None: + await self._genCaCert(network) + + hostname = self._getDnsName() + if hostname is not None and network is not None: + await self._genHostCert(hostname, signas=network) + await s_cell.Cell.initServiceNetwork(self) self.provdmon = None - provurl = self.conf.get('provision:listen') + provurl = self._getProvListen() if provurl is not None: self.provdmon = await ProvDmon.anit(self) self.onfini(self.provdmon) - await self.provdmon.listen(provurl) + self.provaddr = await self.provdmon.listen(provurl) async def _clearInactiveSessions(self): @@ -658,11 +716,7 @@ async def addAhaSvc(self, name, info, network=None): def _getAhaName(self, name): # the modern version of names is absolute or ... if name.endswith('...'): - netw = self.conf.get('aha:network') - if netw is None: # pragma: no cover - mesg = 'AHA Server requires aha:network configuration.' - raise s_exc.NeedConfValu(mesg=mesg) - name = name[:-2] + netw + return name[:-2] + self._getAhaNetwork() return name async def getAhaPool(self, name): @@ -780,6 +834,7 @@ async def _reqAhaSvc(self, svcname): @s_nexus.Pusher.onPushAuto('aha:svc:del') async def delAhaSvc(self, name, network=None): + name = self._getAhaName(name) svcname, svcnetw, svcfull = self._nameAndNetwork(name, network) logger.info(f'Deleting service [{svcfull}].', extra=await self.getLogExtra(name=svcname, netw=svcnetw)) @@ -794,6 +849,8 @@ async def delAhaSvc(self, name, network=None): await self.fire('aha:svcdel', svcname=svcname, svcnetw=svcnetw) async def setAhaSvcDown(self, name, linkiden, network=None): + + name = self._getAhaName(name) svcname, svcnetw, svcfull = self._nameAndNetwork(name, network) path = ('aha', 'services', svcnetw, svcname) @@ -916,8 +973,18 @@ async def genCaCert(self, network): return cacert + async def _genCaCert(self, network): + + if os.path.isfile(os.path.join(self.dirn, 'certs', 'cas', f'{network}.crt')): + return + + await self.genCaCert(network) + async def _genHostCert(self, hostname, signas=None): + if signas is not None: + await self._genCaCert(signas) + if os.path.isfile(os.path.join(self.dirn, 'certs', 'hosts', '{hostname}.crt')): return @@ -1004,7 +1071,8 @@ async def signUserCsr(self, csrtext, signas=None): return self.certdir._certToByts(cert).decode() - def _getAhaUrls(self): + def _getAhaUrls(self, user='root'): + urls = self.conf.get('aha:urls') if urls is not None: if isinstance(urls, str): @@ -1015,29 +1083,29 @@ def _getAhaUrls(self): if ahaname is None: return None - ahanetw = self.conf.get('aha:network') - if ahanetw is None: - return None + ahanetw = self._getAhaNetwork() if self.sockaddr is None or not isinstance(self.sockaddr, tuple): return None + # FIXME this is the problematic use case + # FIXME may need to generate an AHA CA signed server cert for DNS NAME + # TODO this could eventually enumerate others via itself - return (f'ssl://{ahaname}.{ahanetw}:{self.sockaddr[1]}',) + netloc = self._getDnsName() + return (f'ssl://{netloc}:{self.sockaddr[1]}?certname={user}@{ahanetw}',) async def addAhaSvcProv(self, name, provinfo=None): + if not name: + raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.') + ahaurls = self._getAhaUrls() if ahaurls is None: mesg = 'AHA server has no configured aha:urls.' raise s_exc.NeedConfValu(mesg=mesg) - if self.conf.get('provision:listen') is None: - mesg = 'The AHA server does not have a provision:listen URL!' - raise s_exc.NeedConfValu(mesg=mesg) - - if not name: - raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.') + self._reqProvListen() if provinfo is None: provinfo = {} @@ -1048,23 +1116,15 @@ async def addAhaSvcProv(self, name, provinfo=None): conf = provinfo.setdefault('conf', {}) - mynetw = self.conf.get('aha:network') + netw = self._getAhaNetwork() ahaadmin = self.conf.get('aha:admin') if ahaadmin is not None: # pragma: no cover conf.setdefault('aha:admin', ahaadmin) conf.setdefault('aha:user', 'root') - conf.setdefault('aha:network', mynetw) - netw = conf.get('aha:network') - if netw is None: - mesg = 'AHA server has no configured aha:network.' - raise s_exc.NeedConfValu(mesg=mesg) - - if netw != mynetw: - mesg = f'Provisioning aha:network must be equal to the Aha servers network. Expected {mynetw}, got {netw}' - raise s_exc.BadConfValu(mesg=mesg, name='aha:network', expected=mynetw, got=netw) + conf['aha:network'] = netw hostname = f'{name}.{netw}' @@ -1090,13 +1150,14 @@ async def addAhaSvcProv(self, name, provinfo=None): # allow user to win over leader ahauser = conf.get('aha:user') - ahaurls = s_telepath.modurl(ahaurls, user=ahauser) + certname = f'{ahauser}@{netw}' + ahaurls = s_telepath.modurl(ahaurls, certname=certname) conf.setdefault('aha:registry', ahaurls) mirname = provinfo.get('mirror') if mirname is not None: - conf['mirror'] = f'aha://{ahauser}@{mirname}.{netw}' + conf['mirror'] = f'aha://{ahauser}@{mirname}...' user = await self.auth.getUserByName(ahauser) if user is None: @@ -1122,7 +1183,10 @@ async def addAhaSvcProv(self, name, provinfo=None): def _getProvClientUrl(self, iden): - provlisn = self.conf.get('provision:listen') + provlisn = self._getProvListen() + + provport = self.provaddr[1] + provhost = self._getDnsName() urlinfo = s_telepath.chopurl(provlisn) @@ -1133,8 +1197,8 @@ def _getProvClientUrl(self, iden): host = urlinfo.get('host') newinfo = { - 'host': host, - 'port': urlinfo.get('port'), + 'host': provhost, + 'port': provport, 'scheme': scheme, 'path': '/' + iden, } @@ -1177,19 +1241,12 @@ async def delAhaSvcProv(self, iden): async def addAhaUserEnroll(self, name, userinfo=None, again=False): - provurl = self.conf.get('provision:listen') - if provurl is None: - mesg = 'The AHA server does not have a provision:listen URL!' - raise s_exc.NeedConfValu(mesg=mesg) - - ahanetw = self.conf.get('aha:network') - if ahanetw is None: - mesg = 'AHA server requires aha:network configuration.' - raise s_exc.NeedConfValu(mesg=mesg) - if not name: raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.') + provurl = self._reqProvListen() + ahanetw = self._getAhaNetwork() + username = f'{name}@{ahanetw}' if len(username) > 64: diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 35fb5c33f65..db3a66669f5 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -961,7 +961,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware): 'type': 'string', }, 'aha:network': { - 'description': 'The AHA service network. This makes aha:name/aha:leader relative names.', + 'description': 'The AHA service network. Defaults to "synapse".', 'type': 'string', }, 'aha:registry': { @@ -1126,8 +1126,8 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): # we need to know this pretty early... self.ahasvcname = None ahaname = self.conf.get('aha:name') - ahanetw = self.conf.get('aha:network') - if ahaname is not None and ahanetw is not None: + ahanetw = self._getAhaNetwork() + if ahaname is not None: self.ahasvcname = f'{ahaname}.{ahanetw}' # each cell has a guid @@ -1507,7 +1507,7 @@ async def finiaha(): self.onfini(finiaha) ahauser = self.conf.get('aha:user') - ahanetw = self.conf.get('aha:network') + ahanetw = self._getAhaNetwork() ahaadmin = self._getAhaAdmin() if ahaadmin is not None: @@ -1516,6 +1516,20 @@ async def finiaha(): if ahauser is not None: await self._addAdminUser(ahauser) + def _getAhaNetwork(self): + return self.conf.get('aha:network', 'synapse') + + def _getDmonListen(self): + + lisn = self.conf.get('dmon:listen') + if lisn is not None: + return lisn + + network = self._getAhaNetwork() + ahaname = self.conf.get('aha:name') + if ahaname is not None: + return f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}' + async def _addAdminUser(self, username): # add the user in a pre-nexus compatible way user = await self.auth.getUserByName(username) @@ -1562,7 +1576,7 @@ async def initServiceNetwork(self): self.sockaddr = None - turl = self.conf.get('dmon:listen') + turl = self._getDmonListen() if turl is not None: self.sockaddr = await self.dmon.listen(turl) logger.info(f'dmon listening: {turl}') @@ -1628,7 +1642,7 @@ async def _initAhaService(self): return ahalead = self.conf.get('aha:leader') - ahanetw = self.conf.get('aha:network') + ahanetw = self._getAhaNetwork() ahainfo = await self.getAhaInfo() if ahainfo is None: @@ -1747,10 +1761,7 @@ async def promote(self, graceful=False): mesg = 'Cannot gracefully promote without aha:name configured.' raise s_exc.BadArg(mesg=mesg) - ahanetw = self.conf.get('aha:network') - if ahanetw is None: # pragma: no cover - mesg = 'Cannot gracefully promote without aha:network configured.' - raise s_exc.BadArg(mesg=mesg) + ahanetw = self._getAhaNetwork() myurl = f'aha://{ahaname}.{ahanetw}' logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}') @@ -1848,7 +1859,7 @@ async def _setAhaActive(self): await proxy.fini() return - ahanetw = self.conf.get('aha:network') + ahanetw = self._getAhaNetwork() try: await proxy.addAhaSvc(ahalead, ahainfo, network=ahanetw) except asyncio.CancelledError: # pragma: no cover @@ -3876,7 +3887,7 @@ async def _getCellUser(self, link, mesg): if username.find('@') != -1: userpart, hostpart = username.split('@', 1) - if hostpart == self.conf.get('aha:network'): + if hostpart == self._getAhaNetwork(): user = await self.auth.getUserByName(userpart) if user is not None: if user.isLocked(): @@ -4069,7 +4080,7 @@ async def getCellInfo(self): 'aha': { 'name': self.conf.get('aha:name'), 'leader': self.conf.get('aha:leader'), - 'network': self.conf.get('aha:network'), + 'network': self._getAhaNetwork(), } }, 'features': { diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index eace62dfade..9d2f4a9f06e 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -104,15 +104,14 @@ async def test_lib_aha_offon(self): wait00 = aha.waiter(1, 'aha:svcadd') cryo_conf = { - 'aha:name': '0.cryo.mynet', - 'aha:admin': 'root@cryo.mynet', + 'aha:name': '0.cryo', 'aha:registry': f'tcp://root:secret@127.0.0.1:{port}', 'dmon:listen': 'tcp://0.0.0.0:0/', } async with self.getTestCryo(dirn=cryo0_dirn, conf=cryo_conf) as cryo: self.isin(len(await wait00.wait(timeout=6)), (1, 2)) - svc = await aha.getAhaSvc('0.cryo.mynet') + svc = await aha.getAhaSvc('0.cryo...') linkiden = svc.get('svcinfo', {}).get('online') self.nn(linkiden) @@ -122,12 +121,12 @@ async def test_lib_aha_offon(self): async with self.getTestAha(conf=conf.copy(), dirn=dirn) as aha: wait01 = aha.waiter(1, 'aha:svcdown') await wait01.wait(timeout=6) - svc = await aha.getAhaSvc('0.cryo.mynet') + svc = await aha.getAhaSvc('0.cryo...') self.notin('online', svc.get('svcinfo')) # Try setting something down a second time - await aha.setAhaSvcDown('0.cryo.mynet', linkiden, network=None) - svc = await aha.getAhaSvc('0.cryo.mynet') + await aha.setAhaSvcDown('0.cryo', linkiden, network='synapse') + svc = await aha.getAhaSvc('0.cryo...') self.notin('online', svc.get('svcinfo')) async def test_lib_aha(self): @@ -163,9 +162,9 @@ async def test_lib_aha(self): wait00 = aha.waiter(1, 'aha:svcadd') conf = { - 'aha:name': '0.cryo.mynet', - 'aha:leader': 'cryo.mynet', - 'aha:admin': 'root@cryo.mynet', + 'aha:name': '0.cryo', + 'aha:leader': 'cryo', + 'aha:admin': 'root@synapse', 'aha:registry': [f'tcp://root:hehehaha@127.0.0.1:{port}', f'tcp://root:hehehaha@127.0.0.1:{port}'], 'dmon:listen': 'tcp://0.0.0.0:0/', @@ -174,7 +173,7 @@ async def test_lib_aha(self): await cryo.auth.rootuser.setPasswd('secret') - ahaadmin = await cryo.auth.getUserByName('root@cryo.mynet') + ahaadmin = await cryo.auth.getUserByName('root@synapse') self.nn(ahaadmin) self.true(ahaadmin.isAdmin()) @@ -183,14 +182,14 @@ async def test_lib_aha(self): with self.raises(s_exc.NoSuchName): await s_telepath.getAhaProxy({'host': 'hehe.haha'}) - async with await s_telepath.openurl('aha://root:secret@cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy: self.nn(await proxy.getCellIden()) with self.raises(s_exc.BadArg): _proxy = await cryo.ahaclient.proxy(timeout=2) - await _proxy.modAhaSvcInfo('cryo.mynet', {'newp': 'newp'}) + await _proxy.modAhaSvcInfo('cryo...', {'newp': 'newp'}) - async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: self.nn(await proxy.getCellIden()) # force a reconnect... @@ -199,7 +198,7 @@ async def test_lib_aha(self): await proxy.fini() self.nn(await waiter.wait(timeout=6)) - async with await s_telepath.openurl('aha://root:secret@cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy: self.nn(await proxy.getCellIden()) waiter = aha.waiter(1, 'aha:svcadd') @@ -207,17 +206,17 @@ async def test_lib_aha(self): await cryo.setCellActive(False) with self.raises(s_exc.NoSuchName): - async with await s_telepath.openurl('aha://root:secret@cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy: pass self.nn(await waiter.wait(timeout=6)) - async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: self.nn(await proxy.getCellIden()) await cryo.setCellActive(True) - async with await s_telepath.openurl('aha://root:secret@cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy: self.nn(await proxy.getCellIden()) # some coverage edge cases... @@ -234,7 +233,7 @@ async def test_lib_aha(self): self.false(ahaadmin.isAdmin()) async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo: - ahaadmin = await cryo.auth.getUserByName('root@cryo.mynet') + ahaadmin = await cryo.auth.getUserByName('root@synapse') # And we should be unlocked and admin now self.false(ahaadmin.isLocked()) self.true(ahaadmin.isAdmin()) @@ -322,15 +321,16 @@ async def test_lib_aha(self): self.eq(info.get('status'), 'ok') result = info.get('result') self.len(2, result) - self.eq({'0.cryo.mynet', 'cryo.mynet'}, + self.eq({'0.cryo.synapse', 'cryo.synapse'}, {svcinfo.get('name') for svcinfo in result}) - async with sess.get(svcsurl, json={'network': 'mynet'}) as resp: + async with sess.get(svcsurl, json={'network': 'synapse'}) as resp: info = await resp.json() self.eq(info.get('status'), 'ok') result = info.get('result') - self.len(1, result) - self.eq('cryo.mynet', result[0].get('name')) + self.len(2, result) + self.eq({'0.cryo.synapse', 'cryo.synapse'}, + {svcinfo.get('name') for svcinfo in result}) async with sess.get(svcsurl, json={'network': 'newp'}) as resp: info = await resp.json() @@ -385,7 +385,7 @@ async def test_lib_aha(self): uinfo = ahainfo.get('urlinfo', {}) self.eq(uinfo.get('scheme'), 'tcp') self.gt(uinfo.get('port'), 0) - self.eq(aha._getAhaUrls()[0], f'ssl://0.test.foo:{aha.sockaddr[1]}') + self.eq(aha._getAhaUrls()[0], f'ssl://0.test.foo:{aha.sockaddr[1]}?certname=root@foo') async def test_lib_aha_loadenv(self): @@ -441,8 +441,8 @@ async def test_lib_aha_finid_cell(self): wait00 = aha.waiter(1, 'aha:svcadd') conf = { - 'aha:name': '0.cryo.mynet', - 'aha:admin': 'root@cryo.mynet', + 'aha:name': '0.cryo', + 'aha:admin': 'root@synapse', 'aha:registry': aharegistry, 'dmon:listen': 'tcp://0.0.0.0:0/', } @@ -450,7 +450,7 @@ async def test_lib_aha_finid_cell(self): await cryo.auth.rootuser.setPasswd('secret') - ahaadmin = await cryo.auth.getUserByName('root@cryo.mynet') + ahaadmin = await cryo.auth.getUserByName('root@synapse') self.nn(ahaadmin) self.true(ahaadmin.isAdmin()) @@ -458,7 +458,7 @@ async def test_lib_aha_finid_cell(self): self.isin(atup, s_telepath.aha_clients) - async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: self.nn(await proxy.getCellIden()) _ahaclient = s_telepath.aha_clients.get(atup).get('client') @@ -475,7 +475,7 @@ async def quickproxy(self, timeout): with mock.patch('synapse.telepath.Client.proxy', quickproxy): with self.raises(asyncio.TimeoutError): - async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: self.fail('Should never reach a connection.') async def test_lib_aha_onlink_fail(self): @@ -493,8 +493,7 @@ async def test_lib_aha_onlink_fail(self): wait00 = aha.waiter(1, 'aha:svcadd') conf = { - 'aha:name': '0.cryo.mynet', - 'aha:admin': 'root@cryo.mynet', + 'aha:name': '0.cryo', 'aha:registry': f'tcp://root:secret@127.0.0.1:{port}', 'dmon:listen': 'tcp://0.0.0.0:0/', } @@ -504,7 +503,7 @@ async def test_lib_aha_onlink_fail(self): self.none(await wait00.wait(timeout=2)) - svc = await aha.getAhaSvc('0.cryo.mynet') + svc = await aha.getAhaSvc('0.cryo...') self.none(svc) wait01 = aha.waiter(1, 'aha:svcadd') @@ -512,11 +511,11 @@ async def test_lib_aha_onlink_fail(self): self.nn(await wait01.wait(timeout=2)) - svc = await aha.getAhaSvc('0.cryo.mynet') + svc = await aha.getAhaSvc('0.cryo...') self.nn(svc) self.nn(svc.get('svcinfo', {}).get('online')) - async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy: + async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: self.nn(await proxy.getCellIden()) async def test_lib_aha_bootstrap(self): @@ -559,34 +558,20 @@ async def test_lib_aha_noconf(self): with self.raises(s_exc.NeedConfValu): await aha.addAhaUserEnroll('hehe') - aha.conf['provision:listen'] = 'tcp://127.0.0.1:27272' - - with self.raises(s_exc.NeedConfValu): - await aha.addAhaSvcProv('hehe') - with self.raises(s_exc.NeedConfValu): await aha.addAhaUserEnroll('hehe') - aha.conf['aha:network'] = 'haha' - await aha.addAhaSvcProv('hehe') - async def test_lib_aha_provision(self): with self.getTestDir() as dirn: conf = { 'aha:name': 'aha', - 'aha:network': 'loop.vertex.link', - 'provision:listen': 'ssl://aha.loop.vertex.link:0' + 'dns:name': 'aha.loop.vertex.link', } async with self.getTestAha(dirn=dirn, conf=conf) as aha: - addr, port = aha.provdmon.addr - # update the config to reflect the dynamically bound port - aha.conf['provision:listen'] = f'ssl://aha.loop.vertex.link:{port}' - - # do this config ex-post-facto due to port binding... - host, ahaport = await aha.dmon.listen('ssl://0.0.0.0:0?hostname=aha.loop.vertex.link&ca=loop.vertex.link') + ahaport = aha.sockaddr[1] aha.conf['aha:urls'] = f'ssl://aha.loop.vertex.link:{ahaport}' url = aha.getLocalUrl() @@ -655,19 +640,19 @@ async def test_lib_aha_provision(self): self.none(await axon.auth.getUserByName('axon@loop.vertex.link')) self.true(os.path.isfile(s_common.genpath(axon.dirn, 'prov.done'))) - self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'cas', 'loop.vertex.link.crt'))) - self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'hosts', '00.axon.loop.vertex.link.crt'))) - self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'hosts', '00.axon.loop.vertex.link.key'))) - self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'users', 'root@loop.vertex.link.crt'))) - self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'users', 'root@loop.vertex.link.key'))) + self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'cas', 'synapse.crt'))) + self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'hosts', '00.axon.synapse.crt'))) + self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'hosts', '00.axon.synapse.key'))) + self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'users', 'root@synapse.crt'))) + self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'users', 'root@synapse.key'))) yamlconf = s_common.yamlload(axon.dirn, 'cell.yaml') self.eq('axon', yamlconf.get('aha:leader')) self.eq('00.axon', yamlconf.get('aha:name')) - self.eq('loop.vertex.link', yamlconf.get('aha:network')) + self.eq('synapse', yamlconf.get('aha:network')) self.none(yamlconf.get('aha:admin')) - self.eq((f'ssl://root@aha.loop.vertex.link:{ahaport}',), yamlconf.get('aha:registry')) - self.eq(f'ssl://0.0.0.0:0?hostname=00.axon.loop.vertex.link&ca=loop.vertex.link', yamlconf.get('dmon:listen')) + self.eq((f'ssl://aha.loop.vertex.link:{ahaport}?certname=root@synapse',), yamlconf.get('aha:registry')) + self.eq(f'ssl://0.0.0.0:0?hostname=00.axon.synapse&ca=synapse', yamlconf.get('dmon:listen')) unfo = await axon.addUser('visi') @@ -678,9 +663,9 @@ async def test_lib_aha_provision(self): provurl = str(outp).split(':', 1)[1].strip() with self.getTestSynDir() as syndir: - capath = s_common.genpath(syndir, 'certs', 'cas', 'loop.vertex.link.crt') - crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.crt') - keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.key') + capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt') + crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt') + keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.key') for path in (capath, crtpath, keypath): s_common.genfile(path) @@ -693,7 +678,7 @@ async def test_lib_aha_provision(self): teleyaml = s_common.yamlload(syndir, 'telepath.yaml') self.eq(teleyaml.get('version'), 1) - self.eq(teleyaml.get('aha:servers'), (f'ssl://visi@aha.loop.vertex.link:{ahaport}',)) + self.eq(teleyaml.get('aha:servers'), (f'ssl://aha.loop.vertex.link:{ahaport}?certname=visi@synapse',)) certdir = s_telepath.s_certdir.CertDir(os.path.join(syndir, 'certs')) async with await s_telepath.openurl('aha://visi@axon...', certdir=certdir) as prox: @@ -776,7 +761,7 @@ async def test_lib_aha_provision(self): await axon2.sync() self.true(axon.isactive) self.false(axon2.isactive) - self.eq('aha://root@axon.loop.vertex.link', axon2.conf.get('mirror')) + self.eq('aha://root@axon...', axon2.conf.get('mirror')) with s_common.genfile(axn2path, 'prov.done') as fd: axon2providen = fd.read().decode().strip() @@ -787,7 +772,7 @@ async def test_lib_aha_provision(self): await axon2.sync() self.true(axon.isactive) self.false(axon2.isactive) - self.eq('aha://root@axon.loop.vertex.link', axon2.conf.get('mirror')) + self.eq('aha://root@axon...', axon2.conf.get('mirror')) # Provision a mirror using aha:provision in the mirror cell.yaml as well. # This is similar to the previous test block. @@ -815,7 +800,7 @@ async def test_lib_aha_provision(self): await axon03.sync() self.true(axon.isactive) self.false(axon03.isactive) - self.eq('aha://root@axon.loop.vertex.link', axon03.conf.get('mirror')) + self.eq('aha://root@axon...', axon03.conf.get('mirror')) with s_common.genfile(axn3path, 'prov.done') as fd: axon3providen = fd.read().decode().strip() @@ -831,15 +816,15 @@ async def test_lib_aha_provision(self): await axon3.sync() self.true(axon.isactive) self.false(axon3.isactive) - self.eq('aha://root@axon.loop.vertex.link', axon03.conf.get('mirror')) + self.eq('aha://root@axon...', axon03.conf.get('mirror')) retn, outp = await self.execToolMain(s_a_list._main, [aha.getLocalUrl()]) self.eq(retn, 0) outp.expect('Service network leader') - outp.expect('00.axon loop.vertex.link True') - outp.expect('01.axon loop.vertex.link False') - outp.expect('02.axon loop.vertex.link False') - outp.expect('axon loop.vertex.link True') + outp.expect('00.axon synapse True') + outp.expect('01.axon synapse False') + outp.expect('02.axon synapse False') + outp.expect('axon synapse True') # Ensure we can provision a service on a given listening ports outp.clear() @@ -854,14 +839,6 @@ async def test_lib_aha_provision(self): outp.expect('ERROR: Invalid HTTPS port: 123456') self.eq(1, ret) - outp.clear() - bad_conf_path = s_common.genpath(dirn, 'badconf.yaml') - s_common.yamlsave({'aha:network': 'aha.newp.net'}, bad_conf_path) - args = ('--url', aha.getLocalUrl(), 'bazfaz', '--cellyaml', bad_conf_path) - ret = await s_tools_provision_service.main(args, outp=outp) - outp.expect('ERROR: Provisioning aha:network must be equal to the Aha servers network') - self.eq(1, ret) - outp = self.getTestOutp() argv = ('--url', aha.getLocalUrl(), 'bazfaz', '--dmon-port', '1234', '--https-port', '443') await s_tools_provision_service.main(argv, outp=outp) @@ -876,16 +853,6 @@ async def test_lib_aha_provision(self): https_port = conf.get('https:port') self.eq(https_port, 443) - # provisioning against a network that differs from the aha network fails. - bad_netw = 'stuff.goes.beep' - provinfo = {'conf': {'aha:network': bad_netw}} - with self.raises(s_exc.BadConfValu) as cm: - async with self.addSvcToAha(aha, '00.exec', ExecTeleCaller, - provinfo=provinfo) as conn: - pass - self.isin('Provisioning aha:network must be equal to the Aha servers network', - cm.exception.get('mesg')) - # We can generate urls and then drop them en-mass. They will not usable. provurls = [] enrlursl = [] @@ -997,13 +964,6 @@ async def test_aha_httpapi(self): self.eq(info.get('status'), 'err') self.eq(info.get('code'), 'SchemaViolation') - # Break the Aha cell - not will provision after this. - _network = aha.conf.pop('aha:network') - async with sess.post(url, json={'name': '00.newp'}) as resp: - info = await resp.json() - self.eq(info.get('status'), 'err') - self.eq(info.get('code'), 'NeedConfValu') - # Not an admin await aha.addUser('lowuser', passwd='lowuser') async with self.getHttpSess(auth=('lowuser', 'lowuser'), port=httpsport) as sess: diff --git a/synapse/tools/aha/enroll.py b/synapse/tools/aha/enroll.py index 6b5b49095e8..061ee052cd3 100644 --- a/synapse/tools/aha/enroll.py +++ b/synapse/tools/aha/enroll.py @@ -76,7 +76,8 @@ async def main(argv, outp=s_output.stdout): if isinstance(ahaurls, str): ahaurls = (ahaurls,) - ahaurls = set(s_telepath.modurl(ahaurls, user=ahauser)) + certname = f'{ahauser}@{ahanetw}' + ahaurls = set(s_telepath.modurl(ahaurls, certname=certname)) servers = teleyaml.get('aha:servers') # repack the servers so lists are tuplized like values From cba0bfed7e84d9660e495e2ac1851947eda55e1e Mon Sep 17 00:00:00 2001 From: visi Date: Thu, 27 Jun 2024 17:50:51 -0400 Subject: [PATCH 02/86] wip --- docs/synapse/deploymentguide.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index 0326c835642..2b84671c5d1 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -108,9 +108,7 @@ Create the ``/srv/syn/aha/docker-compose.yaml`` file with contents:: environment: - SYN_AHA_HTTPS_PORT=null - SYN_AHA_AHA_NAME=aha - - SYN_AHA_AHA_NETWORK= - - SYN_AHA_DMON_LISTEN=ssl://aha.?ca= - - SYN_AHA_PROVISION_LISTEN=ssl://aha.:27272 + - SYN_AHA_DNS_NAME=aha. .. note:: From 8697e4ad67311c68bd6f9e8554d7ffebe5335d0c Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 28 Jun 2024 12:57:47 -0400 Subject: [PATCH 03/86] wip --- synapse/lib/aha.py | 290 +++++++++++++++++++++---- synapse/lib/cell.py | 35 +-- synapse/tests/test_axon.py | 2 +- synapse/tests/test_cortex.py | 9 +- synapse/tests/test_lib_aha.py | 256 ++++++---------------- synapse/tests/test_lib_cell.py | 8 +- synapse/tests/test_lib_stormlib_aha.py | 70 +++--- synapse/tests/utils.py | 150 +++++-------- 8 files changed, 444 insertions(+), 376 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index f3738eaf52e..b93d4c17833 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -12,6 +12,7 @@ import synapse.daemon as s_daemon import synapse.telepath as s_telepath +import synapse.lib.base as s_base import synapse.lib.cell as s_cell import synapse.lib.coro as s_coro import synapse.lib.nexus as s_nexus @@ -125,11 +126,19 @@ async def get(self): class AhaApi(s_cell.CellApi): - async def getAhaUrls(self): - ahaurls = self.cell._getAhaUrls() - if ahaurls is not None: - return ahaurls - return() + @s_cell.adminapi() + async def getAhaClone(self, iden): + return await self.cell.getAhaClone(iden) + + @s_cell.adminapi() + async def addAhaClone(self, host, port=27492, conf=None): + return await self.cell.addAhaClone(host, port=port, conf=conf) + + async def getAhaUrls(self, user='root'): + ahaurls = await self.cell.getAhaUrls(user=user) + if ahaurls is None: + return () + return ahaurls async def getAhaSvc(self, name, filters=None): ''' @@ -280,6 +289,10 @@ async def addAhaPoolSvc(self, poolname, svcname, info): async def delAhaPoolSvc(self, poolname, svcname): return await self.cell.delAhaPoolSvc(poolname, svcname) + async def iterAhaTopo(self): + async for item in self.cell.iterAhaTopo(): + yield item + async def iterPoolTopo(self, name): username = self.user.name.split('@')[0] @@ -299,6 +312,20 @@ async def getAhaPools(self): async for item in self.cell.getAhaPools(): yield item + async def getAhaServers(self): + return await self.cell.getAhaServers() + + async def getAhaServer(self, host, port): + return await self.cell.getAhaServer(host, port) + + @s_cell.adminapi() + async def addAhaServer(self, server): + return await self.cell.addAhaServer(server) + + @s_cell.adminapi() + async def delAhaServer(self, host, port): + return await self.cell.delAhaServer(host, port) + @s_cell.adminapi() async def addAhaSvcProv(self, name, provinfo=None): ''' @@ -366,10 +393,34 @@ async def _getSharedItem(self, name): await self.aha.delAhaUserEnroll(name) return EnrollApi(self.aha, userinfo) + clone = await self.aha.getAhaClone(name) + if clone is not None: + mesg = f'Retrieved AHA clone info for {name}' + host = clone.get('host') + logger.info(mesg, extra=await self.aha.getLogExtra(iden=name, host=host)) + return CloneApi(self.aha, clone) + mesg = f'Invalid provisioning identifier name={name}. This could be' \ f' caused by the re-use of a provisioning URL.' raise s_exc.NoSuchName(mesg=mesg, name=name) +class CloneApi: + + def __init__(self, aha, clone): + self.aha = aha + self.clone = clone + + async def getCloneDef(self): + return self.clone + + async def readyToMirror(self): + return await self.aha.readyToMirror() + + async def iterNewBackupArchive(self, name=None, remove=False): + async with self.aha.getLocalProxy() as proxy: + async for byts in proxy.iterNewBackupArchive(name=name, remove=remove): + yield byts + class EnrollApi: def __init__(self, aha, userinfo): @@ -377,9 +428,10 @@ def __init__(self, aha, userinfo): self.userinfo = userinfo async def getUserInfo(self): + user = self.userinfo.get('name') return { - 'aha:urls': self.aha._getAhaUrls(), - 'aha:user': self.userinfo.get('name'), + 'aha:urls': await self.aha.getAhaUrls(user=user), + 'aha:user': user, 'aha:network': self.aha._getAhaNetwork(), } @@ -464,12 +516,17 @@ class AhaCell(s_cell.Cell): confbase['mirror']['hidedocs'] = False # type: ignore confbase['mirror']['hidecmdl'] = False # type: ignore confdefs = { + 'clone': { + 'hidecmdl': True, + 'description': 'Bootstrap a clone from the AHA clone URL.', + 'type': ['string', 'null'], + }, 'dns:name': { 'description': 'The registered DNS name used to reach the AHA service.', 'type': ['string', 'null'], }, 'aha:urls': { - 'description': 'A list of all available AHA server URLs.', + 'description': 'Deprecated. AHA servers can now manage this automatically.', 'type': ['string', 'array'], 'items': {'type': 'string'}, }, @@ -484,6 +541,34 @@ class AhaCell(s_cell.Cell): def getEnvPrefix(cls): return (f'SYN_AHA', f'SYN_{cls.__name__.upper()}', ) + async def _initCellBoot(self): + + path = s_common.genpath(self.dirn, 'cell.guid') + if os.path.isfile(path): + return + + curl = self.conf.get('clone') + if curl is None: + return + + logger.warning(f'Cloning AHA: {curl}') + + async with await s_telepath.openurl(curl) as proxy: + clone = await proxy.getCloneDef() + await self._initCloneCell(proxy) + + logger.warning('Cloning AHA: done!') + + conf = s_common.yamlload(self.dirn, 'cell.yaml') + if conf is None: + conf = {} + + conf.update(clone.get('conf', {})) + + s_common.yamlsave(conf, self.dirn, 'cell.yaml') + + self.conf.update(conf) + async def initServiceStorage(self): # TODO plumb using a remote jsonstor? @@ -492,6 +577,7 @@ async def initServiceStorage(self): slab = await s_lmdbslab.Slab.anit(dirn) slab.addResizeCallback(self.checkFreeSpace) + self.topobus = await s_base.Base.anit() self.jsonstor = await s_jsonstor.JsonStor.anit(slab, 'aha') # type: s_jsonstor.JsonStor async def fini(): @@ -503,9 +589,103 @@ async def fini(): self.slab.initdb('aha:provs') self.slab.initdb('aha:enrolls') + self.slab.initdb('aha:clones') + self.slab.initdb('aha:servers') + self.slab.initdb('aha:pools') + + self.topowindows = [] self.poolwindows = collections.defaultdict(list) + async def getAhaServer(self, host, port): + lkey = s_msgpack.en((host, port)) + byts = self.slab.get(lkey, db='aha:servers') + if byts is not None: + return s_msgpack.un(byts) + + async def addAhaServer(self, server): + + host = server.get('host') + port = server.setdefault('port', 27492) + + # avoid a noop nexus change... + oldv = await self.getAhaServer(host, port) + if s_common.flatten(server) == s_common.flatten(oldv): + return False + + return await self._push('aha:server:add', server) + + async def _getTopoUpdate(self): + return ('aha:update', { + 'servers': await self.getAhaServers(), + }), + + async def _fireTopoUpdate(self): + update = await self._getTopoUpdate() + for wind in self.topowindows: + await wind.put(update) + + @s_nexus.Pusher.onPush('aha:server:add') + async def _addAhaServer(self, server): + # TODO schema + host = server.get('host') + port = server.get('port') + + oldv = None + lkey = s_msgpack.en((host, port)) + + byts = self.slab.get(lkey, db='aha:servers') + if byts is not None: + oldv = s_msgpack.un(byts) + if s_common.flatten(server) == s_common.flatten(oldv): + return False + + self.slab.put(lkey, s_msgpack.en(server), db='aha:servers') + + if self.isactive: + await self._fireTopoUpdate() + + return True + + @s_nexus.Pusher.onPushAuto('aha:server:del') + async def delAhaServer(self, host, port): + + lkey = s_msgpack.en((host, port)) + + byts = self.slab.pop(lkey, db='aha:servers') + if byts is None: + return None + + if self.isactive: + await self._fireTopoUpdate() + + return s_msgpack.un(byts) + + async def getAhaServers(self): + servers = [] + for _, byts in self.slab.scanByFull(db='aha:servers'): + servers.append(s_msgpack.un(byts)) + return servers + + async def iterAhaTopo(self): + + if not self.isactive: + async with await self.nexsroot.client.proxy() as proxy: + async for item in proxy.iterAhaTopo(): + yield item + + async with await s_queue.Window.anit(maxsize=1000) as wind: + + async def onfini(): + self.topowindows.remove(wind) + + wind.onfini(onfini) + + async with self.nexsroot.applylock: + update = await self._getTopoUpdate() + await wind.put(update) + self.topowindows.append(wind) + async def iterPoolTopo(self, name): name = self._getAhaName(name) @@ -560,11 +740,12 @@ async def initServiceRuntime(self): logger.info(f'Adding server certificate for {host}') await self._genHostCert(host, signas=netw) + root = f'root@{netw}' + await self._genUserCert(root, signas=netw) + user = self._getAhaAdmin() if user is not None: - if self.certdir.getUserCertPath(user) is None: - logger.info(f'Adding user certificate for {user}@{netw}') - await self._genUserCert(user, signas=netw) + await self._genUserCert(user, signas=netw) def _getDnsName(self): # emulate the old aha name.network behavior if the @@ -593,8 +774,8 @@ def _getProvListen(self): def _getDmonListen(self): - lisn = self.conf.get('dmon:listen') - if lisn is not None: + lisn = self.conf.get('dmon:listen', s_common.novalu) + if lisn is not s_common.novalu: return lisn network = self._getAhaNetwork() @@ -623,6 +804,11 @@ async def initServiceNetwork(self): await s_cell.Cell.initServiceNetwork(self) + # all AHA mirrors are registered + if hostname is not None and self.sockaddr is not None: + server = {'host': hostname, 'port': self.sockaddr[1]} + await self.addAhaServer(server) + self.provdmon = None provurl = self._getProvListen() @@ -712,6 +898,7 @@ async def addAhaSvc(self, name, info, network=None): # mostly for testing... await self.fire('aha:svcadd', svcinfo=svcinfo) + await self.fire(f'aha:svcadd:{svcfull}', svcinfo=svcinfo) def _getAhaName(self, name): # the modern version of names is absolute or ... @@ -994,8 +1181,12 @@ async def _genHostCert(self, hostname, signas=None): await self.saveHostCert(hostname, pkey, cert) async def _genUserCert(self, username, signas=None): + if os.path.isfile(os.path.join(self.dirn, 'certs', 'users', '{username}.crt')): return + + logger.info(f'Adding user certificate for {username}') + pkey, cert = await s_coro.executor(self.certdir.genUserCert, username, signas=signas, save=False) pkey = self.certdir._pkeyToByts(pkey).decode() cert = self.certdir._certToByts(cert).decode() @@ -1071,40 +1262,67 @@ async def signUserCsr(self, csrtext, signas=None): return self.certdir._certToByts(cert).decode() - def _getAhaUrls(self, user='root'): + async def getAhaUrls(self, user='root'): + # for backward compat... urls = self.conf.get('aha:urls') if urls is not None: - if isinstance(urls, str): - return (urls,) return urls - ahaname = self.conf.get('aha:name') - if ahaname is None: - return None + network = self._getAhaNetwork() + urls = [] + for server in await self.getAhaServers(): + host = server.get('host') + port = server.get('port') + urls.append(f'ssl://{host}:{port}?certname={user}@{network}') + return urls + + def _getAhaUrl(self, user='root'): + port = self.sockaddr[1] + host = self._getDnsName() + network = self._getAhaNetwork() + return f'ssl://{host}:{port}?certname={user}@{network}' - ahanetw = self._getAhaNetwork() + async def getAhaClone(self, iden): + lkey = s_common.uhex(iden) + byts = self.slab.get(lkey, db='aha:clones') + if byts is not None: + return s_msgpack.un(byts) - if self.sockaddr is None or not isinstance(self.sockaddr, tuple): - return None + async def addAhaClone(self, host, port=27492, conf=None): + + if conf is None: + conf = {} + + network = self._getAhaNetwork() + + conf['mirror'] = self._getAhaUrl() + + conf['dns:name'] = host + conf['aha:network'] = network + conf['dmon:listen'] = f'ssl://0.0.0.0:{port}?hostname={host}&ca={network}' - # FIXME this is the problematic use case - # FIXME may need to generate an AHA CA signed server cert for DNS NAME + iden = s_common.guid() + clone = { + 'iden': iden, + 'host': host, + 'port': port, + 'conf': conf, + } + return await self._push('aha:clone:add', clone) - # TODO this could eventually enumerate others via itself - netloc = self._getDnsName() - return (f'ssl://{netloc}:{self.sockaddr[1]}?certname={user}@{ahanetw}',) + @s_nexus.Pusher.onPush('aha:clone:add') + async def _addAhaClone(self, clone): + iden = clone.get('iden') + lkey = s_common.uhex(iden) + self.slab.put(lkey, s_msgpack.en(clone), db='aha:clones') + return self._getProvClientUrl(iden) async def addAhaSvcProv(self, name, provinfo=None): if not name: raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.') - ahaurls = self._getAhaUrls() - if ahaurls is None: - mesg = 'AHA server has no configured aha:urls.' - raise s_exc.NeedConfValu(mesg=mesg) - self._reqProvListen() if provinfo is None: @@ -1122,7 +1340,8 @@ async def addAhaSvcProv(self, name, provinfo=None): if ahaadmin is not None: # pragma: no cover conf.setdefault('aha:admin', ahaadmin) - conf.setdefault('aha:user', 'root') + ahauser = conf.setdefault('aha:user', 'root') + ahaurls = await self.getAhaUrls(user=ahauser) conf['aha:network'] = netw @@ -1148,11 +1367,6 @@ async def addAhaSvcProv(self, name, provinfo=None): if peer: conf.setdefault('aha:leader', leader) - # allow user to win over leader - ahauser = conf.get('aha:user') - certname = f'{ahauser}@{netw}' - ahaurls = s_telepath.modurl(ahaurls, certname=certname) - conf.setdefault('aha:registry', ahaurls) mirname = provinfo.get('mirror') diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index db3a66669f5..85b1b65507f 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1174,8 +1174,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): self.backlastexc = None # err, errmsg, errtrace of last backup if self.conf.get('mirror') and not self.conf.get('nexslog:en'): - mesg = 'Mirror mode requires nexslog:en=True' - raise s_exc.BadConfValu(mesg=mesg) + self.modCellConf({'nexslog:en': True}) # construct our nexsroot instance ( but do not start it ) await s_nexus.Pusher.__anit__(self, self.iden) @@ -1521,8 +1520,8 @@ def _getAhaNetwork(self): def _getDmonListen(self): - lisn = self.conf.get('dmon:listen') - if lisn is not None: + lisn = self.conf.get('dmon:listen', s_common.novalu) + if lisn is not s_common.novalu: return lisn network = self._getAhaNetwork() @@ -3407,6 +3406,7 @@ def getArgParser(cls, conf=None): return pars async def _initCellBoot(self): + # NOTE: best hook point for custom provisioning pnfo = await self._bootCellProv() @@ -3700,23 +3700,14 @@ async def readyToMirror(self): await self.nexsroot.enNexsLog() await self.sync() - async def _bootCellMirror(self, pnfo): - # this function must assume almost nothing is initialized - # but that's ok since it will only run rarely. - # It assumes it has a tuple of (provisioning configuration, provisioning iden) available - murl = self.conf.reqConfValu('mirror') - provconf, providen = pnfo - - logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)') + async def _initCloneCell(self, proxy): tarpath = s_common.genpath(self.dirn, 'tmp', 'bootstrap.tgz') - try: - async with await s_telepath.openurl(murl) as cell: - await cell.readyToMirror() + await proxy.readyToMirror() with s_common.genfile(tarpath) as fd: - async for byts in cell.iterNewBackupArchive(remove=True): + async for byts in proxy.iterNewBackupArchive(remove=True): fd.write(byts) with tarfile.open(tarpath) as tgz: @@ -3731,6 +3722,18 @@ async def _bootCellMirror(self, pnfo): if os.path.isfile(tarpath): os.unlink(tarpath) + async def _bootCellMirror(self, pnfo): + # this function must assume almost nothing is initialized + # but that's ok since it will only run rarely. + # It assumes it has a tuple of (provisioning configuration, provisioning iden) available + murl = self.conf.reqConfValu('mirror') + provconf, providen = pnfo + + logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)') + + async with await s_telepath.openurl(murl) as proxy: + await self._initCloneCell(proxy) + # Remove aha:provision from cell.yaml if it exists and the iden differs. mnfo = s_common.yamlload(self.dirn, 'cell.yaml') if mnfo: diff --git a/synapse/tests/test_axon.py b/synapse/tests/test_axon.py index ceb0d5cbbd6..f618e7edf58 100644 --- a/synapse/tests/test_axon.py +++ b/synapse/tests/test_axon.py @@ -1133,7 +1133,7 @@ async def test_axon_blob_v00_v01(self): async def test_axon_mirror(self): - async with self.getTestAhaProv() as aha: + async with self.getTestAha() as aha: axon00dirn = s_common.gendir(aha.dirn, 'tmp', 'axon00') axon01dirn = s_common.gendir(aha.dirn, 'tmp', 'axon01') diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index 5acb96bdffb..cdafc355a0e 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -7904,7 +7904,8 @@ async def test_cortex_ext_httpapi(self): await core.delHttpExtApi('notAGuid') async def test_cortex_query_offload(self): - async with self.getTestAhaProv() as aha: + + async with self.getTestAha() as aha: async with await s_base.Base.anit() as base: @@ -7927,11 +7928,11 @@ async def test_cortex_query_offload(self): msgs = await core00.stormlist('aha.pool.add pool00...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('Created AHA service pool: pool00.loop.vertex.link', msgs) + self.stormIsInPrint('Created AHA service pool: pool00.synapse', msgs) msgs = await core00.stormlist('aha.pool.svc.add pool00... 01.core...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.loop.vertex.link)', msgs) + self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.synapse)', msgs) msgs = await core00.stormlist('cortex.storm.pool.set newp') self.stormIsInErr(':// not found in [newp]', msgs) @@ -8079,7 +8080,7 @@ async def test_cortex_query_offload(self): waiter = core01.stormpool.waiter(1, 'svc:del') msgs = await core01.stormlist('aha.pool.svc.del pool00... 01.core...', opts={'mirror': False}) self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (01.core.loop.vertex.link) removed from service pool (pool00.loop.vertex.link)', msgs) + self.stormIsInPrint('AHA service (01.core.synapse) removed from service pool (pool00.synapse)', msgs) # TODO: this wait should not return None await waiter.wait(timeout=3) diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 9d2f4a9f06e..0e10fb7bf65 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -44,30 +44,37 @@ async def exectelecall(self, url, meth, *args, **kwargs): class AhaTest(s_test.SynTest): - async def test_lib_aha_mirrors(self): + async def test_lib_aha_clone(self): with self.getTestDir() as dirn: + dir0 = s_common.gendir(dirn, 'aha0') dir1 = s_common.gendir(dirn, 'aha1') - conf = {'nexslog:en': True} + async with self.getTestAha(dirn=dir0) as aha0: - async with self.getTestAha(conf={'nexslog:en': True}, dirn=dir0) as aha0: - user = await aha0.auth.addUser('reguser', passwd='secret') - await user.setAdmin(True) + async with aha0.getLocalProxy() as proxy0: + self.len(1, await proxy0.getAhaUrls()) + self.len(1, await proxy0.getAhaServers()) - s_tools_backup.backup(dir0, dir1) + purl = await proxy0.addAhaClone('01.aha.loop.vertex.link') - async with self.getTestAha(conf=conf, dirn=dir0) as aha0: - upstream_url = aha0.getLocalUrl() + conf1 = {'clone': purl} + async with self.getTestAha(conf=conf1, dirn=dir1) as aha1: - mirrorconf = { - 'nexslog:en': True, - 'mirror': upstream_url, - } + await aha1.sync() + + self.eq(aha0.iden, aha1.iden) + self.nn(aha1.conf.get('mirror')) + + serv0 = await aha0.getAhaServers() + serv1 = await aha1.getAhaServers() + + self.len(2, serv0) + self.eq(serv0, serv1) + + # ensure some basic functionality is being properly mirrored - async with self.getTestAha(conf=mirrorconf, dirn=dir1) as aha1: - # CA is nexus-fied cabyts = await aha0.genCaCert('mirrorca') await aha1.sync() mirbyts = await aha1.genCaCert('mirrorca') @@ -98,27 +105,22 @@ async def test_lib_aha_mirrors(self): async def test_lib_aha_offon(self): with self.getTestDir() as dirn: cryo0_dirn = s_common.gendir(dirn, 'cryo0') - conf = {'auth:passwd': 'secret'} - async with self.getTestAha(conf=conf.copy(), dirn=dirn) as aha: - host, port = await aha.dmon.listen('tcp://127.0.0.1:0') + async with self.getTestAha(dirn=dirn) as aha: + purl = await aha.addAhaSvcProv('0.cryo') wait00 = aha.waiter(1, 'aha:svcadd') - cryo_conf = { - 'aha:name': '0.cryo', - 'aha:registry': f'tcp://root:secret@127.0.0.1:{port}', - 'dmon:listen': 'tcp://0.0.0.0:0/', - } - async with self.getTestCryo(dirn=cryo0_dirn, conf=cryo_conf) as cryo: + + conf = {'aha:provision': purl} + async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo: self.isin(len(await wait00.wait(timeout=6)), (1, 2)) svc = await aha.getAhaSvc('0.cryo...') linkiden = svc.get('svcinfo', {}).get('online') - self.nn(linkiden) # Tear down the Aha cell. await aha.__aexit__(None, None, None) - async with self.getTestAha(conf=conf.copy(), dirn=dirn) as aha: + async with self.getTestAha(dirn=dirn) as aha: wait01 = aha.waiter(1, 'aha:svcdown') await wait01.wait(timeout=6) svc = await aha.getAhaSvc('0.cryo...') @@ -377,15 +379,16 @@ async def test_lib_aha(self): uinfo = ahainfo.get('urlinfo', {}) self.eq(uinfo.get('scheme'), 'unix') self.none(uinfo.get('port')) - self.none(aha._getAhaUrls()) + self.len(1, await aha.getAhaUrls()) conf['dmon:listen'] = 'tcp://0.0.0.0:0/' async with self.getTestAha(conf=conf) as aha: ahainfo = await aha.getAhaInfo() + ahaurls = await aha.getAhaUrls() uinfo = ahainfo.get('urlinfo', {}) self.eq(uinfo.get('scheme'), 'tcp') self.gt(uinfo.get('port'), 0) - self.eq(aha._getAhaUrls()[0], f'ssl://0.test.foo:{aha.sockaddr[1]}?certname=root@foo') + self.eq(ahaurls[0], f'ssl://00.aha.loop.vertex.link:{aha.sockaddr[1]}?certname=root@foo') async def test_lib_aha_loadenv(self): @@ -430,53 +433,23 @@ async def test_lib_aha_finid_cell(self): async with self.getTestAha() as aha: - cryo0_dirn = s_common.gendir(aha.dirn, 'cryo0') - - host, port = await aha.dmon.listen('tcp://127.0.0.1:0') - await aha.auth.rootuser.setPasswd('hehehaha') - - aharegistry = [f'tcp://root:hehehaha@127.0.0.1:{port}', - f'tcp://root:hehehaha@127.0.0.1:{port}'] - atup = tuple(aharegistry) - wait00 = aha.waiter(1, 'aha:svcadd') - conf = { - 'aha:name': '0.cryo', - 'aha:admin': 'root@synapse', - 'aha:registry': aharegistry, - 'dmon:listen': 'tcp://0.0.0.0:0/', - } - async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo: + conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')} - await cryo.auth.rootuser.setPasswd('secret') - - ahaadmin = await cryo.auth.getUserByName('root@synapse') - self.nn(ahaadmin) - self.true(ahaadmin.isAdmin()) - - await wait00.wait(timeout=2) + async with self.getTestCryo(conf=conf) as cryo: - self.isin(atup, s_telepath.aha_clients) + self.true(await wait00.wait(timeout=2)) - async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: + async with await s_telepath.openurl('aha://0.cryo...') as proxy: self.nn(await proxy.getCellIden()) - _ahaclient = s_telepath.aha_clients.get(atup).get('client') - _aprx = await _ahaclient.proxy() + proxy = await cryo.ahaclient.proxy() await aha.fini() - self.true(await _aprx.waitfini(timeout=10)) - - orig = s_telepath.Client.proxy - async def quickproxy(self, timeout): - return await orig(self, timeout=0.1) - - with mock.patch('synapse.telepath.Client.proxy', quickproxy): - with self.raises(asyncio.TimeoutError): - - async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: - self.fail('Should never reach a connection.') + self.true(await proxy.waitfini(timeout=10)) + with self.raises(asyncio.TimeoutError): + await cryo.ahaclient.proxy(timeout=0.1) async def test_lib_aha_onlink_fail(self): @@ -484,22 +457,10 @@ async def test_lib_aha_onlink_fail(self): async with self.getTestAha() as aha: - cryo0_dirn = s_common.gendir(aha.dirn, 'cryo0') - - host, port = await aha.dmon.listen('tcp://127.0.0.1:0') - await aha.auth.rootuser.setPasswd('secret') - aha.testerr = True - wait00 = aha.waiter(1, 'aha:svcadd') - conf = { - 'aha:name': '0.cryo', - 'aha:registry': f'tcp://root:secret@127.0.0.1:{port}', - 'dmon:listen': 'tcp://0.0.0.0:0/', - } - async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo: - - await cryo.auth.rootuser.setPasswd('secret') + conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')} + async with self.getTestCryo(conf=conf) as cryo: self.none(await wait00.wait(timeout=2)) @@ -515,7 +476,7 @@ async def test_lib_aha_onlink_fail(self): self.nn(svc) self.nn(svc.get('svcinfo', {}).get('online')) - async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: + async with await s_telepath.openurl('aha://0.cryo...') as proxy: self.nn(await proxy.getCellIden()) async def test_lib_aha_bootstrap(self): @@ -545,7 +506,8 @@ async def test_lib_aha_bootstrap(self): async def test_lib_aha_noconf(self): - async with self.getTestAha() as aha: + conf = {'dns:name': None} + async with self.getTestAha(conf=conf) as aha: with self.raises(s_exc.NeedConfValu): await aha.addAhaSvcProv('hehe') @@ -572,7 +534,6 @@ async def test_lib_aha_provision(self): async with self.getTestAha(dirn=dirn, conf=conf) as aha: ahaport = aha.sockaddr[1] - aha.conf['aha:urls'] = f'ssl://aha.loop.vertex.link:{ahaport}' url = aha.getLocalUrl() @@ -876,21 +837,9 @@ async def test_lib_aha_provision(self): async def test_aha_httpapi(self): - conf = { - 'aha:name': 'aha', - 'aha:network': 'loop.vertex.link', - 'provision:listen': 'ssl://aha.loop.vertex.link:0' - } - async with self.getTestAha(conf=conf) as aha: - await aha.auth.rootuser.setPasswd('secret') - - addr, port = aha.provdmon.addr - # update the config to reflect the dynamically bound port - aha.conf['provision:listen'] = f'ssl://aha.loop.vertex.link:{port}' + async with self.getTestAha() as aha: - # do this config ex-post-facto due to port binding... - host, ahaport = await aha.dmon.listen('ssl://0.0.0.0:0?hostname=aha.loop.vertex.link&ca=loop.vertex.link') - aha.conf['aha:urls'] = f'ssl://aha.loop.vertex.link:{ahaport}' + await aha.auth.rootuser.setPasswd('secret') host, httpsport = await aha.addHttpsPort(0) url = f'https://localhost:{httpsport}/api/v1/aha/provision/service' @@ -973,15 +922,11 @@ async def test_aha_httpapi(self): self.eq(info.get('code'), 'AuthDeny') async def test_aha_connect_back(self): - async with self.getTestAhaProv() as aha: # type: s_aha.AhaCell - async with self.addSvcToAha(aha, '00.exec', ExecTeleCaller) as conn: + async with self.getTestAha() as aha: # type: s_aha.AhaCell - ahaurl = aha.conf.get('aha:urls')[0] - ahaurl = s_telepath.modurl(ahaurl, user='root') - - # This adminapi fails if the ssl://root@aha.loop.vertex.link - # session is not an admin user. + async with self.addSvcToAha(aha, '00.exec', ExecTeleCaller) as conn: + ahaurl = aha._getAhaUrl() await conn.exectelecall(ahaurl, 'getNexsIndx') self.true(conn.ahaclient.isfini) @@ -989,26 +934,7 @@ async def test_aha_connect_back(self): async def test_aha_util_helpers(self): # Mainly for test helper coverage. - - async with self.getTestAhaProv(conf={'auth:passwd': 'secret'}) as aha: # type: s_aha.AhaCell - root = await aha.auth.getUserByName('root') - self.true(await root.tryPasswd('secret')) - - import synapse.cortex as s_cortex - - with self.getTestDir() as dirn: - cdr0 = s_common.genpath(dirn, 'core00') - cdr1 = s_common.genpath(dirn, 'core01') - - async with self.addSvcToAha(aha, '00.core', s_cortex.Cortex, dirn=cdr0) as core00: - async with self.addSvcToAha(aha, '01.core', s_cortex.Cortex, dirn=cdr1, - provinfo={'mirror': 'core'}) as core01: - self.len(1, await core00.nodes('[inet:asn=0]')) - await core01.sync() - self.len(1, await core01.nodes('inet:asn=0')) - - # Simple test setups should work without issue - async with self.getTestAhaProv() as aha: + async with self.getTestAha() as aha: async with self.addSvcToAha(aha, '00.cell', s_cell.Cell) as cell00: # type: s_cell.Cell async with self.addSvcToAha(aha, '01.cell', s_cell.Cell, provinfo={'mirror': 'cell'}) as cell01: # type: s_cell.Cell @@ -1023,26 +949,12 @@ async def test_aha_restart(self): svc0dirn = s_common.gendir(dirn, 'svc00') svc1dirn = s_common.gendir(dirn, 'svc01') async with await s_base.Base.anit() as cm: - aconf = { - 'aha:name': 'aha', - 'aha:network': 'loop.vertex.link', - 'provision:listen': 'ssl://aha.loop.vertex.link:0' - } - name = aconf.get('aha:name') - netw = aconf.get('aha:network') - dnsname = f'{name}.{netw}' + + aconf = {'dns:name': 'aha.loop.vertex.link'} aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf) await cm.enter_context(aha) - addr, port = aha.provdmon.addr - # update the config to reflect the dynamically bound port - aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}' - - # do this config ex-post-facto due to port binding... - host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}') - aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',) - onetime = await aha.addAhaSvcProv('00.svc', provinfo=None) sconf = {'aha:provision': onetime} s_common.yamlsave(sconf, svc0dirn, 'cell.yaml') @@ -1060,7 +972,7 @@ async def test_aha_restart(self): await svc1.sync() # Get Aha services - snfo = await aha.getAhaSvc('01.svc.loop.vertex.link') + snfo = await aha.getAhaSvc('01.svc...') svcinfo = snfo.get('svcinfo') ready = svcinfo.get('ready') self.true(ready) @@ -1068,20 +980,12 @@ async def test_aha_restart(self): # Fini the Aha service. await aha.fini() - # Reuse our listening port we just deployed services with - aconf = { - 'aha:name': 'aha', - 'aha:network': 'loop.vertex.link', - 'provision:listen': 'ssl://aha.loop.vertex.link:0', # we do not care about provisioning - 'dmon:listen': f'ssl://{dnsname}:{ahaport}?hostname={dnsname}&ca={netw}' - } - # Restart aha aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf) await cm.enter_context(aha) # services are cleared - snfo = await aha.getAhaSvc('01.svc.loop.vertex.link') + snfo = await aha.getAhaSvc('01.svc...') svcinfo = snfo.get('svcinfo') ready = svcinfo.get('ready') online = svcinfo.get('online') @@ -1096,7 +1000,7 @@ async def test_aha_restart(self): self.ge(len(await waiter.wait(timeout=12)), n) # svc01 has reconnected and the ready state has been re-registered - snfo = await aha.getAhaSvc('01.svc.loop.vertex.link') + snfo = await aha.getAhaSvc('01.svc...') svcinfo = snfo.get('svcinfo') ready = svcinfo.get('ready') online = svcinfo.get('online') @@ -1105,7 +1009,7 @@ async def test_aha_restart(self): async def test_aha_service_pools(self): - async with self.getTestAhaProv() as aha: + async with self.getTestAha() as aha: async with await s_base.Base.anit() as base: @@ -1126,7 +1030,7 @@ async def test_aha_service_pools(self): msgs = await core00.stormlist('aha.pool.add pool00...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('Created AHA service pool: pool00.loop.vertex.link', msgs) + self.stormIsInPrint('Created AHA service pool: pool00.synapse', msgs) # Pool has no members.... pool = await s_telepath.open('aha://pool00...') @@ -1135,7 +1039,7 @@ async def test_aha_service_pools(self): msgs = await core00.stormlist('aha.pool.svc.add pool00... 00...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (00...) added to service pool (pool00.loop.vertex.link)', msgs) + self.stormIsInPrint('AHA service (00...) added to service pool (pool00.synapse)', msgs) self.len(1, await waiter.wait(timeout=12)) prox = await pool.proxy(timeout=12) @@ -1150,12 +1054,12 @@ async def test_aha_service_pools(self): self.len(1, poolinfo['services']) msgs = await core00.stormlist('aha.pool.list') - self.stormIsInPrint('Pool: pool00.loop.vertex.link', msgs) - self.stormIsInPrint(' 00.loop.vertex.link', msgs) + self.stormIsInPrint('Pool: pool00.synapse', msgs) + self.stormIsInPrint(' 00.synapse', msgs) self.stormIsInPrint('1 pools', msgs) - msgs = await core00.stormlist('$lib.print($lib.aha.pool.get(pool00.loop.vertex.link))') - self.stormIsInPrint('aha:pool: pool00.loop.vertex.link', msgs) + msgs = await core00.stormlist('$lib.print($lib.aha.pool.get(pool00.synapse))') + self.stormIsInPrint('aha:pool: pool00.synapse', msgs) async with await s_telepath.open('aha://pool00...') as pool: @@ -1166,11 +1070,11 @@ async def test_aha_service_pools(self): msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (01...) added to service pool (pool00.loop.vertex.link)', msgs) + self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs) msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (01...) added to service pool (pool00.loop.vertex.link)', msgs) + self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs) await waiter.wait(timeout=3) @@ -1178,12 +1082,12 @@ async def test_aha_service_pools(self): self.len(2, poolinfo['services']) self.nn(poolinfo['created']) - self.nn(poolinfo['services']['00.loop.vertex.link']['created']) - self.nn(poolinfo['services']['01.loop.vertex.link']['created']) + self.nn(poolinfo['services']['00.synapse']['created']) + self.nn(poolinfo['services']['01.synapse']['created']) self.eq(core00.auth.rootuser.iden, poolinfo['creator']) - self.eq(core00.auth.rootuser.iden, poolinfo['services']['00.loop.vertex.link']['creator']) - self.eq(core00.auth.rootuser.iden, poolinfo['services']['01.loop.vertex.link']['creator']) + self.eq(core00.auth.rootuser.iden, poolinfo['services']['00.synapse']['creator']) + self.eq(core00.auth.rootuser.iden, poolinfo['services']['01.synapse']['creator']) for client in pool.clients.values(): await client.proxy(timeout=3) @@ -1205,7 +1109,7 @@ async def test_aha_service_pools(self): msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (00.loop.vertex.link) removed from service pool (pool00.loop.vertex.link)', + self.stormIsInPrint('AHA service (00.synapse) removed from service pool (pool00.synapse)', msgs) msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...') @@ -1221,7 +1125,7 @@ async def test_aha_service_pools(self): msgs = await core00.stormlist('aha.pool.del pool00...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('Removed AHA service pool: pool00.loop.vertex.link', msgs) + self.stormIsInPrint('Removed AHA service pool: pool00.synapse', msgs) async def test_aha_reprovision(self): with self.withNexusReplay() as stack: @@ -1234,23 +1138,14 @@ async def test_aha_reprovision(self): aconf = { 'aha:name': 'aha', 'aha:network': 'loop.vertex.link', - 'provision:listen': 'ssl://aha.loop.vertex.link:0' + 'dns:name': 'aha.loop.vertex.link', } name = aconf.get('aha:name') netw = aconf.get('aha:network') - dnsname = f'{name}.{netw}' aha = await s_aha.AhaCell.anit(aha00dirn, conf=aconf) await cm.enter_context(aha) - addr, port = aha.provdmon.addr - # update the config to reflect the dynamically bound port - aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}' - - # do this config ex-post-facto due to port binding... - host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}') - aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',) - onetime = await aha.addAhaSvcProv('00.svc', provinfo=None) sconf = {'aha:provision': onetime} s_common.yamlsave(sconf, svc0dirn, 'cell.yaml') @@ -1281,23 +1176,14 @@ async def test_aha_reprovision(self): aconf = { 'aha:name': 'aha', 'aha:network': 'loop.vertex.link', - 'provision:listen': 'ssl://aha.loop.vertex.link:0' + 'dns:name': 'aha.loop.vertex.link', } name = aconf.get('aha:name') netw = aconf.get('aha:network') - dnsname = f'{name}.{netw}' aha = await s_aha.AhaCell.anit(aha01dirn, conf=aconf) await cm.enter_context(aha) - addr, port = aha.provdmon.addr - # update the config to reflect the dynamically bound port - aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}' - - # do this config ex-post-facto due to port binding... - host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}') - aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',) - onetime = await aha.addAhaSvcProv('00.svc', provinfo=None) sconf = {'aha:provision': onetime} s_common.yamlsave(sconf, svc0dirn, 'cell.yaml') diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index 306b9df07d6..03659508394 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -1811,9 +1811,7 @@ async def test_backup_restore_aha(self): # ensure the mirror has a # backup the mirror # restore the backup - async with self.getTestAhaProv(conf={'auth:passwd': 'secret'}) as aha: # type: s_aha.AhaCell - root = await aha.auth.getUserByName('root') - self.true(await root.tryPasswd('secret')) + async with self.getTestAha() as aha: # type: s_aha.AhaCell with self.getTestDir() as dirn: cdr0 = s_common.genpath(dirn, 'core00') @@ -1896,9 +1894,7 @@ async def test_backup_restore_double_promote_aha(self): # ensure the mirror has a # backup the mirror # restore the backup - async with self.getTestAhaProv(conf={'auth:passwd': 'secret'}) as aha: # type: s_aha.AhaCell - root = await aha.auth.getUserByName('root') - self.true(await root.tryPasswd('secret')) + async with self.getTestAha() as aha: # type: s_aha.AhaCell with self.getTestDir() as dirn: cdr0 = s_common.genpath(dirn, 'core00') diff --git a/synapse/tests/test_lib_stormlib_aha.py b/synapse/tests/test_lib_stormlib_aha.py index 9c17a317e02..c423088df96 100644 --- a/synapse/tests/test_lib_stormlib_aha.py +++ b/synapse/tests/test_lib_stormlib_aha.py @@ -10,7 +10,7 @@ class AhaLibTest(s_test.SynTest): async def test_stormlib_aha_basics(self): - async with self.getTestAhaProv() as aha: + async with self.getTestAha() as aha: with self.getTestDir() as dirn: @@ -36,24 +36,24 @@ async def test_stormlib_aha_basics(self): self.len(5, svcs) svc = await core00.callStorm('return( $lib.aha.get(core...) )') - self.eq('core.loop.vertex.link', svc.get('name')) - svc = await core00.callStorm('return( $lib.aha.get(core.loop.vertex.link))') - self.eq('core.loop.vertex.link', svc.get('name')) + self.eq('core.synapse', svc.get('name')) + svc = await core00.callStorm('return( $lib.aha.get(core.synapse))') + self.eq('core.synapse', svc.get('name')) svc = await core00.callStorm('return( $lib.aha.get(00.cell...))') - self.eq('00.cell.loop.vertex.link', svc.get('name')) + self.eq('00.cell.synapse', svc.get('name')) svc = await core00.callStorm('return( $lib.aha.get(cell...))') - self.eq('cell.loop.vertex.link', svc.get('name')) + self.eq('cell.synapse', svc.get('name')) svc = await core00.callStorm('$f=({"mirror": (true)}) return( $lib.aha.get(cell..., filters=$f))') - self.eq('01.cell.loop.vertex.link', svc.get('name')) + self.eq('01.cell.synapse', svc.get('name')) # List the aha services available msgs = await core00.stormlist('aha.svc.list --nexus') self.stormIsInPrint('Nexus', msgs) - self.stormIsInPrint('00.cell.loop.vertex.link true true true', msgs) - self.stormIsInPrint('01.cell.loop.vertex.link false true true', msgs) - self.stormIsInPrint('cell.loop.vertex.link true true true', msgs) - self.stormIsInPrint('core.loop.vertex.link null true true', msgs) - self.stormIsInPrint('mysvc.loop.vertex.link null true true', msgs) + self.stormIsInPrint('00.cell.synapse true true true', msgs, whitespace=False) + self.stormIsInPrint('01.cell.synapse false true true', msgs, whitespace=False) + self.stormIsInPrint('cell.synapse true true true', msgs, whitespace=False) + self.stormIsInPrint('core.synapse null true true', msgs, whitespace=False) + self.stormIsInPrint('mysvc.synapse null true true', msgs, whitespace=False) msgs = await core00.stormlist('aha.svc.list') self.stormNotInPrint('Nexus', msgs) @@ -63,24 +63,24 @@ async def test_stormlib_aha_basics(self): # Omit checking part of that. emsg = '''Resolved cell... to an AHA Service. -Name: cell.loop.vertex.link +Name: cell.synapse Online: true Ready: true Run iden: ******************************** Cell iden: ******************************** Leader: cell Connection information: - ca: loop.vertex.link + ca: synapse ''' self.stormIsInPrint(emsg, msgs, deguid=True) - self.stormIsInPrint(' hostname: 00.cell.loop.vertex.link', msgs) + self.stormIsInPrint(' hostname: 00.cell.synapse', msgs) self.stormIsInPrint(' scheme: ssl', msgs) self.stormIsInPrint(' user: root', msgs) msgs = await core00.stormlist('aha.svc.stat --nexus cell...') emsg = '''Resolved cell... to an AHA Service. -Name: cell.loop.vertex.link +Name: cell.synapse Online: true Ready: true Run iden: ******************************** @@ -88,14 +88,14 @@ async def test_stormlib_aha_basics(self): Leader: cell Nexus: 1 Connection information: - ca: loop.vertex.link + ca: synapse ''' self.stormIsInPrint(emsg, msgs, deguid=True) msgs = await core00.stormlist('aha.svc.stat --nexus 01.cell...') emsg = '''Resolved 01.cell... to an AHA Service. -Name: 01.cell.loop.vertex.link +Name: 01.cell.synapse Online: true Ready: true Run iden: ******************************** @@ -103,15 +103,15 @@ async def test_stormlib_aha_basics(self): Leader: cell Nexus: 1 Connection information: - ca: loop.vertex.link + ca: synapse ''' self.stormIsInPrint(emsg, msgs, deguid=True) # Full name works - msgs = await core00.stormlist('aha.svc.stat 01.cell.loop.vertex.link') - emsg = '''Resolved 01.cell.loop.vertex.link to an AHA Service. + msgs = await core00.stormlist('aha.svc.stat 01.cell.synapse') + emsg = '''Resolved 01.cell.synapse to an AHA Service. -Name: 01.cell.loop.vertex.link +Name: 01.cell.synapse ''' self.stormIsInPrint(emsg, msgs, deguid=True) @@ -128,8 +128,8 @@ async def test_stormlib_aha_basics(self): emsg = '''Resolved pool00... to an AHA Pool. The pool currently has 1 members. -AHA Pool: pool00.loop.vertex.link -Member: 00.cell.loop.vertex.link''' +AHA Pool: pool00.synapse +Member: 00.cell.synapse''' self.stormIsInPrint(emsg, msgs) # Shut down a service @@ -139,16 +139,16 @@ async def test_stormlib_aha_basics(self): self.len(nevents, await waiter.wait(timeout=12)) msgs = await core00.stormlist('aha.svc.list') - self.stormIsInPrint('01.cell.loop.vertex.link false false false', msgs) + self.stormIsInPrint('01.cell.synapse false false false', msgs, whitespace=False) # Fake a record await aha.addAhaSvc('00.newp', info={'urlinfo': {'scheme': 'tcp', 'host': '0.0.0.0', 'port': '3030'}}, - network='loop.vertex.link') + network='synapse') msgs = await core00.stormlist('aha.svc.list --nexus') - emsg = '00.newp.loop.vertex.link null false null 0.0.0.0 3030 ' \ + emsg = '00.newp.synapse null false null 0.0.0.0 3030 ' \ 'Service is not online. Will not attempt to retrieve its nexus offset.' - self.stormIsInPrint(emsg, msgs) + self.stormIsInPrint(emsg, msgs, whitespace=False) self.none(await core00.callStorm('return($lib.aha.del(00.newp...))')) msgs = await core00.stormlist('aha.svc.list') @@ -161,22 +161,22 @@ async def test_stormlib_aha_basics(self): 'port': '3030'}, 'online': guid, }, - network='loop.vertex.link') + network='synapse') msgs = await core00.stormlist('aha.svc.list --nexus') - emsg = '00.newp.loop.vertex.link null true null 0.0.0.0 3030 ' \ - 'Failed to connect to Telepath service: "aha://00.newp.loop.vertex.link/" error:' - self.stormIsInPrint(emsg, msgs) + emsg = '00.newp.synapse null true null 0.0.0.0 3030 ' \ + 'Failed to connect to Telepath service: "aha://00.newp.synapse/" error:' + self.stormIsInPrint(emsg, msgs, whitespace=False) msgs = await core00.stormlist('aha.svc.stat --nexus 00.newp...') emsg = '''Resolved 00.newp... to an AHA Service. -Name: 00.newp.loop.vertex.link +Name: 00.newp.synapse Online: true Ready: null Run iden: $lib.null Cell iden: $lib.null Leader: Service did not register itself with a leader name. -Nexus: Failed to connect to Telepath service: "aha://00.newp.loop.vertex.link/" error: [Errno 111] Connect call failed ('0.0.0.0', 3030) +Nexus: Failed to connect to Telepath service: "aha://00.newp.synapse/" error: [Errno 111] Connect call failed ('0.0.0.0', 3030) Connection information: host: 0.0.0.0 port: 3030 @@ -185,7 +185,7 @@ async def test_stormlib_aha_basics(self): self.stormIsInPrint(emsg, msgs) # Delete the fake service with its full service name - self.none(await core00.callStorm('return($lib.aha.del(00.newp.loop.vertex.link))')) + self.none(await core00.callStorm('return($lib.aha.del(00.newp.synapse))')) self.none(await core00.callStorm('return($lib.aha.get(00.newp...))')) # Coverage for sad paths diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index ab8ddba3f24..4df3109121f 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -677,7 +677,7 @@ async def fini(self): class TstOutPut(s_output.OutPutStr): - def expect(self, substr, throw=True): + def expect(self, substr, throw=True, whitespace=True): ''' Check if a string is present in the messages captured by the OutPutStr object. @@ -689,6 +689,11 @@ def expect(self, substr, throw=True): bool: True if the string is present; False if the string is not present and throw is False. ''' outs = str(self) + + if not whitespace: + outs = ' '.join(outs.split()) + substr = ' '.join(outs.split()) + if outs.find(substr) == -1: if throw: mesg = 'TestOutPut.expect(%s) not in %s' % (substr, outs) @@ -1297,12 +1302,7 @@ async def getTestCore(self, conf=None, dirn=None): Returns: s_cortex.Cortex: A Cortex object. ''' - if conf is None: - conf = { - 'health:sysctl:checks': False, - } - - conf = copy.deepcopy(conf) + conf = self.getCellConf(conf=conf) mods = conf.get('modules') @@ -1342,11 +1342,7 @@ async def getTestCoreAndProxy(self, conf=None, dirn=None): @contextlib.asynccontextmanager async def getTestJsonStor(self, dirn=None, conf=None): - if conf is None: - conf = { - 'health:sysctl:checks': False, - } - conf = copy.deepcopy(conf) + conf = self.getCellConf(conf=conf) with self.withNexusReplay(): if dirn is not None: @@ -1367,20 +1363,10 @@ async def getTestCryo(self, dirn=None, conf=None): Returns: s_cryotank.CryoCell: Test cryocell. ''' - if conf is None: - conf = { - 'health:sysctl:checks': False, - } - conf = copy.deepcopy(conf) + conf = self.getCellConf(conf=conf) with self.withNexusReplay(): - if dirn is not None: - async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo: - yield cryo - - return - - with self.getTestDir() as dirn: + with self.mayTestDir(dirn) as dirn: async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo: yield cryo @@ -1451,62 +1437,59 @@ async def getTestCoreProxSvc(self, ssvc, ssvc_conf=None, core_conf=None): yield core, prox, testsvc + @contextlib.contextmanager + def mayTestDir(self, dirn): + + if dirn is not None: + yield dirn + return + + with self.getTestDir() as dirn: + yield dirn + @contextlib.asynccontextmanager async def getTestAha(self, conf=None, dirn=None): if conf is None: - conf = { - 'health:sysctl:checks': False, - } - conf = copy.deepcopy(conf) + conf = {} - with self.withNexusReplay(): - if dirn: - async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha: - yield aha - else: - with self.getTestDir() as dirn: - async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha: - yield aha + conf = s_msgpack.deepcopy(conf) - @contextlib.asynccontextmanager - async def getTestAhaProv(self, conf=None, dirn=None): - ''' - Get an Aha cell that is configured for provisioning on aha.loop.vertex.link. + hasdmonlisn = 'dmon:listen' in conf + hasprovlisn = 'provision:listen' in conf - Args: - conf: Optional configuration information for the Aha cell. - dirn: Optional path to create the Aha cell in. + network = conf.setdefault('aha:network', 'synapse') + hostname = conf.setdefault('dns:name', '00.aha.loop.vertex.link') - Returns: - s_aha.AhaCell: The provisioned Aha cell. - ''' - bconf = { - 'aha:name': 'aha', - 'aha:network': 'loop.vertex.link', - 'provision:listen': 'ssl://aha.loop.vertex.link:0' - } + if hostname: + conf.setdefault('provision:listen', f'ssl://0.0.0.0:0?hostname={hostname}') + conf.setdefault('dmon:listen', f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}') - if conf is None: - conf = bconf - else: - for k, v in bconf.items(): - conf.setdefault(k, v) + conf.setdefault('health:sysctl:checks', False) - name = conf.get('aha:name') - netw = conf.get('aha:network') - dnsname = f'{name}.{netw}' + with self.withNexusReplay(): + with self.mayTestDir(dirn) as dirn: - async with self.getTestAha(conf=conf, dirn=dirn) as aha: - addr, port = aha.provdmon.addr - # update the config to reflect the dynamically bound port - aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}' + async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha: - # do this config ex-post-facto due to port binding... - host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}') - aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',) + mods = {} + if not hasdmonlisn and hostname: + mods['dmon:listen'] = f'ssl://0.0.0.0:{aha.sockaddr[1]}?hostname={hostname}&ca={network}' - yield aha + if not hasprovlisn and hostname: + mods['provision:listen'] = f'ssl://0.0.0.0:{aha.provaddr[1]}?hostname={hostname}' + + if mods: + aha.modCellConf(mods) + + yield aha + + def getCellConf(self, conf=None): + if conf is None: + conf = {} + conf = s_msgpack.deepcopy(conf) + conf.setdefault('health:sysctl:checks', False) + return conf @contextlib.asynccontextmanager async def addSvcToAha(self, aha, svcname, ctor, @@ -1528,36 +1511,18 @@ async def addSvcToAha(self, aha, svcname, ctor, The config data for the cell is pushed into dirn/cell.yaml. The cells are created with the ``ctor.anit()`` function. ''' + svcfull = f'{svcname}.{aha._getAhaNetwork()}' onetime = await aha.addAhaSvcProv(svcname, provinfo=provinfo) - if conf is None: - conf = { - 'health:sysctl:checks': False, - } - + conf = self.getCellConf(conf=conf) conf['aha:provision'] = onetime - logger.info(f'Adding {svcname}') - - n = 1 # a simple svcname like "foo" - if len(svcname.split('.')) > 1: - n = 2 # A replica name like 00.foo - if provinfo and 'mirror' in provinfo: - n = 1 # A mirror like 01.foo with mirror: foo - - waiter = aha.waiter(n, 'aha:svcadd') - - if dirn: + waiter = aha.waiter(1, f'aha:svcadd:{svcfull}') + with self.mayTestDir(dirn) as dirn: s_common.yamlsave(conf, dirn, 'cell.yaml') async with await ctor.anit(dirn) as svc: - self.ge(len(await waiter.wait(timeout=12)), n) + self.true(await waiter.wait(timeout=12)) yield svc - else: - with self.getTestDir() as dirn: - s_common.yamlsave(conf, dirn, 'cell.yaml') - async with await ctor.anit(dirn) as svc: - self.ge(len(await waiter.wait(timeout=12)), n) - yield svc async def addSvcToCore(self, svc, core, svcname='svc'): ''' @@ -2154,7 +2119,7 @@ async def agenlen(self, x, obj, msg=None): count += 1 self.eq(x, count, msg=msg) - def stormIsInPrint(self, mesg, mesgs, deguid=False): + def stormIsInPrint(self, mesg, mesgs, deguid=False, whitespace=True): ''' Check if a string is present in all of the print messages from a stream of storm messages. @@ -2166,6 +2131,9 @@ def stormIsInPrint(self, mesg, mesgs, deguid=False): mesg = deguidify(mesg) print_str = '\n'.join([m[1].get('mesg') for m in mesgs if m[0] == 'print']) + if not whitespace: + mesg = ' '.join(mesg.split()) + print_str = ' '.join(print_str.split()) if deguid: print_str = deguidify(print_str) From c9a9a9d0d9ee3621b1d57f4dbb2b140e6949b15d Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 28 Jun 2024 15:42:24 -0400 Subject: [PATCH 04/86] wip --- synapse/tests/test_cortex.py | 53 ++++++++++-------------------------- synapse/tests/utils.py | 7 ++--- 2 files changed, 16 insertions(+), 44 deletions(-) diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index cdafc355a0e..bf2b9ca9413 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -54,40 +54,21 @@ async def test_cortex_cellguid(self): async def test_cortex_handoff(self): with self.getTestDir() as dirn: - ahadir = s_common.genpath(dirn, 'aha00') - coredir0 = s_common.genpath(dirn, 'core00') - coredir1 = s_common.genpath(dirn, 'core01') - coredir2 = s_common.genpath(dirn, 'core02',) - - conf = { - 'aha:name': 'aha', - 'aha:network': 'newp', - 'provision:listen': 'tcp://127.0.0.1:0', - } - async with self.getTestAha(dirn=ahadir, conf=conf) as aha: - - provaddr, provport = aha.provdmon.addr - aha.conf['provision:listen'] = f'tcp://127.0.0.1:{provport}' + async with self.getTestAha() as aha: - ahahost, ahaport = await aha.dmon.listen('ssl://127.0.0.1:0?hostname=aha.newp&ca=newp') - aha.conf['aha:urls'] = (f'ssl://127.0.0.1:{ahaport}?hostname=aha.newp',) + conf = {'aha:provision': await aha.addAhaSvcProv('00.cortex')} - provurl = await aha.addAhaSvcProv('00.cortex') - coreconf = {'aha:provision': provurl, 'nexslog:en': False} - - async with self.getTestCore(dirn=coredir0, conf=coreconf) as core00: + async with self.getTestCore(conf=conf) as core00: with self.raises(s_exc.BadArg): await core00.handoff(core00.getLocalUrl()) self.false((await core00.getCellInfo())['cell']['uplink']) - provinfo = {'mirror': '00.cortex'} - provurl = await aha.addAhaSvcProv('01.cortex', provinfo=provinfo) - # provision with the new hostname and mirror config - coreconf = {'aha:provision': provurl} - async with self.getTestCore(dirn=coredir1, conf=coreconf) as core01: + provinfo = {'mirror': '00.cortex'} + conf = {'aha:provision': await aha.addAhaSvcProv('01.cortex', provinfo=provinfo)} + async with self.getTestCore(conf=conf) as core01: # test out connecting to the leader but having aha chose a mirror async with s_telepath.loadTeleCell(core01.dirn): @@ -117,9 +98,9 @@ async def test_cortex_handoff(self): self.true((await core00.getCellInfo())['cell']['uplink']) self.false((await core01.getCellInfo())['cell']['uplink']) - mods00 = s_common.yamlload(coredir0, 'cell.mods.yaml') - mods01 = s_common.yamlload(coredir1, 'cell.mods.yaml') - self.eq(mods00, {'mirror': 'aha://01.cortex.newp'}) + mods00 = s_common.yamlload(core00.dirn, 'cell.mods.yaml') + mods01 = s_common.yamlload(core01.dirn, 'cell.mods.yaml') + self.eq(mods00, {'mirror': 'aha://01.cortex.synapse'}) self.eq(mods01, {'mirror': None}) await core00.nodes('[inet:ipv4=5.5.5.5]') @@ -129,12 +110,11 @@ async def test_cortex_handoff(self): # This pops the mirror config out of the mods file we copied # from the backup. provinfo = {'mirror': '01.cortex'} - provurl = await aha.addAhaSvcProv('02.cortex', provinfo=provinfo) - coreconf = {'aha:provision': provurl} - async with self.getTestCore(dirn=coredir2, conf=coreconf) as core02: + conf = {'aha:provision': await aha.addAhaSvcProv('02.cortex', provinfo=provinfo)} + async with self.getTestCore(conf=conf) as core02: self.false(core02.isactive) - self.eq(core02.conf.get('mirror'), 'aha://root@01.cortex.newp') - mods02 = s_common.yamlload(coredir2, 'cell.mods.yaml') + self.eq(core02.conf.get('mirror'), 'aha://root@01.cortex...') + mods02 = s_common.yamlload(core02.dirn, 'cell.mods.yaml') self.eq(mods02, {}) # The mirror writeback and change distribution works self.len(0, await core01.nodes('inet:ipv4=6.6.6.6')) @@ -144,7 +124,7 @@ async def test_cortex_handoff(self): self.len(1, await core01.nodes('inet:ipv4=6.6.6.6')) self.len(1, await core00.nodes('inet:ipv4=6.6.6.6')) # list mirrors - exp = ['aha://00.cortex.newp', 'aha://02.cortex.newp'] + exp = ['aha://00.cortex.synapse', 'aha://02.cortex.synapse'] self.sorteq(exp, await core00.getMirrorUrls()) self.sorteq(exp, await core01.getMirrorUrls()) self.sorteq(exp, await core02.getMirrorUrls()) @@ -5373,11 +5353,6 @@ async def test_cortex_mirror(self): url = core00.getLocalUrl() - core01conf = {'nexslog:en': False, 'mirror': url} - with self.raises(s_exc.BadConfValu): - async with self.getTestCore(dirn=path01, conf=core01conf) as core01: - self.fail('Should never get here.') - core01conf = {'mirror': url} async with self.getTestCore(dirn=path01, conf=core01conf) as core01: diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index 4df3109121f..86fdce2a9c8 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -1304,11 +1304,8 @@ async def getTestCore(self, conf=None, dirn=None): ''' conf = self.getCellConf(conf=conf) - mods = conf.get('modules') - - if mods is None: - mods = [] - conf['modules'] = mods + mods = list(conf.get('modules', ())) + conf['modules'] = mods mods.insert(0, ('synapse.tests.utils.TestModule', {'key': 'valu'})) From c15ba7c12a4914cef0f7302bf3693d8bf57b8891 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 28 Jun 2024 17:07:00 -0400 Subject: [PATCH 05/86] wip --- synapse/tests/test_lib_stormlib_cell.py | 19 +-- synapse/tests/test_tools_aha.py | 169 +++++++++--------------- 2 files changed, 69 insertions(+), 119 deletions(-) diff --git a/synapse/tests/test_lib_stormlib_cell.py b/synapse/tests/test_lib_stormlib_cell.py index 0ce45cb1d3e..95732aa4934 100644 --- a/synapse/tests/test_lib_stormlib_cell.py +++ b/synapse/tests/test_lib_stormlib_cell.py @@ -76,18 +76,7 @@ async def test_stormlib_cell_uptime(self): async def test_stormlib_cell_getmirrors(self): - conf = { - 'aha:name': 'aha', - 'aha:network': 'mynet', - 'provision:listen': 'tcp://127.0.0.1:0', - } - async with self.getTestCell(s_aha.AhaCell, conf=conf) as aha: - - provaddr, provport = aha.provdmon.addr - aha.conf['provision:listen'] = f'tcp://127.0.0.1:{provport}' - - ahahost, ahaport = await aha.dmon.listen('ssl://127.0.0.1:0?hostname=aha.mynet&ca=mynet') - aha.conf['aha:urls'] = (f'ssl://127.0.0.1:{ahaport}?hostname=aha.mynet',) + async with self.getTestAha() as aha: provurl = await aha.addAhaSvcProv('00.cortex') coreconf = {'aha:provision': provurl} @@ -116,7 +105,7 @@ async def test_stormlib_cell_getmirrors(self): await core01.nodes('[ inet:ipv4=1.2.3.4 ]') self.len(1, await core00.nodes('inet:ipv4=1.2.3.4')) - expurls = ['aha://01.cortex.mynet'] + expurls = ['aha://01.cortex.synapse'] self.eq(expurls, await core00.callStorm('return($lib.cell.getMirrorUrls())')) self.eq(expurls, await core01.callStorm('return($lib.cell.getMirrorUrls())')) @@ -148,11 +137,11 @@ async def test_stormlib_cell_getmirrors(self): await svc01.sync() - expurls = ('aha://01.testsvc.mynet',) + expurls = ('aha://01.testsvc.synapse',) self.eq(expurls, await core00.callStorm('return($lib.cell.getMirrorUrls(name=testsvc))')) - await aha.delAhaSvc('00.testsvc.mynet') + await aha.delAhaSvc('00.testsvc.synapse') with self.raises(s_exc.NoSuchName): await core00.callStorm('return($lib.cell.getMirrorUrls(name=testsvc))') diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py index 647a6919546..a4fd7012451 100644 --- a/synapse/tests/test_tools_aha.py +++ b/synapse/tests/test_tools_aha.py @@ -17,40 +17,37 @@ class AhaToolsTest(s_t_utils.SynTest): async def test_aha_list(self): - ephemeral_address = 'tcp://0.0.0.0:0/' - async with self.getTestAha(conf={'auth:passwd': 'root', - 'dmon:listen': ephemeral_address}) as aha: - _, port = aha.sockaddr - ahaurl = f'tcp://root:root@127.0.0.1:{port}' - conf0 = { - 'aha:registry': ahaurl, - 'aha:name': 'cell0', - 'aha:network': 'demo.net', - 'dmon:listen': ephemeral_address, - } - conf1 = { - 'aha:registry': ahaurl, - 'aha:name': 'cell1', - 'aha:network': 'example.net', - 'dmon:listen': ephemeral_address, - } + async with self.getTestAha() as aha: + waiter = aha.waiter(2, 'aha:svcadd') + conf0 = {'aha:provision': await aha.addAhaSvcProv('cell0')} + + provinfo = {'aha:network': 'example.net'} + conf1 = {'aha:provision': await aha.addAhaSvcProv('cell1', provinfo=provinfo)} + + ahaurl = aha.getLocalUrl() + async with self.getTestCell(s_cell.Cell, conf=conf0) as cell0: + async with self.getTestCell(s_cell.Cell, conf=conf1) as cell1: + self.true(await waiter.wait(timeout=6)) + argv = [ahaurl] retn, outp = await self.execToolMain(s_a_list._main, argv) self.eq(retn, 0) - outp.expect('Service network leader') - outp.expect('cell0 demo.net None') - outp.expect('cell1 example.net None') + + outp.expect(''' + Service network leader + cell0 synapse None + cell1 example.net None + ''', whitespace=False) argv = [ahaurl, 'demo.net'] retn, outp = await self.execToolMain(s_a_list._main, argv) self.eq(retn, 0) - outp.expect('Service network') - outp.expect('cell0 demo.net') - self.false(outp.expect('cell1 example.net', throw=False)) + outp.expect('Service network', whitespace=False) + outp.expect('cell0 demo.net', whitespace=False) async with self.getTestCore() as core: curl = core.getLocalUrl() @@ -94,102 +91,66 @@ async def test_aha_easycert(self): async def test_aha_enroll(self): - with self.getTestDir() as dirn: - - conf = { - 'aha:name': 'aha', - 'aha:network': 'loop.vertex.link', - 'provision:listen': 'ssl://aha.loop.vertex.link:0', - 'dmon:listen': 'tcp://0.0.0.0:0' - } - async with self.getTestAha(dirn=dirn, conf=conf) as aha: + async with self.getTestAha() as aha: - addr, port = aha.provdmon.addr - aha.conf['provision:listen'] = f'ssl://aha.loop.vertex.link:{port}' + argv = ['--url', aha.getLocalUrl(), 'visi'] + retn, outp = await self.execToolMain(s_a_provision_user.main, argv) + self.isin('one-time use URL:', str(outp)) - host_, ahaport = aha.sockaddr - self.eq(aha._getAhaUrls(), [f'ssl://aha.loop.vertex.link:{ahaport}']) + provurl = str(outp).split(':', 1)[1].strip() + with self.getTestSynDir() as syndir: - argv = ['--url', aha.getLocalUrl(), 'visi'] - retn, outp = await self.execToolMain(s_a_provision_user.main, argv) - self.isin('one-time use URL:', str(outp)) - - provurl = str(outp).split(':', 1)[1].strip() - with self.getTestSynDir() as syndir: + capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt') + crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt') + keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt') - capath = s_common.genpath(syndir, 'certs', 'cas', 'loop.vertex.link.crt') - crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.crt') - keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.key') + retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,)) + self.eq(0, retn) - for path in (capath, crtpath, keypath): - s_common.genfile(path) + for path in (capath, crtpath, keypath): + self.true(os.path.isfile(path)) + self.gt(os.path.getsize(path), 0) - yamlpath = s_common.genpath(syndir, 'telepath.yaml') - s_common.yamlsave({'aha:servers': 'cell://aha'}, yamlpath) + teleyaml = s_common.yamlload(syndir, 'telepath.yaml') + self.eq(teleyaml.get('version'), 1) + self.len(1, teleyaml.get('aha:servers')) - retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,)) - self.eq(0, retn) + shutil.rmtree(s_common.genpath(syndir, 'certs')) - for path in (capath, crtpath, keypath): - self.gt(os.path.getsize(path), 0) - - teleyaml = s_common.yamlload(syndir, 'telepath.yaml') - self.eq(teleyaml.get('version'), 1) - self.eq(teleyaml.get('aha:servers'), ('cell://aha', f'ssl://visi@aha.loop.vertex.link:{ahaport}')) - - shutil.rmtree(s_common.genpath(syndir, 'certs')) - - def _strUrl(self): - return 'ssl://aha.loop.vertex.link' - - with mock.patch('synapse.lib.aha.AhaCell._getAhaUrls', _strUrl): - - argv = ['--again', '--url', aha.getLocalUrl(), 'visi'] - retn, outp = await self.execToolMain(s_a_provision_user.main, argv) - self.eq(retn, 0) - self.isin('one-time use URL:', str(outp)) - - provurl = str(outp).split(':', 1)[1].strip() - - retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,)) - - servers = ['cell://aha', - 'ssl://visi@aha.loop.vertex.link', - f'ssl://visi@aha.loop.vertex.link:{ahaport}'] + argv = ['--again', '--url', aha.getLocalUrl(), 'visi'] + retn, outp = await self.execToolMain(s_a_provision_user.main, argv) + self.eq(retn, 0) + self.isin('one-time use URL:', str(outp)) - teleyaml = s_common.yamlload(syndir, 'telepath.yaml') - self.sorteq(teleyaml.get('aha:servers'), servers) + provurl = str(outp).split(':', 1)[1].strip() - # Just return the URL - retn, outp = await self.execToolMain(s_a_provision_user.main, argv + ['--only-url']) - self.eq(retn, 0) - self.notin('one-time use URL:', str(outp)) - self.isin('ssl://', str(outp)) + retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,)) - with self.getTestSynDir() as syndir: + # Just return the URL + retn, outp = await self.execToolMain(s_a_provision_user.main, argv + ['--only-url']) + self.eq(retn, 0) + self.notin('one-time use URL:', str(outp)) + self.isin('ssl://', str(outp)) - argv = ['--again', '--url', aha.getLocalUrl(), 'visi'] - retn, outp = await self.execToolMain(s_a_provision_user.main, argv) - self.eq(retn, 0) - provurl = str(outp).split(':', 1)[1].strip() + with self.getTestSynDir() as syndir: - capath = s_common.genpath(syndir, 'certs', 'cas', 'loop.vertex.link.crt') - crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.crt') - keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.key') + argv = ['--again', '--url', aha.getLocalUrl(), 'visi'] + retn, outp = await self.execToolMain(s_a_provision_user.main, argv) + self.eq(retn, 0) + provurl = str(outp).split(':', 1)[1].strip() - for path in (capath, crtpath, keypath): - s_common.genfile(path) + capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt') + crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt') + keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.key') - yamlpath = s_common.genpath(syndir, 'telepath.yaml') - s_common.yamlsave({'aha:servers': ['cell://aha', 'cell://foo', ['cell://bar']]}, yamlpath) + for path in (capath, crtpath, keypath): + s_common.genfile(path) - retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,)) - self.eq(0, retn) + retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,)) + self.eq(0, retn) - for path in (capath, crtpath, keypath): - self.gt(os.path.getsize(path), 0) + for path in (capath, crtpath, keypath): + self.gt(os.path.getsize(path), 0) - teleyaml = s_common.yamlload(syndir, 'telepath.yaml') - self.eq(teleyaml.get('version'), 1) - self.eq(teleyaml.get('aha:servers'), ('cell://aha', 'cell://foo', ('cell://bar',), - f'ssl://visi@aha.loop.vertex.link:{ahaport}')) + teleyaml = s_common.yamlload(syndir, 'telepath.yaml') + self.eq(teleyaml.get('version'), 1) From 9ac5fbf6841c413efdcc47e0ada10766a8673efa Mon Sep 17 00:00:00 2001 From: visi Date: Sat, 29 Jun 2024 17:26:05 -0400 Subject: [PATCH 06/86] wip --- synapse/lib/cell.py | 20 +++++++++++++------- synapse/tests/test_lib_cell.py | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 85b1b65507f..576b018c804 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1491,22 +1491,23 @@ def _getAhaAdmin(self): async def _initAhaRegistry(self): + # NOTE: This API needs to be safe to call again ahaurl = self.conf.get('aha:registry') if ahaurl is not None: info = await s_telepath.addAhaUrl(ahaurl) - if self.ahaclient is None: - self.ahaclient = await s_telepath.Client.anit(info.get('url')) - self.ahaclient._fini_atexit = True - self.onfini(self.ahaclient) + if self.ahaclient is not None: + await self.ahaclient.fini() - async def finiaha(): + self.ahaclient = await s_telepath.Client.anit(ahaurl) + self.onfini(self.ahaclient) + + async def fini(): await s_telepath.delAhaUrl(ahaurl) - self.onfini(finiaha) + self.ahaclient.onfini(fini) ahauser = self.conf.get('aha:user') - ahanetw = self._getAhaNetwork() ahaadmin = self._getAhaAdmin() if ahaadmin is not None: @@ -1652,15 +1653,19 @@ async def _initAhaService(self): async def _runAhaRegLoop(): while not self.isfini: + try: proxy = await self.ahaclient.proxy() + info = await self.getAhaInfo() await proxy.addAhaSvc(ahaname, info, network=ahanetw) if self.isactive and ahalead is not None: await proxy.addAhaSvc(ahalead, info, network=ahanetw) + except Exception as e: logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}') await self.waitfini(1) + else: await proxy.waitfini() @@ -4073,6 +4078,7 @@ async def getCellInfo(self): 'type': self.getCellType(), 'iden': self.getCellIden(), 'active': self.isactive, + 'started': self.startms, 'ready': self.nexsroot.ready.is_set(), 'commit': self.COMMIT, 'version': self.VERSION, diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index 03659508394..c7a942dd57e 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -520,7 +520,7 @@ async def test_cell_getinfo(self): self.isin('cortex:defaults', cnfo.get('cellvers', {})) # Defaults aha data is - self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': None}) + self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': 'synapse'}) # Synapse information self.eq(snfo.get('version'), s_version.version) From 4b699093870c8157197b1675eda115e3b743c3d9 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 1 Jul 2024 16:10:38 -0400 Subject: [PATCH 07/86] wip --- synapse/lib/aha.py | 57 ++++------------------------- synapse/lib/cell.py | 69 ++++++++++++++++++----------------- synapse/lib/nexus.py | 23 +++++++++--- synapse/tests/test_lib_aha.py | 1 + synapse/tests/utils.py | 8 +++- 5 files changed, 67 insertions(+), 91 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index b93d4c17833..d68160acdb6 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -289,10 +289,6 @@ async def addAhaPoolSvc(self, poolname, svcname, info): async def delAhaPoolSvc(self, poolname, svcname): return await self.cell.delAhaPoolSvc(poolname, svcname) - async def iterAhaTopo(self): - async for item in self.cell.iterAhaTopo(): - yield item - async def iterPoolTopo(self, name): username = self.user.name.split('@')[0] @@ -577,7 +573,6 @@ async def initServiceStorage(self): slab = await s_lmdbslab.Slab.anit(dirn) slab.addResizeCallback(self.checkFreeSpace) - self.topobus = await s_base.Base.anit() self.jsonstor = await s_jsonstor.JsonStor.anit(slab, 'aha') # type: s_jsonstor.JsonStor async def fini(): @@ -594,7 +589,6 @@ async def fini(): self.slab.initdb('aha:pools') - self.topowindows = [] self.poolwindows = collections.defaultdict(list) async def getAhaServer(self, host, port): @@ -615,16 +609,6 @@ async def addAhaServer(self, server): return await self._push('aha:server:add', server) - async def _getTopoUpdate(self): - return ('aha:update', { - 'servers': await self.getAhaServers(), - }), - - async def _fireTopoUpdate(self): - update = await self._getTopoUpdate() - for wind in self.topowindows: - await wind.put(update) - @s_nexus.Pusher.onPush('aha:server:add') async def _addAhaServer(self, server): # TODO schema @@ -642,9 +626,6 @@ async def _addAhaServer(self, server): self.slab.put(lkey, s_msgpack.en(server), db='aha:servers') - if self.isactive: - await self._fireTopoUpdate() - return True @s_nexus.Pusher.onPushAuto('aha:server:del') @@ -656,9 +637,6 @@ async def delAhaServer(self, host, port): if byts is None: return None - if self.isactive: - await self._fireTopoUpdate() - return s_msgpack.un(byts) async def getAhaServers(self): @@ -667,25 +645,6 @@ async def getAhaServers(self): servers.append(s_msgpack.un(byts)) return servers - async def iterAhaTopo(self): - - if not self.isactive: - async with await self.nexsroot.client.proxy() as proxy: - async for item in proxy.iterAhaTopo(): - yield item - - async with await s_queue.Window.anit(maxsize=1000) as wind: - - async def onfini(): - self.topowindows.remove(wind) - - wind.onfini(onfini) - - async with self.nexsroot.applylock: - update = await self._getTopoUpdate() - await wind.put(update) - self.topowindows.append(wind) - async def iterPoolTopo(self, name): name = self._getAhaName(name) @@ -734,16 +693,16 @@ async def initServiceRuntime(self): await self.genCaCert(netw) name = self.conf.get('aha:name') - - host = f'{name}.{netw}' - if self.certdir.getHostCertPath(host) is None: - logger.info(f'Adding server certificate for {host}') - await self._genHostCert(host, signas=netw) + if name is not None: + host = f'{name}.{netw}' + if self.certdir.getHostCertPath(host) is None: + logger.info(f'Adding server certificate for {host}') + await self._genHostCert(host, signas=netw) root = f'root@{netw}' await self._genUserCert(root, signas=netw) - user = self._getAhaAdmin() + user = self.conf.get('aha:admin') if user is not None: await self._genUserCert(user, signas=netw) @@ -1309,14 +1268,14 @@ async def addAhaClone(self, host, port=27492, conf=None): 'port': port, 'conf': conf, } - return await self._push('aha:clone:add', clone) + await self._push('aha:clone:add', clone) + return self._getProvClientUrl(iden) @s_nexus.Pusher.onPush('aha:clone:add') async def _addAhaClone(self, clone): iden = clone.get('iden') lkey = s_common.uhex(iden) self.slab.put(lkey, s_msgpack.en(clone), db='aha:clones') - return self._getProvClientUrl(iden) async def addAhaSvcProv(self, name, provinfo=None): diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 576b018c804..a1e6c25ea13 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -961,7 +961,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware): 'type': 'string', }, 'aha:network': { - 'description': 'The AHA service network. Defaults to "synapse".', + 'description': 'The AHA service network.', 'type': 'string', }, 'aha:registry': { @@ -1104,6 +1104,8 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): self._checkspace = s_coro.Event() self._reloadfuncs = {} # name -> func + self.nexslock = asyncio.Lock() + self.conf = self._initCellConf(conf) self.minfree = self.conf.get('limit:disk:free') @@ -1127,7 +1129,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): self.ahasvcname = None ahaname = self.conf.get('aha:name') ahanetw = self._getAhaNetwork() - if ahaname is not None: + if ahaname is not None and ahanetw is not None: self.ahasvcname = f'{ahaname}.{ahanetw}' # each cell has a guid @@ -1176,20 +1178,20 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): if self.conf.get('mirror') and not self.conf.get('nexslog:en'): self.modCellConf({'nexslog:en': True}) - # construct our nexsroot instance ( but do not start it ) await s_nexus.Pusher.__anit__(self, self.iden) self._initCertDir() - root = await self._ctorNexsRoot() + await self.enter_context(s_telepath.loadTeleCell(self.dirn)) - # mutually assured destruction with our nexs root - self.onfini(root.fini) - root.onfini(self.fini) + await self._initCellSlab(readonly=readonly) - self.setNexsRoot(root) + await self.initServiceEarly() + + root = await self._ctorNexsRoot() + + await self.setNexsRoot(root) - await self._initCellSlab(readonly=readonly) self.apikeydb = self.slab.initdb('user:apikeys') # apikey -> useriden self.usermetadb = self.slab.initdb('user:meta') # useriden + -> dict valu self.rolemetadb = self.slab.initdb('role:meta') # roleiden + -> dict valu @@ -1434,10 +1436,10 @@ def checkFreeSpace(self): async def _runFreeSpaceLoop(self): - nexsroot = self.getCellNexsRoot() - while not self.isfini: + nexsroot = self.getCellNexsRoot() + self._checkspace.clear() disk = shutil.disk_usage(self.dirn) @@ -1484,11 +1486,6 @@ async def _runSysctlLoop(self): await self.waitfini(self.SYSCTL_CHECK_FREQ) - def _getAhaAdmin(self): - name = self.conf.get('aha:admin') - if name is not None: - return name - async def _initAhaRegistry(self): # NOTE: This API needs to be safe to call again @@ -1507,17 +1504,12 @@ async def fini(): self.ahaclient.onfini(fini) - ahauser = self.conf.get('aha:user') - - ahaadmin = self._getAhaAdmin() + ahaadmin = self.conf.get('aha:admin') if ahaadmin is not None: await self._addAdminUser(ahaadmin) - if ahauser is not None: - await self._addAdminUser(ahauser) - def _getAhaNetwork(self): - return self.conf.get('aha:network', 'synapse') + return self.conf.get('aha:network') def _getDmonListen(self): @@ -1545,6 +1537,9 @@ async def _addAdminUser(self, username): if user.isLocked(): await user.setLocked(False, logged=False) + async def initServiceEarly(self): + pass + async def initServiceStorage(self): pass @@ -1578,8 +1573,8 @@ async def initServiceNetwork(self): turl = self._getDmonListen() if turl is not None: - self.sockaddr = await self.dmon.listen(turl) logger.info(f'dmon listening: {turl}') + self.sockaddr = await self.dmon.listen(turl) await self._initAhaService() @@ -1641,13 +1636,16 @@ async def _initAhaService(self): if ahaname is None: return - ahalead = self.conf.get('aha:leader') ahanetw = self._getAhaNetwork() + if ahanetw is None: + return ahainfo = await self.getAhaInfo() if ahainfo is None: return + ahalead = self.conf.get('aha:leader') + self.ahasvcname = f'{ahaname}.{ahanetw}' async def _runAhaRegLoop(): @@ -1804,11 +1802,11 @@ async def handoff(self, turl, timeout=30): mesg = 'Cannot handoff mirror leadership to myself!' raise s_exc.BadArg(mesg=mesg) - logger.debug(f'HANDOFF: Obtaining nexus applylock ahaname={ahaname}') + logger.debug(f'HANDOFF: Obtaining nexus lock ahaname={ahaname}') - async with self.nexsroot.applylock: + async with self.nexslock: - logger.debug(f'HANDOFF: Obtained nexus applylock ahaname={ahaname}') + logger.debug(f'HANDOFF: Obtained nexus lock ahaname={ahaname}') indx = await self.getNexsIndx() logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}, ahaname={ahaname}') @@ -1829,7 +1827,7 @@ async def handoff(self, turl, timeout=30): logger.debug(f'HANDOFF: Restarting the nexus ahaname={ahaname}') await self.nexsroot.startup() - logger.debug(f'HANDOFF: Released nexus applylock ahaname={ahaname}') + logger.debug(f'HANDOFF: Released nexus lock ahaname={ahaname}') logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)} ahaname={self.conf.get("aha:name")}') async def reqAhaProxy(self, timeout=None): @@ -2129,7 +2127,7 @@ async def _execBackupTask(self, dirn): try: - async with self.nexsroot.applylock: + async with self.nexslock: logger.debug('Syncing LMDB Slabs') @@ -2983,6 +2981,12 @@ async def _initCellHive(self): return hive + async def _initSlabFile(self, path, readonly=False): + slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly) + slab.addResizeCallback(self.checkFreeSpace) + self.onfini(slab) + return slab + async def _initCellSlab(self, readonly=False): s_common.gendir(self.dirn, 'slabs') @@ -2994,10 +2998,7 @@ async def _initCellSlab(self, readonly=False): _slab.initdb('hive') await _slab.fini() - self.slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly) - self.slab.addResizeCallback(self.checkFreeSpace) - - self.onfini(self.slab.fini) + self.slab = await self._initSlabFile(path) async def _initCellAuth(self): diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py index 657979c1173..fd9af2bfddd 100644 --- a/synapse/lib/nexus.py +++ b/synapse/lib/nexus.py @@ -92,7 +92,6 @@ async def __anit__(self, cell): self.writeholds = set() self.applytask = None - self.applylock = asyncio.Lock() self.ready = asyncio.Event() self.donexslog = self.cell.conf.get('nexslog:en') @@ -231,7 +230,7 @@ def _getResponseFuture(self, iden=None): async def enNexsLog(self): - async with self.applylock: + async with self.cell.nexslock: if self.donexslog: return @@ -345,7 +344,7 @@ async def eat(self, nexsiden, event, args, kwargs, meta): if meta is None: meta = {} - async with self.applylock: + async with self.cell.nexslock: self.reqNotReadOnly() # Keep a reference to the shielded task to ensure it isn't GC'd self.applytask = asyncio.create_task(self._eat((nexsiden, event, args, kwargs, meta))) @@ -616,12 +615,12 @@ async def __anit__(self, iden: str, nexsroot: NexsRoot = None): # type: ignore self.nexsroot: Optional[NexsRoot] = None if nexsroot is not None: - self.setNexsRoot(nexsroot) + await self.setNexsRoot(nexsroot) for event, func, passitem in self._regclstupls: # type: ignore self._nexshands[event] = func, passitem - def setNexsRoot(self, nexsroot): + async def setNexsRoot(self, nexsroot): nexsroot._nexskids[self.nexsiden] = self @@ -629,10 +628,22 @@ def onfini(): prev = nexsroot._nexskids.pop(self.nexsiden, None) assert prev is not None, f'Failed removing {self.nexsiden}' - self.onfini(onfini) + nexsroot.onfini(onfini) + self.onfini(nexsroot) self.nexsroot = nexsroot + async def modNexsRoot(self, ctor): + + if self.nexsroot is not None: + await self.nexsroot.fini() + + nexsroot = await ctor() + + await self.setNexsRoot(nexsroot) + + await self.nexsroot.startup() + @classmethod def onPush(cls, event: str, passitem=False) -> Callable: ''' diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 0e10fb7bf65..d6f7809adc6 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -166,6 +166,7 @@ async def test_lib_aha(self): conf = { 'aha:name': '0.cryo', 'aha:leader': 'cryo', + 'aha:network': 'synapse', 'aha:admin': 'root@synapse', 'aha:registry': [f'tcp://root:hehehaha@127.0.0.1:{port}', f'tcp://root:hehehaha@127.0.0.1:{port}'], diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index 86fdce2a9c8..15a1aab1b8c 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -1445,11 +1445,14 @@ def mayTestDir(self, dirn): yield dirn @contextlib.asynccontextmanager - async def getTestAha(self, conf=None, dirn=None): + async def getTestAha(self, conf=None, dirn=None, ctor=None): if conf is None: conf = {} + if ctor is None: + ctor = s_aha.AhaCell.anit + conf = s_msgpack.deepcopy(conf) hasdmonlisn = 'dmon:listen' in conf @@ -1465,9 +1468,10 @@ async def getTestAha(self, conf=None, dirn=None): conf.setdefault('health:sysctl:checks', False) with self.withNexusReplay(): + with self.mayTestDir(dirn) as dirn: - async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha: + async with await ctor(dirn, conf=conf) as aha: mods = {} if not hasdmonlisn and hostname: From f1d1cad16d26d544d0eee47bb49a33d0ea1fdebc Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 1 Jul 2024 16:55:46 -0400 Subject: [PATCH 08/86] wip --- synapse/lib/nexus.py | 3 +-- synapse/tests/test_lib_cell.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py index fd9af2bfddd..2f00e49c343 100644 --- a/synapse/lib/nexus.py +++ b/synapse/lib/nexus.py @@ -514,8 +514,7 @@ async def runMirrorLoop(self, proxy): if self.celliden is not None: if self.celliden != await proxy.getCellIden(): logger.error('remote cell has different iden! Aborting mirror sync') - await proxy.fini() - await self.fini() + await self.cell.fini() return while not proxy.isfini: diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index c7a942dd57e..f9778dbeb53 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -520,7 +520,7 @@ async def test_cell_getinfo(self): self.isin('cortex:defaults', cnfo.get('cellvers', {})) # Defaults aha data is - self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': 'synapse'}) + self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': None}) # Synapse information self.eq(snfo.get('version'), s_version.version) @@ -1710,7 +1710,7 @@ async def test_mirror_badiden(self): 'has different iden') as stream: async with self.getTestCell(s_cell.Cell, dirn=path01, conf=conf01) as cell01: await stream.wait(timeout=2) - self.true(await cell01.waitfini(6)) + self.true(await cell01.nexsroot.waitfini(6)) async def test_backup_restore_base(self): From 9c18de4a5783808dd18994b75a02d0e5a19efe97 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 08:20:08 -0400 Subject: [PATCH 09/86] wip --- synapse/lib/aha.py | 39 ++++++++++++++------------- synapse/lib/cell.py | 33 ++++++++++++++--------- synapse/lib/config.py | 26 +++++------------- synapse/tests/files/rstorm/testsvc.py | 2 +- synapse/tests/test_lib_aha.py | 2 +- synapse/tests/test_lib_cell.py | 12 ++++----- synapse/tests/test_lib_config.py | 7 +++-- synapse/tests/utils.py | 3 ++- 8 files changed, 60 insertions(+), 64 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index d68160acdb6..66264e8f0b1 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -269,7 +269,6 @@ async def signHostCsr(self, csrtext, signas=None, sans=None): return await self.cell.signHostCsr(csrtext, signas=signas, sans=sans) async def signUserCsr(self, csrtext, signas=None): - await self._reqUserAllowed(('aha', 'csr', 'user')) return await self.cell.signUserCsr(csrtext, signas=signas) @@ -428,17 +427,17 @@ async def getUserInfo(self): return { 'aha:urls': await self.aha.getAhaUrls(user=user), 'aha:user': user, - 'aha:network': self.aha._getAhaNetwork(), + 'aha:network': self.aha.conf.req('aha:network'), } async def getCaCert(self): - ahanetw = self.aha._getAhaNetwork() + ahanetw = self.aha.conf.req('aha:network') return self.aha.certdir.getCaCertBytes(ahanetw) async def signUserCsr(self, byts): ahauser = self.userinfo.get('name') - ahanetw = self.aha._getAhaNetwork() + ahanetw = self.aha.conf.req('aha:network') username = f'{ahauser}@{ahanetw}' @@ -464,7 +463,7 @@ async def getProvInfo(self): return self.provinfo async def getCaCert(self): - ahanetw = self.aha._getAhaNetwork() + ahanetw = self.aha.conf.req('aha:network') return self.aha.certdir.getCaCertBytes(ahanetw) async def signHostCsr(self, byts): @@ -686,7 +685,7 @@ async def initServiceRuntime(self): if self.isactive: # bootstrap a CA for our aha:network - netw = self._getAhaNetwork() + netw = self.conf.req('aha:network') if self.certdir.getCaCertPath(netw) is None: logger.info(f'Adding CA certificate for {netw}') @@ -714,9 +713,10 @@ def _getDnsName(self): if hostname is not None: return hostname - name = self.conf.get('aha:name') - if name is not None: - return f'{name}.{self._getAhaNetwork()}' + ahaname = self.conf.get('aha:name') + ahanetw = self.conf.get('aha:network') + if ahaname is not None and ahanetw is not None: + return f'{ahaname}.{ahanetw}' def _getProvListen(self): @@ -737,7 +737,7 @@ def _getDmonListen(self): if lisn is not s_common.novalu: return lisn - network = self._getAhaNetwork() + network = self.conf.req('aha:network') dnsname = self._getDnsName() if dnsname is not None: return f'ssl://0.0.0.0?hostname={dnsname}&ca={network}' @@ -753,9 +753,8 @@ def _reqProvListen(self): async def initServiceNetwork(self): # bootstrap CA/host certs first - network = self._getAhaNetwork() - if network is not None: - await self._genCaCert(network) + network = self.conf.req('aha:network') + await self._genCaCert(network) hostname = self._getDnsName() if hostname is not None and network is not None: @@ -862,7 +861,7 @@ async def addAhaSvc(self, name, info, network=None): def _getAhaName(self, name): # the modern version of names is absolute or ... if name.endswith('...'): - return name[:-2] + self._getAhaNetwork() + return name[:-2] + self.conf.req('aha:network') return name async def getAhaPool(self, name): @@ -1228,18 +1227,20 @@ async def getAhaUrls(self, user='root'): if urls is not None: return urls - network = self._getAhaNetwork() + network = self.conf.req('aha:network') + urls = [] for server in await self.getAhaServers(): host = server.get('host') port = server.get('port') urls.append(f'ssl://{host}:{port}?certname={user}@{network}') + return urls def _getAhaUrl(self, user='root'): port = self.sockaddr[1] host = self._getDnsName() - network = self._getAhaNetwork() + network = self.conf.req('aha:network') return f'ssl://{host}:{port}?certname={user}@{network}' async def getAhaClone(self, iden): @@ -1253,7 +1254,7 @@ async def addAhaClone(self, host, port=27492, conf=None): if conf is None: conf = {} - network = self._getAhaNetwork() + network = self.conf.req('aha:network') conf['mirror'] = self._getAhaUrl() @@ -1293,7 +1294,7 @@ async def addAhaSvcProv(self, name, provinfo=None): conf = provinfo.setdefault('conf', {}) - netw = self._getAhaNetwork() + netw = self.conf.req('aha:network') ahaadmin = self.conf.get('aha:admin') if ahaadmin is not None: # pragma: no cover @@ -1418,7 +1419,7 @@ async def addAhaUserEnroll(self, name, userinfo=None, again=False): raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.') provurl = self._reqProvListen() - ahanetw = self._getAhaNetwork() + ahanetw = self.conf.req('aha:network') username = f'{name}@{ahanetw}' diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index a1e6c25ea13..c06497ffe81 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1128,7 +1128,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): # we need to know this pretty early... self.ahasvcname = None ahaname = self.conf.get('aha:name') - ahanetw = self._getAhaNetwork() + ahanetw = self.conf.get('aha:network') if ahaname is not None and ahanetw is not None: self.ahasvcname = f'{ahaname}.{ahanetw}' @@ -1508,8 +1508,14 @@ async def fini(): if ahaadmin is not None: await self._addAdminUser(ahaadmin) - def _getAhaNetwork(self): - return self.conf.get('aha:network') + def reqAhaUser(self): + + ahauser = self.conf.get('aha:user') + if ahauser is not None: + return ahauser + + mesg = 'An aha:user configuration option is required.' + raise s_exc.NeedConfValu(mesg=mesg) def _getDmonListen(self): @@ -1517,10 +1523,11 @@ def _getDmonListen(self): if lisn is not s_common.novalu: return lisn - network = self._getAhaNetwork() ahaname = self.conf.get('aha:name') - if ahaname is not None: - return f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}' + ahanetw = self.conf.get('aha:network') + if ahaname is not None and ahanetw is not None: + hostname = f'{ahaname}.{ahanetw}' + return f'ssl://0.0.0.0:0?hostname={hostname}&ca={ahanetw}' async def _addAdminUser(self, username): # add the user in a pre-nexus compatible way @@ -1636,7 +1643,7 @@ async def _initAhaService(self): if ahaname is None: return - ahanetw = self._getAhaNetwork() + ahanetw = self.conf.get('aha:network') if ahanetw is None: return @@ -1763,7 +1770,7 @@ async def promote(self, graceful=False): mesg = 'Cannot gracefully promote without aha:name configured.' raise s_exc.BadArg(mesg=mesg) - ahanetw = self._getAhaNetwork() + ahanetw = self.conf.req('aha:network') myurl = f'aha://{ahaname}.{ahanetw}' logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}') @@ -1861,7 +1868,7 @@ async def _setAhaActive(self): await proxy.fini() return - ahanetw = self._getAhaNetwork() + ahanetw = self.conf.req('aha:network') try: await proxy.addAhaSvc(ahalead, ahainfo, network=ahanetw) except asyncio.CancelledError: # pragma: no cover @@ -2819,7 +2826,7 @@ async def addHttpsPort(self, port, host='0.0.0.0', sslctx=None): sslctx = self.initSslCtx(certpath, pkeypath) kwargs = { - 'xheaders': self.conf.reqConfValu('https:parse:proxy:remoteip') + 'xheaders': self.conf.req('https:parse:proxy:remoteip') } serv = self.wapp.listen(port, address=addr, ssl_options=sslctx, **kwargs) self.httpds.append(serv) @@ -3732,7 +3739,7 @@ async def _bootCellMirror(self, pnfo): # this function must assume almost nothing is initialized # but that's ok since it will only run rarely. # It assumes it has a tuple of (provisioning configuration, provisioning iden) available - murl = self.conf.reqConfValu('mirror') + murl = self.conf.req('mirror') provconf, providen = pnfo logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)') @@ -3896,7 +3903,7 @@ async def _getCellUser(self, link, mesg): if username.find('@') != -1: userpart, hostpart = username.split('@', 1) - if hostpart == self._getAhaNetwork(): + if hostpart == self.conf.get('aha:network'): user = await self.auth.getUserByName(userpart) if user is not None: if user.isLocked(): @@ -4090,7 +4097,7 @@ async def getCellInfo(self): 'aha': { 'name': self.conf.get('aha:name'), 'leader': self.conf.get('aha:leader'), - 'network': self._getAhaNetwork(), + 'network': self.conf.get('aha:network'), } }, 'features': { diff --git a/synapse/lib/config.py b/synapse/lib/config.py index 4e6b18dcd4b..5295a4bac7a 100644 --- a/synapse/lib/config.py +++ b/synapse/lib/config.py @@ -392,28 +392,16 @@ def reqConfValid(self): else: return - def reqConfValu(self, key): + def req(self, name): ''' - Get a configuration value. If that value is not present in the schema - or is not set, then raise an exception. - - Args: - key (str): The key to require. - - Returns: - The requested value. + Return a configuration value or raise NeedConfValu if it is unset. ''' - # Ensure that the key is in self.json_schema - if key not in self.json_schema.get('properties', {}): - raise s_exc.BadArg(mesg='Required key is not present in the configuration schema.', - key=key) - - # Ensure that the key is present in self.conf - if key not in self.conf: - raise s_exc.NeedConfValu(mesg='Required key is not present in configuration data.', - key=key) + valu = self.conf.get(name, s_common.novalu) + if valu is not s_common.novalu: + return valu - return self.conf.get(key) + mesg = f'The {name} configuration option is required.' + raise s_exc.NeedConfValu(mesg=mesg, name=name) def reqKeyValid(self, key, value): ''' diff --git a/synapse/tests/files/rstorm/testsvc.py b/synapse/tests/files/rstorm/testsvc.py index 55f4822d946..678fac85bee 100644 --- a/synapse/tests/files/rstorm/testsvc.py +++ b/synapse/tests/files/rstorm/testsvc.py @@ -37,7 +37,7 @@ class Testsvc(s_cell.Cell): async def __anit__(self, dirn, conf): await s_cell.Cell.__anit__(self, dirn, conf=conf) - self.secret = self.conf.reqConfValu('secret') + self.secret = self.conf.req('secret') async def test(self): return self.secret diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index d6f7809adc6..ffc716e0d4f 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -951,7 +951,7 @@ async def test_aha_restart(self): svc1dirn = s_common.gendir(dirn, 'svc01') async with await s_base.Base.anit() as cm: - aconf = {'dns:name': 'aha.loop.vertex.link'} + aconf = {'dns:name': 'aha.loop.vertex.link', 'aha:network': 'synapse'} aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf) await cm.enter_context(aha) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index f9778dbeb53..a202ec1dab0 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -925,11 +925,11 @@ async def test_cell_initargv_conf(self): # 1) cmdline # 2) envars # 3) cell.yaml - self.true(cell.conf.reqConfValu('nexslog:en')) - self.true(cell.conf.reqConfValu('nexslog:async')) - self.none(cell.conf.reqConfValu('dmon:listen')) - self.none(cell.conf.reqConfValu('https:port')) - self.eq(cell.conf.reqConfValu('aha:name'), 'some:cell') + self.true(cell.conf.req('nexslog:en')) + self.true(cell.conf.req('nexslog:async')) + self.none(cell.conf.req('dmon:listen')) + self.none(cell.conf.req('https:port')) + self.eq(cell.conf.req('aha:name'), 'some:cell') root = cell.auth.rootuser self.true(await root.tryPasswd('secret')) @@ -937,7 +937,7 @@ async def test_cell_initargv_conf(self): with self.getTestDir() as dirn: s_common.yamlsave({'nexslog:en': False}, dirn, 'cell.mods.yaml') async with await s_cell.Cell.initFromArgv([dirn]) as cell: - self.false(cell.conf.reqConfValu('nexslog:en')) + self.false(cell.conf.req('nexslog:en')) # We can remove the valu from the overrides file with the pop API # This is NOT reactive API which causes the whole behavior # of the cell to suddenly change. This is intended to be used with diff --git a/synapse/tests/test_lib_config.py b/synapse/tests/test_lib_config.py index ec3fb0080c6..cb61a6ae3d4 100644 --- a/synapse/tests/test_lib_config.py +++ b/synapse/tests/test_lib_config.py @@ -31,7 +31,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, *args, **kwargs): await s_cell.Cell.__anit__(self, dirn, conf, readonly, *args, **kwargs) # This captures a design pattern that reduces boilerplate # code used by Cell implementators. - self.conf.reqConfValu('apikey') + self.conf.req('apikey') class ConfTest(s_test.SynTest): @@ -162,10 +162,9 @@ async def test_config_basics(self): }) # We can ensure that certain vars are loaded - self.eq('Funky string time!', conf.reqConfValu('key:string')) + self.eq('Funky string time!', conf.req('key:string')) # And throw if they are not, or if the requested key isn't even schema valid - self.raises(s_exc.NeedConfValu, conf.reqConfValu, 'key:bool:nodefval') - self.raises(s_exc.BadArg, conf.reqConfValu, 'key:newp') + self.raises(s_exc.NeedConfValu, conf.req, 'key:bool:nodefval') # Since we're an Mutable mapping, we have some dict methods available to us self.len(8, conf) # __len__ diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index 15a1aab1b8c..f1487db931d 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -1512,7 +1512,8 @@ async def addSvcToAha(self, aha, svcname, ctor, The config data for the cell is pushed into dirn/cell.yaml. The cells are created with the ``ctor.anit()`` function. ''' - svcfull = f'{svcname}.{aha._getAhaNetwork()}' + ahanetw = aha.conf.req('aha:network') + svcfull = f'{svcname}.{ahanetw}' onetime = await aha.addAhaSvcProv(svcname, provinfo=provinfo) conf = self.getCellConf(conf=conf) From a1664e53088cdc0705b6656814fb9034c24e579e Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 10:27:32 -0400 Subject: [PATCH 10/86] wip --- docs/synapse/deploymentguide.rst | 120 +++++++++++++++++++++---------- synapse/tests/test_tools_aha.py | 5 ++ synapse/tools/aha/clone.py | 42 +++++++++++ 3 files changed, 128 insertions(+), 39 deletions(-) create mode 100644 synapse/tools/aha/clone.py diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index 2b84671c5d1..106bf0f03ff 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -65,19 +65,6 @@ When using AHA, you may run any of the **other** services on additional hosts as directly to the AHA service. You may also shutdown a service, move it's volume to a different host, and start it backup without changing anything. -Decide on a Name -================ - -Throughout the examples, we will be using ```` as the AHA network name which is also used as the -common-name (CN) for the CA certificate. This should be changed to an appropriate network name used by your -synapse deployment such as ``syn.acmecorp.com``. We will use ```` in the following configs to -specify locations which should be replaced with your selected AHA network name. For a **test** deployment which -runs **all** docker containers on one host, you may use ``loop.vertex.link``. - -.. note:: - It is important that you choose a name and stick with it for a given deployment. Once we begin generating - host and service account certificates, changing this name will be difficult. - Deploy AHA Service ================== @@ -87,18 +74,43 @@ by name, so it is likely that you need to create a DNS A/AAAA record in your exi using AHA, the only host that needs DNS or other external name resolution is the AHA service. .. note:: - It is important to ensure that ``aha.`` is resolvable via DNS or docker container service - name resolution from within the container environment! There are configuration options you may use if - this is impossible, but the configuration is far simpler if we can make this assumption. + FIXME AHA FLAT NETWORK STUFF + +Choose an AHA Network Name +-------------------------- + +Your AHA network name is separate from DNS and is used by services within the AHA service deployment. It is +generally a good idea to chose a name that aligns with the use case for the synapse deployment. For example, +if you plan to have a test/dev/prod deployment, choosing a name like ``prod.synapse`` will make it clear which +deployment is which. Changing this name later is difficult, so choose carefully! Throughout the examples, we +will be using ``prod.synapse`` as the AHA network name which is also used as the common-name (CN) for the CA +certificate. + +Configure an AHA DNS Name +------------------------- + +When choosing the DNS name for your AHA server, it is important to keep in mind that you may eventually want to +deploy AHA mirrors for a high-availability deployment. Choosing a name like ``00.aha.``, +where ```` is a DNS zone you control, will allow you to deploy mirrors with a consistent naming +convention in the future. Once you have selected the DNS name for your AHA server, you will need to ensure that +all deployed Synapse services will be able to resolve that name. + +.. note:: + It is important to ensure that ``00.aha.`` is resolvable via DNS or docker container service + name resolution from within the container environment! + + +Deploy the AHA Container +------------------------ Create the container directory:: - mkdir -p /srv/syn/aha/storage + mkdir -p /srv/syn/00.aha/storage -Create the ``/srv/syn/aha/docker-compose.yaml`` file with contents:: +Create the ``/srv/syn/00.aha/docker-compose.yaml`` file with contents:: services: - aha: + 00.aha: user: "999" image: vertexproject/synapse-aha:v2.x.x network_mode: host @@ -106,22 +118,23 @@ Create the ``/srv/syn/aha/docker-compose.yaml`` file with contents:: volumes: - ./storage:/vertex/storage environment: + # disable HTTPS API for now to prevent port collisions - SYN_AHA_HTTPS_PORT=null - - SYN_AHA_AHA_NAME=aha - - SYN_AHA_DNS_NAME=aha. + - SYN_AHA_AHA_NETWORK=prod.synapse + - SYN_AHA_DNS_NAME=00.aha. .. note:: - Don't forget to replace ```` with your chosen network name! + Don't forget to replace ```` with your configured DNS suffix. Change ownership of the storage directory to the user you will use to run the container:: - chown -R 999 /srv/syn/aha/storage + chown -R 999 /srv/syn/00.aha/storage Start the container using ``docker compose``:: - docker compose --file /srv/syn/aha/docker-compose.yaml pull - docker compose --file /srv/syn/aha/docker-compose.yaml up -d + docker compose --file /srv/syn/00.aha/docker-compose.yaml pull + docker compose --file /srv/syn/00.aha/docker-compose.yaml up -d To view the container logs at any time you may run the following command on the *host* from the ``/srv/syn/aha`` directory:: @@ -131,11 +144,40 @@ To view the container logs at any time you may run the following command on the You may also execute a shell inside the container using ``docker compose`` from the ``/srv/syn/aha`` directory on the *host*. This will be necessary for some of the additional provisioning steps:: - docker compose exec aha /bin/bash - + docker compose exec 00.aha /bin/bash .. _deploy_axon: +Deploy AHA Mirrors (optional) +============================= + +For high-availability deployments, you will want to deploy an AHA mirror or two. Typical Synapse service +mirroring is configured using AHA based provisioning. Bootstrapping AHA mirrors is simple, but requires +a slightly different procedure because you cannot bootstrap AHA via AHA :) + +For this example, we will assume you chose a DNS name for your primary AHA server similar to the steps +listed above. If so, you can simply replace ``00`` with sequential numbers and repeat this step to deploy +however many AHA mirrors you deem appropriate. + +NOTE: AHA uses two default ports ETC. The following steps assume you will be running each of your AHA servers +on a different host. The use of ``network_mode; host`` ETC + +Generate a one-time use provisioning URL:: + + python -m synapse.tools.aha.clone 01.aha. + +You should see output that looks similar to this:: + + one-time use URL: ssl://00.aha.:27272/?certhash= + +Create the container directory:: + + mkdir -p /srv/syn/01.aha/storage + chown -R 999 /srv/syn/01.aha/storage + +::NOTE You can deploy AHA mirrors at any time in the future. Once the mirrors are deployed, each AHA enabled + Synapse service will retrieve the updated list of AHA servers the next time it is restarted. + Deploy Axon Service =================== @@ -161,11 +203,11 @@ to re-use the one-time use URL! We strongly encourage you to use a numbered hierarchical naming convention for services where the first part of the name is a 0 padded number and the second part is the service type. The above example ``00.axon`` will allow you to deploy mirror instances in the future, such as ``01.axon``, where the AHA - name ``axon.`` will automatically resolve to which ever one is the current leader. + name ``axon.prod.synapse`` will automatically resolve to which ever one is the current leader. You should see output that looks similar to this:: - one-time use URL: ssl://aha.:27272/?certhash= + one-time use URL: ssl://00.aha.:27272/?certhash= **On the Host** @@ -187,7 +229,7 @@ Create the ``/srv/syn/00.axon/docker-compose.yaml`` file with contents:: environment: # disable HTTPS API for now to prevent port collisions - SYN_AXON_HTTPS_PORT=null - - SYN_AXON_AHA_PROVISION=ssl://aha.:27272/?certhash= + - SYN_AXON_AHA_PROVISION=ssl://00.aha.:27272/?certhash= .. note:: @@ -209,7 +251,7 @@ Generate a one-time use provisioning URL:: You should see output that looks similar to this:: - one-time use URL: ssl://aha.:27272/?certhash= + one-time use URL: ssl://00.aha.:27272/?certhash= **On the Host** @@ -231,7 +273,7 @@ Create the ``/srv/syn/00.jsonstor/docker-compose.yaml`` file with contents:: environment: # disable HTTPS API for now to prevent port collisions - SYN_JSONSTOR_HTTPS_PORT=null - - SYN_JSONSTOR_AHA_PROVISION=ssl://aha.:27272/?certhash= + - SYN_JSONSTOR_AHA_PROVISION=ssl://00.aha.:27272/?certhash= .. note:: @@ -253,7 +295,7 @@ Generate a one-time use provisioning URL:: You should see output that looks similar to this:: - one-time use URL: ssl://aha.:27272/?certhash= + one-time use URL: ssl://00.aha.:27272/?certhash= **On the Host** @@ -275,7 +317,7 @@ Create the ``/srv/syn/00.cortex/docker-compose.yaml`` file with contents:: environment: - SYN_CORTEX_AXON=aha://axon... - SYN_CORTEX_JSONSTOR=aha://jsonstor... - - SYN_CORTEX_AHA_PROVISION=ssl://aha.:27272/?certhash= + - SYN_CORTEX_AHA_PROVISION=ssl://00.aha.:27272/?certhash= .. note:: @@ -309,7 +351,7 @@ Generate a one-time use URL for provisioning from *inside the AHA container*:: You should see output that looks similar to this:: - one-time use URL: ssl://aha.:27272/?certhash= + one-time use URL: ssl://00.aha.:27272/?certhash= **On the Host** @@ -333,7 +375,7 @@ Create the ``/srv/syn/01.cortex/docker-compose.yaml`` file with contents:: - SYN_CORTEX_JSONSTOR=aha://jsonstor... # disable HTTPS API for now to prevent port collisions - SYN_CORTEX_HTTPS_PORT=null - - SYN_CORTEX_AHA_PROVISION=ssl://aha.:27272/?certhash= + - SYN_CORTEX_AHA_PROVISION=ssl://00.aha.:27272/?certhash= .. note:: @@ -372,17 +414,17 @@ following command from **inside the AHA container** to generate a one-time use U You should see output that looks similar to this:: - one-time use URL: ssl://aha.:27272/?certhash= + one-time use URL: ssl://00.aha.:27272/?certhash= Then the **user** may run:: - python -m synapse.tools.aha.enroll ssl://aha.:27272/?certhash= + python -m synapse.tools.aha.enroll ssl://00.aha.:27272/?certhash= Once they are enrolled, they will have a user certificate located in ``~/.syn/certs/users`` and their telepath configuration located in ``~/.syn/telepath.yaml`` will be updated to reflect the use of the AHA server. From there the user should be able to use standard Synapse CLI tools using the ``aha://`` URL such as:: - python -m synapse.tools.storm aha://visi@cortex. + python -m synapse.tools.storm aha://visi@cortex. .. _deployment-guide-storm-pool: diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py index a4fd7012451..e2c3c633544 100644 --- a/synapse/tests/test_tools_aha.py +++ b/synapse/tests/test_tools_aha.py @@ -9,6 +9,7 @@ import synapse.tests.utils as s_t_utils import synapse.tools.aha.list as s_a_list +import synapse.tools.aha.clone as s_a_clone import synapse.tools.aha.enroll as s_a_enroll import synapse.tools.aha.easycert as s_a_easycert import synapse.tools.aha.provision.user as s_a_provision_user @@ -93,6 +94,10 @@ async def test_aha_enroll(self): async with self.getTestAha() as aha: + argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link'] + retn, outp = await self.execToolMain(s_a_clone.main, argv) + self.isin('one-time use URL:', str(outp)) + argv = ['--url', aha.getLocalUrl(), 'visi'] retn, outp = await self.execToolMain(s_a_provision_user.main, argv) self.isin('one-time use URL:', str(outp)) diff --git a/synapse/tools/aha/clone.py b/synapse/tools/aha/clone.py new file mode 100644 index 00000000000..e9b76be96f5 --- /dev/null +++ b/synapse/tools/aha/clone.py @@ -0,0 +1,42 @@ +import sys +import asyncio +import argparse + +import synapse.exc as s_exc +import synapse.telepath as s_telepath + +import synapse.lib.output as s_output + +descr = ''' +Generate a new clone URL to deploy an AHA mirror. + +Examples: + + python -m synapse.tools.aha.clone 01.aha.loop.vertex.link + +''' + +async def main(argv, outp=s_output.stdout): + + pars = argparse.ArgumentParser(prog='synapse.tools.aha.clone', description=descr) + pars.add_argument('dnsname', help='The DNS name of the new AHA server.') + pars.add_argument('--port', type=int, default=27492, help='The port that the new AHA server should listen on.') + pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL to connect to the AHA service.') + + opts = pars.parse_args(argv) + + async with s_telepath.withTeleEnv(): + + try: + async with await s_telepath.openurl(opts.url) as aha: + curl = await aha.addAhaClone(opts.dnsname, port=opts.port) + outp.printf('one-time use URL: {curl}') + return 0 + + except s_exc.SynErr as e: + mesg = e.errinfo.get('mesg', repr(e)) + outp.printf(f'ERROR: {mesg}') + return 1 + +if __name__ == '__main__': # pragma: no cover + sys.exit(asyncio.run(main(sys.argv[1:]))) From df24baa8b910abdee74991d7f1200e40c263a31c Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 10:48:06 -0400 Subject: [PATCH 11/86] wip --- docs/synapse/deploymentguide.rst | 33 +++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index 106bf0f03ff..cd84c056ca4 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -20,6 +20,7 @@ orchestration mechanisms such as Kubernetes but for simplicity's sake, this guid ``docker compose`` based deployments. .. note:: + Due to `known networking limitations of docker on Mac`_ we do **not** support or recommend the use of Docker for Mac for testing or deploying production Synapse instances. Containers run within separate ``docker compose`` commands will not be able to reliably communicate with each other. @@ -29,6 +30,7 @@ within the directory ``/vertex/storage`` which should be a persistent mapped vol given volume at a time. .. note:: + To allow hosts to be provisioned on one system, this guide instructs you to disable HTTP API listening ports on all services other than the main Cortex. You may remove those configuration options if you are running on separate hosts or select alternate ports which do not conflict. @@ -74,6 +76,7 @@ by name, so it is likely that you need to create a DNS A/AAAA record in your exi using AHA, the only host that needs DNS or other external name resolution is the AHA service. .. note:: + FIXME AHA FLAT NETWORK STUFF Choose an AHA Network Name @@ -96,6 +99,7 @@ convention in the future. Once you have selected the DNS name for your AHA serve all deployed Synapse services will be able to resolve that name. .. note:: + It is important to ensure that ``00.aha.`` is resolvable via DNS or docker container service name resolution from within the container environment! @@ -121,7 +125,7 @@ Create the ``/srv/syn/00.aha/docker-compose.yaml`` file with contents:: # disable HTTPS API for now to prevent port collisions - SYN_AHA_HTTPS_PORT=null - SYN_AHA_AHA_NETWORK=prod.synapse - - SYN_AHA_DNS_NAME=00.aha. + - SYN_AHA_DNS_NAME=00.aha. .. note:: @@ -155,6 +159,11 @@ For high-availability deployments, you will want to deploy an AHA mirror or two. mirroring is configured using AHA based provisioning. Bootstrapping AHA mirrors is simple, but requires a slightly different procedure because you cannot bootstrap AHA via AHA :) +.. note:: + + You can deploy AHA mirrors at any time in the future. Once the mirrors are deployed, each AHA enabled + Synapse service will retrieve the updated list of AHA servers the next time it is restarted. + For this example, we will assume you chose a DNS name for your primary AHA server similar to the steps listed above. If so, you can simply replace ``00`` with sequential numbers and repeat this step to deploy however many AHA mirrors you deem appropriate. @@ -175,8 +184,25 @@ Create the container directory:: mkdir -p /srv/syn/01.aha/storage chown -R 999 /srv/syn/01.aha/storage -::NOTE You can deploy AHA mirrors at any time in the future. Once the mirrors are deployed, each AHA enabled - Synapse service will retrieve the updated list of AHA servers the next time it is restarted. +Create the ``/srv/syn/01.aha/docker-compose.yaml`` file with contents:: + + services: + 01.aha: + user: "999" + image: vertexproject/synapse-aha:v2.x.x + network_mode: host + restart: unless-stopped + volumes: + - ./storage:/vertex/storage + environment: + # disable HTTPS API for now to prevent port collisions + - SYN_AHA_HTTPS_PORT=null + - SYN_AHA_CLONE=ssl://00.aha.:27272/?certhash= + +Start the container:: + + docker compose --file /srv/syn/01.aha/docker-compose.yaml pull + docker compose --file /srv/syn/01.aha/docker-compose.yaml up -d Deploy Axon Service =================== @@ -404,6 +430,7 @@ Cortex container**:: python -m synapse.tools.moduser --add --admin true visi .. note:: + If you are a Synapse Enterprise customer, using the Synapse UI with SSO, the admin may now login to the Synapse UI. You may skip the following steps if the admin will not be using CLI tools to access the Cortex. From b5a543ea11bed2b811c4d3e5e94c36db1c67f261 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 11:32:05 -0400 Subject: [PATCH 12/86] wip --- docs/synapse/deploymentguide.rst | 4 ++++ synapse/lib/cell.py | 4 ++++ synapse/tests/test_lib_config.py | 3 ++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index cd84c056ca4..c4bc8b8b54f 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -171,6 +171,8 @@ however many AHA mirrors you deem appropriate. NOTE: AHA uses two default ports ETC. The following steps assume you will be running each of your AHA servers on a different host. The use of ``network_mode; host`` ETC +**Inside the AHA container** + Generate a one-time use provisioning URL:: python -m synapse.tools.aha.clone 01.aha. @@ -179,6 +181,8 @@ You should see output that looks similar to this:: one-time use URL: ssl://00.aha.:27272/?certhash= +**On the Host** + Create the container directory:: mkdir -p /srv/syn/01.aha/storage diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index c06497ffe81..20f1621338d 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1496,6 +1496,10 @@ async def _initAhaRegistry(self): if self.ahaclient is not None: await self.ahaclient.fini() + async def onlink(proxy): + ahauser = self.conf.req('aha:user') + ahaurls = await proxy.getAhaUrls(user=ahauser) + self.ahaclient = await s_telepath.Client.anit(ahaurl) self.onfini(self.ahaclient) diff --git a/synapse/tests/test_lib_config.py b/synapse/tests/test_lib_config.py index cb61a6ae3d4..f04de15b829 100644 --- a/synapse/tests/test_lib_config.py +++ b/synapse/tests/test_lib_config.py @@ -285,7 +285,8 @@ async def test_config_fromcell(self): # Trying to make a cell with a missing key it wants fails async with await SchemaCell.anit(dirn, conf={}) as cell: pass - self.eq(cm.exception.get('key'), 'apikey') + + self.eq(cm.exception.get('name'), 'apikey') def test_hideconf(self): hide_schema = { From d891c82a477913590ea441445c164bfbda36f916 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 11:47:42 -0400 Subject: [PATCH 13/86] wip --- synapse/lib/cell.py | 12 +++++++----- synapse/lib/nexus.py | 11 +++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 20f1621338d..a02be00bce8 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1190,7 +1190,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): root = await self._ctorNexsRoot() - await self.setNexsRoot(root) + self.setNexsRoot(root) self.apikeydb = self.slab.initdb('user:apikeys') # apikey -> useriden self.usermetadb = self.slab.initdb('user:meta') # useriden + -> dict valu @@ -1489,16 +1489,18 @@ async def _runSysctlLoop(self): async def _initAhaRegistry(self): # NOTE: This API needs to be safe to call again - ahaurl = self.conf.get('aha:registry') - if ahaurl is not None: + ahaurls = self.conf.get('aha:registry') + if ahaurls is not None: - info = await s_telepath.addAhaUrl(ahaurl) + info = await s_telepath.addAhaUrl(ahaurls) if self.ahaclient is not None: await self.ahaclient.fini() async def onlink(proxy): ahauser = self.conf.req('aha:user') - ahaurls = await proxy.getAhaUrls(user=ahauser) + newurls = await proxy.getAhaUrls(user=ahauser) + if newurls and newurls != self.conf.get('aha:registry'): + self.modCellConf({'aha:registry': newurls}) self.ahaclient = await s_telepath.Client.anit(ahaurl) self.onfini(self.ahaclient) diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py index 2f00e49c343..e3c6f4dca2d 100644 --- a/synapse/lib/nexus.py +++ b/synapse/lib/nexus.py @@ -614,12 +614,12 @@ async def __anit__(self, iden: str, nexsroot: NexsRoot = None): # type: ignore self.nexsroot: Optional[NexsRoot] = None if nexsroot is not None: - await self.setNexsRoot(nexsroot) + self.setNexsRoot(nexsroot) for event, func, passitem in self._regclstupls: # type: ignore self._nexshands[event] = func, passitem - async def setNexsRoot(self, nexsroot): + def setNexsRoot(self, nexsroot): nexsroot._nexskids[self.nexsiden] = self @@ -627,8 +627,7 @@ def onfini(): prev = nexsroot._nexskids.pop(self.nexsiden, None) assert prev is not None, f'Failed removing {self.nexsiden}' - nexsroot.onfini(onfini) - self.onfini(nexsroot) + self.onfini(onfini) self.nexsroot = nexsroot @@ -639,9 +638,9 @@ async def modNexsRoot(self, ctor): nexsroot = await ctor() - await self.setNexsRoot(nexsroot) + self.setNexsRoot(nexsroot) - await self.nexsroot.startup() + await nexsroot.startup() @classmethod def onPush(cls, event: str, passitem=False) -> Callable: From f31f1af2924c3a478b5677c6acf43c4280fecc17 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 11:54:29 -0400 Subject: [PATCH 14/86] wip --- synapse/lib/cell.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index a02be00bce8..f8105eef443 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1488,7 +1488,6 @@ async def _runSysctlLoop(self): async def _initAhaRegistry(self): - # NOTE: This API needs to be safe to call again ahaurls = self.conf.get('aha:registry') if ahaurls is not None: @@ -1514,15 +1513,6 @@ async def fini(): if ahaadmin is not None: await self._addAdminUser(ahaadmin) - def reqAhaUser(self): - - ahauser = self.conf.get('aha:user') - if ahauser is not None: - return ahauser - - mesg = 'An aha:user configuration option is required.' - raise s_exc.NeedConfValu(mesg=mesg) - def _getDmonListen(self): lisn = self.conf.get('dmon:listen', s_common.novalu) From 7e2fb641d3189f8e82a4cc56ca418332a4bb4770 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 12:09:29 -0400 Subject: [PATCH 15/86] wip --- synapse/lib/aha.py | 4 ---- synapse/lib/cell.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 66264e8f0b1..0fd1272dccd 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -126,10 +126,6 @@ async def get(self): class AhaApi(s_cell.CellApi): - @s_cell.adminapi() - async def getAhaClone(self, iden): - return await self.cell.getAhaClone(iden) - @s_cell.adminapi() async def addAhaClone(self, host, port=27492, conf=None): return await self.cell.addAhaClone(host, port=port, conf=conf) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index f8105eef443..a94dfafa5dd 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1496,7 +1496,7 @@ async def _initAhaRegistry(self): await self.ahaclient.fini() async def onlink(proxy): - ahauser = self.conf.req('aha:user') + ahauser = self.conf.get('aha:user', 'root') newurls = await proxy.getAhaUrls(user=ahauser) if newurls and newurls != self.conf.get('aha:registry'): self.modCellConf({'aha:registry': newurls}) From 1d5e0df683737bc61c79897e0fd9eec961d93a27 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 14:00:37 -0400 Subject: [PATCH 16/86] wip --- synapse/lib/cell.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index a94dfafa5dd..101e1940021 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1501,11 +1501,11 @@ async def onlink(proxy): if newurls and newurls != self.conf.get('aha:registry'): self.modCellConf({'aha:registry': newurls}) - self.ahaclient = await s_telepath.Client.anit(ahaurl) + self.ahaclient = await s_telepath.Client.anit(ahaurls) self.onfini(self.ahaclient) async def fini(): - await s_telepath.delAhaUrl(ahaurl) + await s_telepath.delAhaUrl(ahaurls) self.ahaclient.onfini(fini) From ff884dcb83d7bcf042824395b8f3b00f1b7d7e5d Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 14:31:55 -0400 Subject: [PATCH 17/86] wip --- synapse/lib/cell.py | 2 +- synapse/lib/lmdbslab.py | 2 ++ synapse/lib/nexus.py | 11 ++++------- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 101e1940021..dc09a99156e 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -904,7 +904,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware): }, 'nexslog:async': { 'default': True, - 'description': 'Set to false to disable async memory mapping of the nexus change log.', + 'description': 'Deprecated. This option ignored.', 'type': 'boolean', 'hidedocs': True, 'hidecmdl': True, diff --git a/synapse/lib/lmdbslab.py b/synapse/lib/lmdbslab.py index c1148d3ad77..8aeb0b19fa3 100644 --- a/synapse/lib/lmdbslab.py +++ b/synapse/lib/lmdbslab.py @@ -709,6 +709,8 @@ async def __anit__(self, path, **kwargs): kwargs.setdefault('lockmemory', False) kwargs.setdefault('map_async', True) + assert kwargs.get('map_async') + opts = kwargs self.path = path diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py index 9a4f1071379..805d602ea06 100644 --- a/synapse/lib/nexus.py +++ b/synapse/lib/nexus.py @@ -109,9 +109,7 @@ async def __anit__(self, cell): logpath = s_common.genpath(self.dirn, 'slabs', 'nexuslog') - self.map_async = self.cell.conf.get('nexslog:async') - self.nexsslab = await s_lmdbslab.Slab.anit(path, map_async=self.map_async) - self.nexsslab.addResizeCallback(cell.checkFreeSpace) + self.nexsslab = await cell._initSlabFile(path) self.nexshot = await self.nexsslab.getHotCount('nexs:indx') @@ -125,8 +123,7 @@ async def __anit__(self, cell): elif vers != 2: raise s_exc.BadStorageVersion(mesg=f'Got nexus log version {vers}. Expected 2. Accidental downgrade?') - slabopts = {'map_async': self.map_async} - self.nexslog = await s_multislabseqn.MultiSlabSeqn.anit(logpath, slabopts=slabopts, cell=cell) + self.nexslog = await s_multislabseqn.MultiSlabSeqn.anit(logpath, cell=cell) # just in case were previously configured differently logindx = self.nexslog.index() @@ -194,8 +191,7 @@ async def _migrateV1toV2(self, nexspath, logpath): # Open a fresh slab where the old one used to be logger.warning(f'Re-opening fresh nexslog slab at {nexspath} for nexshot') - self.nexsslab = await s_lmdbslab.Slab.anit(nexspath, map_async=self.map_async) - self.nexsslab.addResizeCallback(self.cell.checkFreeSpace) + self.nexsslab = await self.cell._initSlabFile(nexspath) self.nexshot = await self.nexsslab.getHotCount('nexs:indx') @@ -633,6 +629,7 @@ def onfini(): assert prev is not None, f'Failed removing {self.nexsiden}' self.onfini(onfini) + self.onfini(nexsroot) self.nexsroot = nexsroot From 562cea715941bfbcd48ac6e5b31e846738ad205d Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 15:27:10 -0400 Subject: [PATCH 18/86] wip --- synapse/cortex.py | 12 ++++++++---- synapse/lib/cell.py | 9 +++++++-- synapse/lib/layer.py | 5 ----- synapse/lib/lmdbslab.py | 4 +--- synapse/lib/nexus.py | 3 --- synapse/tests/test_cortex.py | 19 +++---------------- synapse/tests/utils.py | 8 +------- 7 files changed, 20 insertions(+), 40 deletions(-) diff --git a/synapse/cortex.py b/synapse/cortex.py index 168d18ba865..0350872ec94 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -773,13 +773,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell): # type: ignore }, 'layer:lmdb:map_async': { 'default': True, - 'description': 'Set the default lmdb:map_async value in LMDB layers.', - 'type': 'boolean' + 'description': 'Deprecated. This value is ignored.', + 'type': 'boolean', + 'hidecmdl': True, + 'hideconf': True, }, 'layer:lmdb:max_replay_log': { 'default': 10000, - 'description': 'Set the max size of the replay log for all layers.', - 'type': 'integer' + 'description': 'Deprecated. This value is ignored.', + 'type': 'integer', + 'hidecmdl': True, + 'hideconf': True, }, 'layers:lockmemory': { 'default': False, diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index dc09a99156e..e1af664d45a 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1188,9 +1188,14 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): await self.initServiceEarly() - root = await self._ctorNexsRoot() + nexsroot = await self._ctorNexsRoot() - self.setNexsRoot(root) + self.setNexsRoot(nexsroot) + + async def fini(): + await self.nexsroot.fini() + + self.onfini(fini) self.apikeydb = self.slab.initdb('user:apikeys') # apikey -> useriden self.usermetadb = self.slab.initdb('user:meta') # useriden + -> dict valu diff --git a/synapse/lib/layer.py b/synapse/lib/layer.py index 5ae38bf3c9d..e7f4063fee9 100644 --- a/synapse/lib/layer.py +++ b/synapse/lib/layer.py @@ -1409,9 +1409,6 @@ async def __anit__(self, core, layrinfo): self.growsize = self.layrinfo.get('growsize') self.logedits = self.layrinfo.get('logedits') - self.mapasync = core.conf.get('layer:lmdb:map_async') - self.maxreplaylog = core.conf.get('layer:lmdb:max_replay_log') - # slim hooks to avoid async/fire self.nodeAddHook = None self.nodeDelHook = None @@ -2704,8 +2701,6 @@ async def _initLayerStorage(self): slabopts = { 'readahead': True, 'lockmemory': self.lockmemory, - 'map_async': self.mapasync, - 'max_replay_log': self.maxreplaylog, } if self.growsize is not None: diff --git a/synapse/lib/lmdbslab.py b/synapse/lib/lmdbslab.py index 8aeb0b19fa3..02023fa2bcc 100644 --- a/synapse/lib/lmdbslab.py +++ b/synapse/lib/lmdbslab.py @@ -696,7 +696,7 @@ async def getSlabStats(clas): 'recovering': slab.recovering, 'maxsize': slab.maxsize, 'growsize': slab.growsize, - 'mapasync': slab.mapasync, + 'mapasync': True, }) return retn @@ -755,8 +755,6 @@ async def __anit__(self, path, **kwargs): logger.info(f'SYN_LOCKMEM_DISABLE envar set, skipping lockmem for {self.path}') self.lockmemory = False - self.mapasync = opts.setdefault('map_async', True) - self.mapsize = _mapsizeround(mapsize) if self.maxsize is not None: self.mapsize = min(self.mapsize, self.maxsize) diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py index 805d602ea06..fbebc3efdf2 100644 --- a/synapse/lib/nexus.py +++ b/synapse/lib/nexus.py @@ -304,7 +304,6 @@ async def issue(self, nexsiden, event, args, kwargs, meta=None): If I'm not a follower, mutate, otherwise, ask the leader to make the change and wait for the follower loop to hand me the result through a future. ''' - # pick up a reference to avoid race when we eventually can promote client = self.client @@ -629,8 +628,6 @@ def onfini(): assert prev is not None, f'Failed removing {self.nexsiden}' self.onfini(onfini) - self.onfini(nexsroot) - self.nexsroot = nexsroot async def modNexsRoot(self, ctor): diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py index bf2b9ca9413..e265ec605bc 100644 --- a/synapse/tests/test_cortex.py +++ b/synapse/tests/test_cortex.py @@ -4410,10 +4410,7 @@ async def test_cortex_nexslogen_off(self): ''' Everything still works when no nexus log is kept ''' - conf = {'layer:lmdb:map_async': True, - 'nexslog:en': False, - 'layers:logedits': True, - } + conf = {'nexslog:en': False, 'layers:logedits': True} async with self.getTestCore(conf=conf) as core: self.len(2, await core.nodes('[test:str=foo test:str=bar]')) self.len(2, await core.nodes('test:str')) @@ -4422,10 +4419,7 @@ async def test_cortex_logedits_off(self): ''' Everything still works when no layer log is kept ''' - conf = {'layer:lmdb:map_async': True, - 'nexslog:en': True, - 'layers:logedits': False, - } + conf = {'nexslog:en': True, 'layers:logedits': False} async with self.getTestCore(conf=conf) as core: self.len(2, await core.nodes('[test:str=foo test:str=bar]')) self.len(2, await core.nodes('test:str')) @@ -4443,18 +4437,12 @@ async def test_cortex_layer_settings(self): ''' Make sure settings make it down to the slab ''' - conf = { - 'layer:lmdb:map_async': False, - 'layer:lmdb:max_replay_log': 500, - 'layers:lockmemory': True, - } + conf = {'layers:lockmemory': True} async with self.getTestCore(conf=conf) as core: layr = core.getLayer() slab = layr.layrslab self.true(slab.lockmemory) - self.eq(500, slab.max_xactops_len) - self.true(500, slab.mapasync) async def test_feed_syn_nodes(self): @@ -8141,7 +8129,6 @@ async def initNexusSubsystem(self): return ret conf = { - 'layer:lmdb:map_async': True, 'nexslog:en': True, 'layers:logedits': True, } diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index f1487db931d..f1cf8019fc4 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -1311,14 +1311,8 @@ async def getTestCore(self, conf=None, dirn=None): with self.withNexusReplay(): - if dirn is not None: - - async with await s_cortex.Cortex.anit(dirn, conf=conf) as core: - yield core - - return + with self.mayTestDir(dirn) as dirn: - with self.getTestDir() as dirn: async with await s_cortex.Cortex.anit(dirn, conf=conf) as core: yield core From 0fc325733d182a2cc58d8afa9f2a44e2a32c5268 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 15:45:37 -0400 Subject: [PATCH 19/86] wip --- synapse/tests/test_lib_cell.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index a202ec1dab0..fb7d488f9c8 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -606,7 +606,6 @@ async def coro(prox, offs): async with self.getTestCell(s_cell.Cell, dirn=dir0, conf=conf) as cell00, \ cell00.getLocalProxy() as prox00: - self.true(cell00.nexsroot.map_async) self.true(cell00.nexsroot.donexslog) await prox00.addUser('test') From 27cfd9f6407df82bf3012c57138a43141b9e417c Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 5 Jul 2024 18:17:36 -0400 Subject: [PATCH 20/86] improve coverage --- synapse/telepath.py | 5 ++++- synapse/tests/test_lib_nexus.py | 8 ++++++++ synapse/tests/test_tools_aha.py | 4 ++++ synapse/tools/aha/clone.py | 7 +++++-- 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/synapse/telepath.py b/synapse/telepath.py index 63a98875b8f..76508f061aa 100644 --- a/synapse/telepath.py +++ b/synapse/telepath.py @@ -1543,7 +1543,7 @@ async def openinfo(info): path, name = path.split(':') link = await s_link.unixconnect(path) - else: + elif scheme in ('tcp', 'ssl'): path = info.get('path') name = info.get('name', path[1:]) @@ -1592,6 +1592,9 @@ async def openinfo(info): link = await s_link.connect(host, port, linkinfo=linkinfo) + else: + raise s_exc.BadUrl(mesg=f'Invalid URL scheme: {scheme}') + prox = await Proxy.anit(link, name) prox.onfini(link) diff --git a/synapse/tests/test_lib_nexus.py b/synapse/tests/test_lib_nexus.py index 81f5ea3eb70..e4b1521b155 100644 --- a/synapse/tests/test_lib_nexus.py +++ b/synapse/tests/test_lib_nexus.py @@ -111,6 +111,14 @@ async def test_nexus(self): stream.seek(0) self.isin('while replaying log', stream.read()) + async def test_nexus_modroot(self): + + async with self.getTestCell(s_cell.Cell) as cell: + await cell.sync() + async with cell.nexslock: + cell.modNexsRoot(cell._ctorNexsRoot) + await cell.sync() + async def test_nexus_mixin(self): with self.getTestDir() as dirn: dir1 = s_common.genpath(dirn, 'nexus1') diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py index e2c3c633544..e3ca70d4ba0 100644 --- a/synapse/tests/test_tools_aha.py +++ b/synapse/tests/test_tools_aha.py @@ -98,6 +98,10 @@ async def test_aha_enroll(self): retn, outp = await self.execToolMain(s_a_clone.main, argv) self.isin('one-time use URL:', str(outp)) + argv = ['--url', 'newp://1.2.3.4', '01.aha.loop.vertex.link'] + retn, outp = await self.execToolMain(s_a_clone.main, argv) + self.isin('ERROR: Invalid URL scheme: newp', str(outp)) + argv = ['--url', aha.getLocalUrl(), 'visi'] retn, outp = await self.execToolMain(s_a_provision_user.main, argv) self.isin('one-time use URL:', str(outp)) diff --git a/synapse/tools/aha/clone.py b/synapse/tools/aha/clone.py index e9b76be96f5..878b08f7425 100644 --- a/synapse/tools/aha/clone.py +++ b/synapse/tools/aha/clone.py @@ -33,8 +33,11 @@ async def main(argv, outp=s_output.stdout): outp.printf('one-time use URL: {curl}') return 0 - except s_exc.SynErr as e: - mesg = e.errinfo.get('mesg', repr(e)) + except Exception as e: + mesg = repr(e) + if isinstance(e, s_exc.SynErr): + mesg = e.errinfo.get('mesg', repr(e)) + outp.printf(f'ERROR: {mesg}') return 1 From b9bb3f06f58ec94a0bd79207c58cff51cc0f0c9c Mon Sep 17 00:00:00 2001 From: visi Date: Sat, 6 Jul 2024 10:19:06 -0400 Subject: [PATCH 21/86] wip --- synapse/lib/cell.py | 6 +- synapse/lib/nexus.py | 3 + synapse/telepath.py | 40 ++++++---- synapse/tests/test_lib_aha.py | 138 +++++++++++----------------------- 4 files changed, 76 insertions(+), 111 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index e1af664d45a..b6281b42e76 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1503,10 +1503,12 @@ async def _initAhaRegistry(self): async def onlink(proxy): ahauser = self.conf.get('aha:user', 'root') newurls = await proxy.getAhaUrls(user=ahauser) - if newurls and newurls != self.conf.get('aha:registry'): + oldurls = tuple(self.conf.get('aha:registry')) + if newurls and newurls != oldurls: self.modCellConf({'aha:registry': newurls}) + self.ahaclient.setBootUrls(newurls) - self.ahaclient = await s_telepath.Client.anit(ahaurls) + self.ahaclient = await s_telepath.Client.anit(ahaurls, onlink=onlink) self.onfini(self.ahaclient) async def fini(): diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py index fbebc3efdf2..8322f5d8f17 100644 --- a/synapse/lib/nexus.py +++ b/synapse/lib/nexus.py @@ -571,6 +571,9 @@ async def runMirrorLoop(self, proxy): if respfutu is not None: respfutu.set_result(retn) + except s_exc.LinkShutDown: + logger.warning(f'mirror loop: leader closed the connection.') + except Exception as exc: # pragma: no cover logger.exception(f'error in mirror loop: {exc}') diff --git a/synapse/telepath.py b/synapse/telepath.py index 76508f061aa..5c538bd8a03 100644 --- a/synapse/telepath.py +++ b/synapse/telepath.py @@ -997,12 +997,6 @@ async def __anit__(self, urlinfo, onlink=None): await s_base.Base.__anit__(self) - # some ugly stuff in order to be backward compatible... - if not isinstance(urlinfo, (list, tuple)): - urlinfo = (urlinfo,) - - urlinfo = [chopurl(u) for u in urlinfo] - self.aha = None self.clients = {} @@ -1012,9 +1006,9 @@ async def __anit__(self, urlinfo, onlink=None): self.onlink = onlink - self.booturls = urlinfo self.bootdeque = collections.deque() - self.bootdeque.extend(self.booturls) + + self.setBootUrls(urlinfo) self.ready = asyncio.Event() self.deque = collections.deque() @@ -1033,6 +1027,16 @@ async def fini(): self.schedCoro(self._initBootProxy()) + def setBootUrls(self, urlinfo): + + if not isinstance(urlinfo, (list, tuple)): + urlinfo = (urlinfo,) + + self.booturls = [chopurl(u) for u in urlinfo] + + self.bootdeque.clear() + self.bootdeque.extend(self.booturls) + def getNextBootUrl(self): if not self.bootdeque: self.bootdeque.extend(self.booturls) @@ -1210,20 +1214,16 @@ async def __anit__(self, urlinfo, opts=None, conf=None, onlink=None): await s_base.Base.__anit__(self) - if isinstance(urlinfo, (str, dict)): - urlinfo = (urlinfo,) - - urlinfo = [chopurl(u) for u in urlinfo] - if conf is None: conf = {} if opts is None: opts = {} - self._t_urlinfo = urlinfo self._t_urldeque = collections.deque() + self.setBootUrls(urlinfo) + self._t_opts = opts self._t_conf = conf @@ -1247,6 +1247,16 @@ async def fini(): await self._fireLinkLoop() + def setBootUrls(self, urlinfo): + + if not isinstance(urlinfo, (list, tuple)): + urlinfo = (urlinfo,) + + self._t_urlinfo = [chopurl(u) for u in urlinfo] + + self._t_urldeque.clear() + self._t_urldeque.extend(self._t_urlinfo) + def _getNextUrl(self): if not self._t_urldeque: self._t_urldeque.extend(self._t_urlinfo) @@ -1281,7 +1291,7 @@ async def _teleLinkLoop(self): now = time.monotonic() if now > lastlog + 60.0: # don't logspam the disconnect message more than 1/min url = s_urlhelp.sanitizeUrl(zipurl(urlinfo)) - logger.exception(f'telepath client ({url}) encountered an error: {e}') + logger.warning(f'telepath client ({url}) encountered an error: {e}') lastlog = now await self.waitfini(timeout=self._t_conf.get('retrysleep', 0.2)) diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index ffc716e0d4f..083c0ffd55b 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -131,7 +131,7 @@ async def test_lib_aha_offon(self): svc = await aha.getAhaSvc('0.cryo...') self.notin('online', svc.get('svcinfo')) - async def test_lib_aha(self): + async def test_lib_aha_basics(self): with self.raises(s_exc.NoSuchName): await s_telepath.getAhaProxy({}) @@ -159,40 +159,26 @@ async def test_lib_aha(self): cryo0_dirn = s_common.gendir(aha.dirn, 'cryo0') - host, port = await aha.dmon.listen('tcp://127.0.0.1:0') - await aha.auth.rootuser.setPasswd('hehehaha') + ahaurls = await aha.getAhaUrls() wait00 = aha.waiter(1, 'aha:svcadd') - conf = { - 'aha:name': '0.cryo', - 'aha:leader': 'cryo', - 'aha:network': 'synapse', - 'aha:admin': 'root@synapse', - 'aha:registry': [f'tcp://root:hehehaha@127.0.0.1:{port}', - f'tcp://root:hehehaha@127.0.0.1:{port}'], - 'dmon:listen': 'tcp://0.0.0.0:0/', - } - async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo: - await cryo.auth.rootuser.setPasswd('secret') - - ahaadmin = await cryo.auth.getUserByName('root@synapse') - self.nn(ahaadmin) - self.true(ahaadmin.isAdmin()) + conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')} + async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo: await wait00.wait(timeout=2) with self.raises(s_exc.NoSuchName): await s_telepath.getAhaProxy({'host': 'hehe.haha'}) - async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy: + async with await s_telepath.openurl('aha://cryo...') as proxy: self.nn(await proxy.getCellIden()) with self.raises(s_exc.BadArg): _proxy = await cryo.ahaclient.proxy(timeout=2) await _proxy.modAhaSvcInfo('cryo...', {'newp': 'newp'}) - async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: + async with await s_telepath.openurl('aha://0.cryo...') as proxy: self.nn(await proxy.getCellIden()) # force a reconnect... @@ -201,7 +187,7 @@ async def test_lib_aha(self): await proxy.fini() self.nn(await waiter.wait(timeout=6)) - async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy: + async with await s_telepath.openurl('aha://cryo...') as proxy: self.nn(await proxy.getCellIden()) waiter = aha.waiter(1, 'aha:svcadd') @@ -209,72 +195,51 @@ async def test_lib_aha(self): await cryo.setCellActive(False) with self.raises(s_exc.NoSuchName): - async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy: + async with await s_telepath.openurl('aha://cryo...') as proxy: pass self.nn(await waiter.wait(timeout=6)) - async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy: + async with await s_telepath.openurl('aha://0.cryo...') as proxy: self.nn(await proxy.getCellIden()) await cryo.setCellActive(True) - async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy: + async with await s_telepath.openurl('aha://cryo...') as proxy: self.nn(await proxy.getCellIden()) # some coverage edge cases... cryo.conf.pop('aha:leader', None) await cryo.setCellActive(False) - - # lock the aha:admin account so we can confirm it is unlocked upon restart # remove the admin flag from the account. - self.false(ahaadmin.isLocked()) - await ahaadmin.setLocked(True, logged=False) - self.true(ahaadmin.isLocked()) - # remove the admin status so we can confirm its an admin upon restart - await ahaadmin.setAdmin(False, logged=False) - self.false(ahaadmin.isAdmin()) - - async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo: - ahaadmin = await cryo.auth.getUserByName('root@synapse') - # And we should be unlocked and admin now - self.false(ahaadmin.isLocked()) - self.true(ahaadmin.isAdmin()) wait01 = aha.waiter(1, 'aha:svcadd') - conf = { - 'aha:name': '0.cryo', - 'aha:leader': 'cryo', - 'aha:network': 'foo', - 'aha:registry': f'tcp://root:hehehaha@127.0.0.1:{port}', - 'dmon:listen': 'tcp://0.0.0.0:0/', - } + conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')} async with self.getTestCryo(conf=conf) as cryo: info = await cryo.getCellInfo() cnfo = info.get('cell') anfo = cnfo.get('aha') - self.eq(cnfo.get('aha'), {'name': '0.cryo', 'leader': 'cryo', 'network': 'foo'}) - - await cryo.auth.rootuser.setPasswd('secret') + self.eq(cnfo.get('aha'), {'name': '0.cryo', 'leader': 'cryo', 'network': 'synapse'}) await wait01.wait(timeout=2) - async with await s_telepath.openurl('aha://root:secret@cryo.foo') as proxy: + async with await s_telepath.openurl('aha://cryo.synapse') as proxy: self.nn(await proxy.getCellIden()) - async with await s_telepath.openurl('aha://root:secret@0.cryo.foo') as proxy: + async with await s_telepath.openurl('aha://0.cryo.synapse') as proxy: self.nn(await proxy.getCellIden()) await proxy.puts('hehe', ('hehe', 'haha')) - async with await s_telepath.openurl('aha://root:secret@0.cryo.foo/*/hehe') as proxy: + async with await s_telepath.openurl('aha://0.cryo.synapse/*/hehe') as proxy: self.nn(await proxy.iden()) - async with await s_telepath.openurl(f'tcp://root:hehehaha@127.0.0.1:{port}') as ahaproxy: - svcs = [x async for x in ahaproxy.getAhaSvcs('foo')] + async with aha.getLocalProxy() as ahaproxy: + + svcs = [x async for x in ahaproxy.getAhaSvcs('synapse')] self.len(2, svcs) names = [s['name'] for s in svcs] - self.sorteq(('cryo.foo', '0.cryo.foo'), names) + self.sorteq(('cryo.synapse', '0.cryo.synapse'), names) self.none(await ahaproxy.getCaCert('vertex.link')) cacert0 = await ahaproxy.genCaCert('vertex.link') @@ -301,20 +266,10 @@ async def test_lib_aha(self): self.nn(usercert01) self.ne(usercert00, usercert01) - async with await s_telepath.openurl(f'tcp://root:hehehaha@127.0.0.1:{port}') as ahaproxy: - await ahaproxy.delAhaSvc('cryo', network='foo') - await ahaproxy.delAhaSvc('0.cryo', network='foo') - self.none(await ahaproxy.getAhaSvc('cryo.foo')) - self.none(await ahaproxy.getAhaSvc('0.cryo.foo')) - self.len(2, [s async for s in ahaproxy.getAhaSvcs()]) - - with self.raises(s_exc.BadArg): - info = {'urlinfo': {'host': '127.0.0.1', 'port': 8080, 'scheme': 'tcp'}} - await ahaproxy.addAhaSvc('newp', info, network=None) - # We can use HTTP API to get the registered services await aha.addUser('lowuser', passwd='lowuser') await aha.auth.rootuser.setPasswd('secret') + host, httpsport = await aha.addHttpsPort(0) svcsurl = f'https://localhost:{httpsport}/api/v1/aha/services' @@ -359,37 +314,32 @@ async def test_lib_aha(self): self.eq(info.get('status'), 'err') self.eq(info.get('code'), 'AuthDeny') - # The aha service can also be configured with a set of URLs that could represent itself. - urls = ('cell://home0', 'cell://home1') - conf = {'aha:urls': urls} - async with self.getTestAha(conf=conf) as aha: async with aha.getLocalProxy() as ahaproxy: - aurls = await ahaproxy.getAhaUrls() - self.eq(urls, aurls) + await ahaproxy.delAhaSvc('cryo', network='synapse') + await ahaproxy.delAhaSvc('0.cryo', network='synapse') + self.none(await ahaproxy.getAhaSvc('cryo.synapse')) + self.none(await ahaproxy.getAhaSvc('0.cryo.synapse')) + self.len(0, [s async for s in ahaproxy.getAhaSvcs()]) - with self.getTestDir() as dirn: - conf = { - 'aha:name': '0.test', - 'aha:leader': 'test', - 'aha:network': 'foo', - 'aha:registry': f'tcp://root:hehehaha@127.0.0.1:{port}', - 'dmon:listen': f'unix://{dirn}/sock' - } - async with self.getTestAha(conf=conf) as aha: - ahainfo = await aha.getAhaInfo() - uinfo = ahainfo.get('urlinfo', {}) - self.eq(uinfo.get('scheme'), 'unix') - self.none(uinfo.get('port')) - self.len(1, await aha.getAhaUrls()) - - conf['dmon:listen'] = 'tcp://0.0.0.0:0/' - async with self.getTestAha(conf=conf) as aha: - ahainfo = await aha.getAhaInfo() - ahaurls = await aha.getAhaUrls() - uinfo = ahainfo.get('urlinfo', {}) - self.eq(uinfo.get('scheme'), 'tcp') - self.gt(uinfo.get('port'), 0) - self.eq(ahaurls[0], f'ssl://00.aha.loop.vertex.link:{aha.sockaddr[1]}?certname=root@foo') + with self.raises(s_exc.BadArg): + info = {'urlinfo': {'host': '127.0.0.1', 'port': 8080, 'scheme': 'tcp'}} + await ahaproxy.addAhaSvc('newp', info, network=None) + + # test that services get updated aha server list + with self.getTestDir() as dirn: + + conf = {'aha:provision': await aha.addAhaSvcProv('00.cell')} + + async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell: + self.len(1, cell.conf.get('aha:registry')) + + await aha.addAhaServer({'host': '01.aha.loop.vertex.link'}) + + self.len(2, await aha.getAhaServers()) + + async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell: + await cell.ahaclient.proxy() + self.len(2, cell.conf.get('aha:registry')) async def test_lib_aha_loadenv(self): From 92b498dd447aeb0412555a1cccbd01be976f2c1b Mon Sep 17 00:00:00 2001 From: visi Date: Sat, 6 Jul 2024 11:11:53 -0400 Subject: [PATCH 22/86] wip --- synapse/lib/base.py | 19 +++++++-- synapse/tests/test_lib_aha.py | 73 ++++++++++++++--------------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/synapse/lib/base.py b/synapse/lib/base.py index 1507a11bfe8..8fe9d6bb7a0 100644 --- a/synapse/lib/base.py +++ b/synapse/lib/base.py @@ -590,7 +590,7 @@ async def main(self): await self.addSignalHandlers() return await self.waitfini() - def waiter(self, count, *names): + def waiter(self, count, *names, timeout=None): ''' Construct and return a new Waiter for events on this base. @@ -615,16 +615,17 @@ def waiter(self, count, *names): race conditions with this mechanism ;) ''' - return Waiter(self, count, *names) + return Waiter(self, count, *names, timeout=timeout) class Waiter: ''' A helper to wait for a given number of events on a Base. ''' - def __init__(self, base, count, *names): + def __init__(self, base, count, *names, timeout=None): self.base = base self.names = names self.count = count + self.timeout = timeout self.event = asyncio.Event() self.events = [] @@ -656,6 +657,9 @@ async def wait(self, timeout=None): doStuff(evnt) ''' + if timeout is None: + timeout = self.timeout + try: retn = await s_coro.event_wait(self.event, timeout) @@ -676,6 +680,15 @@ def fini(self): self.base.unlink(self._onWaitEvent) del self.event + async def __aenter__(self): + return self + + async def __aexit__(self, exc, cls, tb): + if exc is None: + if await self.wait() is None: + mesg = f'timeout waiting for {self.count} events {self.names}' + raise s_exc.TimeOut(mesg=mesg) + class BaseRef(Base): ''' An object for managing multiple Base instances by name. diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 083c0ffd55b..7db50afc29d 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -182,45 +182,38 @@ async def test_lib_aha_basics(self): self.nn(await proxy.getCellIden()) # force a reconnect... - waiter = aha.waiter(1, 'aha:svcadd') proxy = await cryo.ahaclient.proxy(timeout=2) - await proxy.fini() - self.nn(await waiter.wait(timeout=6)) + async with aha.waiter(2, 'aha:svcadd'): + await proxy.fini() async with await s_telepath.openurl('aha://cryo...') as proxy: self.nn(await proxy.getCellIden()) - waiter = aha.waiter(1, 'aha:svcadd') # force the service into passive mode... - await cryo.setCellActive(False) + async with aha.waiter(3, 'aha:svcdown', 'aha:svcadd', timeout=6): + await cryo.setCellActive(False) with self.raises(s_exc.NoSuchName): async with await s_telepath.openurl('aha://cryo...') as proxy: pass - self.nn(await waiter.wait(timeout=6)) - async with await s_telepath.openurl('aha://0.cryo...') as proxy: self.nn(await proxy.getCellIden()) - await cryo.setCellActive(True) + async with aha.waiter(1, 'aha:svcadd', timeout=6): + await cryo.setCellActive(True) async with await s_telepath.openurl('aha://cryo...') as proxy: self.nn(await proxy.getCellIden()) - # some coverage edge cases... - cryo.conf.pop('aha:leader', None) - await cryo.setCellActive(False) - # remove the admin flag from the account. + wait01 = aha.waiter(2, 'aha:svcadd') - wait01 = aha.waiter(1, 'aha:svcadd') conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')} async with self.getTestCryo(conf=conf) as cryo: info = await cryo.getCellInfo() - cnfo = info.get('cell') - anfo = cnfo.get('aha') - self.eq(cnfo.get('aha'), {'name': '0.cryo', 'leader': 'cryo', 'network': 'synapse'}) + + self.eq(info['cell']['aha'], {'name': '0.cryo', 'leader': 'cryo', 'network': 'synapse'}) await wait01.wait(timeout=2) @@ -986,13 +979,12 @@ async def test_aha_service_pools(self): # Pool has no members.... pool = await s_telepath.open('aha://pool00...') self.eq(0, pool.size()) - waiter = pool.waiter(0, 'svc:add') - msgs = await core00.stormlist('aha.pool.svc.add pool00... 00...') - self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (00...) added to service pool (pool00.synapse)', msgs) + async with pool.waiter(1, 'svc:add', timeout=12): + msgs = await core00.stormlist('aha.pool.svc.add pool00... 00...') + self.stormHasNoWarnErr(msgs) + self.stormIsInPrint('AHA service (00...) added to service pool (pool00.synapse)', msgs) - self.len(1, await waiter.wait(timeout=12)) prox = await pool.proxy(timeout=12) info = await prox.getCellInfo() self.eq('00', info.get('cell').get('aha').get('name')) @@ -1017,17 +1009,15 @@ async def test_aha_service_pools(self): replay = s_common.envbool('SYNDEV_NEXUS_REPLAY') nevents = 5 if replay else 3 - waiter = pool.waiter(nevents, 'svc:add') - - msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...') - self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs) + async with pool.waiter(nevents, 'svc:add', timeout=3): - msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...') - self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs) + msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...') + self.stormHasNoWarnErr(msgs) + self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs) - await waiter.wait(timeout=3) + msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...') + self.stormHasNoWarnErr(msgs) + self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs) poolinfo = await aha.getAhaPool('pool00...') self.len(2, poolinfo['services']) @@ -1050,24 +1040,21 @@ async def test_aha_service_pools(self): waiter = pool.waiter(1, 'pool:reset') - ahaproxy = await pool.aha.proxy() - await ahaproxy.fini() - - await waiter.wait(timeout=3) + async with pool.waiter(1, 'pool:reset', timeout=3): + ahaproxy = await pool.aha.proxy() + await ahaproxy.fini() # wait for the pool to be notified of the topology change - waiter = pool.waiter(1, 'svc:del') + async with pool.waiter(1, 'svc:del', timeout=3): - msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...') - self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (00.synapse) removed from service pool (pool00.synapse)', - msgs) + msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...') + self.stormHasNoWarnErr(msgs) + self.stormIsInPrint('AHA service (00.synapse) removed from service pool (pool00.synapse)', msgs) - msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...') - self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('Did not remove (00...) from the service pool.', msgs) + msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...') + self.stormHasNoWarnErr(msgs) + self.stormIsInPrint('Did not remove (00...) from the service pool.', msgs) - await waiter.wait(timeout=3) run00 = await (await pool.proxy(timeout=3)).getCellRunId() self.eq(run00, await (await pool.proxy(timeout=3)).getCellRunId()) From 40c5e0478e7f65c40c0bf01ffb055a9dcc278261 Mon Sep 17 00:00:00 2001 From: visi Date: Sun, 7 Jul 2024 09:47:50 -0400 Subject: [PATCH 23/86] wip --- synapse/lib/nexus.py | 7 ++++- synapse/telepath.py | 1 + synapse/tests/test_lib_aha.py | 55 +++++++++++++++++++---------------- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py index 8322f5d8f17..8418232e8a5 100644 --- a/synapse/lib/nexus.py +++ b/synapse/lib/nexus.py @@ -143,6 +143,9 @@ async def fini(): self.onfini(fini) + def getNexsKids(self): + return list(self._nexskids.values()) + async def _migrateV1toV2(self, nexspath, logpath): ''' Close the slab, move it to the new multislab location, then copy out the nexshot @@ -635,12 +638,14 @@ def onfini(): async def modNexsRoot(self, ctor): + kids = [self] if self.nexsroot is not None: + kids = self.nexsroot.getNexsKids() await self.nexsroot.fini() nexsroot = await ctor() - self.setNexsRoot(nexsroot) + [kid.setNexsRoot(nexsroot) for kid in kids] await nexsroot.startup() diff --git a/synapse/telepath.py b/synapse/telepath.py index 5c538bd8a03..388bc2b108c 100644 --- a/synapse/telepath.py +++ b/synapse/telepath.py @@ -1273,6 +1273,7 @@ async def offlink(self, func): async def _fireLinkLoop(self): self._t_proxy = None self._t_ready.clear() + await self.fire('tele:client:linkloop') self.schedCoro(self._teleLinkLoop()) async def _teleLinkLoop(self): diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 7db50afc29d..e6ebb5988c2 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -53,11 +53,12 @@ async def test_lib_aha_clone(self): async with self.getTestAha(dirn=dir0) as aha0: + ahacount = len(await aha0.getAhaUrls()) async with aha0.getLocalProxy() as proxy0: - self.len(1, await proxy0.getAhaUrls()) - self.len(1, await proxy0.getAhaServers()) + self.len(ahacount, await proxy0.getAhaUrls()) + self.len(ahacount, await proxy0.getAhaServers()) - purl = await proxy0.addAhaClone('01.aha.loop.vertex.link') + purl = await proxy0.addAhaClone('zoinks.aha.loop.vertex.link') conf1 = {'clone': purl} async with self.getTestAha(conf=conf1, dirn=dir1) as aha1: @@ -70,7 +71,7 @@ async def test_lib_aha_clone(self): serv0 = await aha0.getAhaServers() serv1 = await aha1.getAhaServers() - self.len(2, serv0) + self.len(ahacount + 1, serv0) self.eq(serv0, serv1) # ensure some basic functionality is being properly mirrored @@ -89,16 +90,17 @@ async def test_lib_aha_clone(self): mnfo = await aha1.getAhaSvc('test.example.net') self.eq(mnfo.get('name'), 'test.example.net') - wait00 = aha0.waiter(1, 'aha:svcdown') - await aha0.setAhaSvcDown('test', iden, network='example.net') - self.isin(len(await wait00.wait(timeout=6)), (1, 2)) + async with aha0.waiter(1, 'aha:svcdown', timeout=6): + await aha0.setAhaSvcDown('test', iden, network='example.net') await aha1.sync() + mnfo = await aha1.getAhaSvc('test.example.net') self.notin('online', mnfo) await aha0.delAhaSvc('test', network='example.net') await aha1.sync() + mnfo = await aha1.getAhaSvc('test.example.net') self.none(mnfo) @@ -323,16 +325,19 @@ async def test_lib_aha_basics(self): conf = {'aha:provision': await aha.addAhaSvcProv('00.cell')} + # can't assume just one due to enterprise tests with raft... + ahacount = len(await aha.getAhaUrls()) + async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell: - self.len(1, cell.conf.get('aha:registry')) + self.len(ahacount, cell.conf.get('aha:registry')) - await aha.addAhaServer({'host': '01.aha.loop.vertex.link'}) + await aha.addAhaServer({'host': 'zoinks.aha.loop.vertex.link'}) - self.len(2, await aha.getAhaServers()) + self.len(ahacount + 1, await aha.getAhaServers()) async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell: await cell.ahaclient.proxy() - self.len(2, cell.conf.get('aha:registry')) + self.len(ahacount + 1, cell.conf.get('aha:registry')) async def test_lib_aha_loadenv(self): @@ -389,9 +394,11 @@ async def test_lib_aha_finid_cell(self): proxy = await cryo.ahaclient.proxy() - await aha.fini() + # avoid race to notify client... + async with cryo.ahaclient.waiter(1, 'tele:client:linkloop', timeout=2): + await aha.fini() + self.true(await proxy.waitfini(timeout=10)) - self.true(await proxy.waitfini(timeout=10)) with self.raises(asyncio.TimeoutError): await cryo.ahaclient.proxy(timeout=0.1) @@ -436,12 +443,12 @@ async def test_lib_aha_bootstrap(self): } async with self.getTestAha(dirn=dirn, conf=conf) as aha: - self.true(os.path.isfile(os.path.join(dirn, 'certs', 'cas', 'do.vertex.link.crt'))) - self.true(os.path.isfile(os.path.join(dirn, 'certs', 'cas', 'do.vertex.link.key'))) - self.true(os.path.isfile(os.path.join(dirn, 'certs', 'hosts', 'aha.do.vertex.link.crt'))) - self.true(os.path.isfile(os.path.join(dirn, 'certs', 'hosts', 'aha.do.vertex.link.key'))) - self.true(os.path.isfile(os.path.join(dirn, 'certs', 'users', 'root@do.vertex.link.crt'))) - self.true(os.path.isfile(os.path.join(dirn, 'certs', 'users', 'root@do.vertex.link.key'))) + self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'cas', 'do.vertex.link.crt'))) + self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'cas', 'do.vertex.link.key'))) + self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'hosts', 'aha.do.vertex.link.crt'))) + self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'hosts', 'aha.do.vertex.link.key'))) + self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'users', 'root@do.vertex.link.crt'))) + self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'users', 'root@do.vertex.link.key'))) host, port = await aha.dmon.listen('ssl://127.0.0.1:0?hostname=aha.do.vertex.link&ca=do.vertex.link') @@ -471,10 +478,7 @@ async def test_lib_aha_provision(self): with self.getTestDir() as dirn: - conf = { - 'aha:name': 'aha', - 'dns:name': 'aha.loop.vertex.link', - } + conf = {'dns:name': 'aha.loop.vertex.link'} async with self.getTestAha(dirn=dirn, conf=conf) as aha: ahaport = aha.sockaddr[1] @@ -556,7 +560,8 @@ async def test_lib_aha_provision(self): self.eq('00.axon', yamlconf.get('aha:name')) self.eq('synapse', yamlconf.get('aha:network')) self.none(yamlconf.get('aha:admin')) - self.eq((f'ssl://aha.loop.vertex.link:{ahaport}?certname=root@synapse',), yamlconf.get('aha:registry')) + + self.eq(await aha.getAhaUrls(), yamlconf.get('aha:registry')) self.eq(f'ssl://0.0.0.0:0?hostname=00.axon.synapse&ca=synapse', yamlconf.get('dmon:listen')) unfo = await axon.addUser('visi') @@ -583,7 +588,7 @@ async def test_lib_aha_provision(self): teleyaml = s_common.yamlload(syndir, 'telepath.yaml') self.eq(teleyaml.get('version'), 1) - self.eq(teleyaml.get('aha:servers'), (f'ssl://aha.loop.vertex.link:{ahaport}?certname=visi@synapse',)) + self.sorteq(teleyaml.get('aha:servers'), await aha.getAhaUrls(user='visi')) certdir = s_telepath.s_certdir.CertDir(os.path.join(syndir, 'certs')) async with await s_telepath.openurl('aha://visi@axon...', certdir=certdir) as prox: From 2c063ed7365c3d06e0c33dca090a56b975b26db8 Mon Sep 17 00:00:00 2001 From: visi Date: Sun, 7 Jul 2024 14:33:52 -0400 Subject: [PATCH 24/86] wip --- synapse/lib/aha.py | 49 ++++++++++- synapse/lib/cell.py | 33 ++++---- synapse/tests/test_lib_aha.py | 151 ++++++++++++---------------------- synapse/tests/utils.py | 3 +- 4 files changed, 117 insertions(+), 119 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 0fd1272dccd..9f9c03f2454 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -534,14 +534,15 @@ def getEnvPrefix(cls): async def _initCellBoot(self): - path = s_common.genpath(self.dirn, 'cell.guid') - if os.path.isfile(path): - return - curl = self.conf.get('clone') if curl is None: return + path = s_common.genpath(self.dirn, 'cell.guid') + if os.path.isfile(path): + logger.info('Cloneing AHA: cell.guid detected. Skipping.') + return + logger.warning(f'Cloning AHA: {curl}') async with await s_telepath.openurl(curl) as proxy: @@ -788,6 +789,43 @@ async def _clearInactiveSessions(self): # Wait until we are cancelled or the cell is fini. await self.waitfini() + async def _waitAhaSvcOnline(self, name, timeout=None): + + name = self._getAhaName(name) + print(f'WAITING FOR ONLINE {name}') + + while True: + + async with self.nexslock: + + retn = await self.getAhaSvc(name) + if retn['svcinfo'].get('online') is not None: + return retn + + waiter = self.waiter(1, f'aha:svcadd:{name}') + + if await waiter.wait(timeout=timeout) is None: + raise s_exc.TimeOut(mesg=f'Timeout waiting for aha:svcadd:{name}') + + async def _waitAhaSvcDown(self, name, timeout=None): + + name = self._getAhaName(name) + print(f'WAITING FOR DOWN {name}') + + while True: + + async with self.nexslock: + + retn = await self.getAhaSvc(name) + online = retn['svcinfo'].get('online') + if online is None: + return retn + + waiter = self.waiter(1, f'aha:svcdown:{name}') + + if await waiter.wait(timeout=timeout) is None: + raise s_exc.TimeOut(mesg=f'Timeout waiting for aha:svcdown:{name}') + async def getAhaSvcs(self, network=None): path = ('aha', 'services') if network is not None: @@ -1003,8 +1041,10 @@ async def setAhaSvcDown(self, name, linkiden, network=None): @s_nexus.Pusher.onPush('aha:svc:down') async def _setAhaSvcDown(self, name, linkiden, network=None): + svcname, svcnetw, svcfull = self._nameAndNetwork(name, network) path = ('aha', 'services', svcnetw, svcname) + if await self.jsonstor.cmpDelPathObjProp(path, 'svcinfo/online', linkiden): await self.jsonstor.setPathObjProp(path, 'svcinfo/ready', False) @@ -1016,6 +1056,7 @@ async def _setAhaSvcDown(self, name, linkiden, network=None): await link.fini() await self.fire('aha:svcdown', svcname=svcname, svcnetw=svcnetw) + await self.fire(f'aha:svcdown:{svcfull}', svcname=svcname, svcnetw=svcnetw) logger.info(f'Set [{svcfull}] offline.', extra=await self.getLogExtra(name=svcname, netw=svcnetw)) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index b6281b42e76..c32898ab894 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1105,6 +1105,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): self._reloadfuncs = {} # name -> func self.nexslock = asyncio.Lock() + self.netready = asyncio.Event() self.conf = self._initCellConf(conf) @@ -1186,6 +1187,11 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): await self._initCellSlab(readonly=readonly) + # initialize network daemons (but do not listen yet) + # to allow registration of callbacks and shared objects + await self._initCellHttp() + await self._initCellDmon() + await self.initServiceEarly() nexsroot = await self._ctorNexsRoot() @@ -1267,12 +1273,6 @@ async def fini(): # initialize network backend infrastructure await self._initAhaRegistry() - # initialize network daemons (but do not listen yet) - # to allow registration of callbacks and shared objects - # within phase 2/4. - await self._initCellHttp() - await self._initCellDmon() - # phase 2 - service storage await self.initServiceStorage() # phase 3 - nexus subsystem @@ -1562,7 +1562,11 @@ async def initNexusSubsystem(self): if self.minfree is not None: self.schedCoro(self._runFreeSpaceLoop()) - async def initServiceNetwork(self): + async def _bindDmonListen(self): + + # functionalized so Raft code can bind early... + if self.sockaddr is not None: + return # start a unix local socket daemon listener sockpath = os.path.join(self.dirn, 'sock') @@ -1570,24 +1574,25 @@ async def initServiceNetwork(self): try: await self.dmon.listen(sockurl) - except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only - raise except OSError as e: logger.error(f'Failed to listen on unix socket at: [{sockpath}][{e}]') logger.error('LOCAL UNIX SOCKET WILL BE UNAVAILABLE') except Exception: # pragma: no cover logging.exception('Unknown dmon listen error.') - raise - - self.sockaddr = None turl = self._getDmonListen() if turl is not None: logger.info(f'dmon listening: {turl}') self.sockaddr = await self.dmon.listen(turl) + async def initServiceNetwork(self): + + await self._bindDmonListen() + await self._initAhaService() + self.netready.set() + port = self.conf.get('https:port') if port is not None: await self.addHttpsPort(port) @@ -2975,9 +2980,7 @@ def fini(): async def _initCellDmon(self): - ahainfo = { - 'name': self.ahasvcname - } + ahainfo = {'name': self.ahasvcname} self.dmon = await s_daemon.Daemon.anit(ahainfo=ahainfo) self.dmon.share('*', self) diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index e6ebb5988c2..24f82a28598 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -891,30 +891,26 @@ async def test_aha_util_helpers(self): # This should teardown cleanly. async def test_aha_restart(self): - with self.withNexusReplay() as stack: - with self.getTestDir() as dirn: - ahadirn = s_common.gendir(dirn, 'aha') - svc0dirn = s_common.gendir(dirn, 'svc00') - svc1dirn = s_common.gendir(dirn, 'svc01') - async with await s_base.Base.anit() as cm: + with self.getTestDir() as dirn: - aconf = {'dns:name': 'aha.loop.vertex.link', 'aha:network': 'synapse'} + ahadirn = s_common.gendir(dirn, 'aha') + svc0dirn = s_common.gendir(dirn, 'svc00') + svc1dirn = s_common.gendir(dirn, 'svc01') - aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf) - await cm.enter_context(aha) + async with await s_base.Base.anit() as cm: - onetime = await aha.addAhaSvcProv('00.svc', provinfo=None) - sconf = {'aha:provision': onetime} - s_common.yamlsave(sconf, svc0dirn, 'cell.yaml') - svc0 = await s_cell.Cell.anit(svc0dirn, conf=sconf) - await cm.enter_context(svc0) + async with self.getTestAha(dirn=ahadirn) as aha: - onetime = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'}) - sconf = {'aha:provision': onetime} - s_common.yamlsave(sconf, svc1dirn, 'cell.yaml') - svc1 = await s_cell.Cell.anit(svc1dirn, conf=sconf) - await cm.enter_context(svc1) + async with aha.waiter(3, 'aha:svcadd', timeout=10): + + onetime = await aha.addAhaSvcProv('00.svc', provinfo=None) + conf = {'aha:provision': onetime} + svc0 = await cm.enter_context(self.getTestCell(conf=conf)) + + onetime = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'}) + conf = {'aha:provision': onetime} + svc1 = await cm.enter_context(self.getTestCell(conf=conf)) # Ensure that services have connected await asyncio.wait_for(svc1.nexsroot._mirready.wait(), timeout=6) @@ -922,39 +918,22 @@ async def test_aha_restart(self): # Get Aha services snfo = await aha.getAhaSvc('01.svc...') - svcinfo = snfo.get('svcinfo') - ready = svcinfo.get('ready') - self.true(ready) + self.true(snfo['svcinfo']['ready']) - # Fini the Aha service. - await aha.fini() - - # Restart aha - aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf) - await cm.enter_context(aha) - - # services are cleared - snfo = await aha.getAhaSvc('01.svc...') - svcinfo = snfo.get('svcinfo') - ready = svcinfo.get('ready') - online = svcinfo.get('online') - self.none(online) - self.false(ready) # Ready is cleared upon restart / setting service down. + online = snfo['svcinfo']['online'] + self.nn(online) - n = 3 - if len(stack._exit_callbacks) > 0: - n = n * 2 + # Restart aha + async with self.getTestAha(dirn=ahadirn) as aha: - waiter = aha.waiter(n, 'aha:svcadd') - self.ge(len(await waiter.wait(timeout=12)), n) + snfo = await aha._waitAhaSvcDown('01.svc...', timeout=10) + self.none(snfo['svcinfo'].get('online')) + self.false(snfo['svcinfo']['ready']) # svc01 has reconnected and the ready state has been re-registered - snfo = await aha.getAhaSvc('01.svc...') - svcinfo = snfo.get('svcinfo') - ready = svcinfo.get('ready') - online = svcinfo.get('online') - self.nn(online) - self.true(ready) + snfo = await aha._waitAhaSvcOnline('01.svc...', timeout=10) + self.nn(snfo['svcinfo']['online']) + self.true(snfo['svcinfo']['ready']) async def test_aha_service_pools(self): @@ -1050,7 +1029,7 @@ async def test_aha_service_pools(self): await ahaproxy.fini() # wait for the pool to be notified of the topology change - async with pool.waiter(1, 'svc:del', timeout=3): + async with pool.waiter(1, 'svc:del', timeout=10): msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...') self.stormHasNoWarnErr(msgs) @@ -1071,83 +1050,57 @@ async def test_aha_service_pools(self): self.stormIsInPrint('Removed AHA service pool: pool00.synapse', msgs) async def test_aha_reprovision(self): + with self.withNexusReplay() as stack: with self.getTestDir() as dirn: + aha00dirn = s_common.gendir(dirn, 'aha00') aha01dirn = s_common.gendir(dirn, 'aha01') svc0dirn = s_common.gendir(dirn, 'svc00') svc1dirn = s_common.gendir(dirn, 'svc01') + async with await s_base.Base.anit() as cm: - aconf = { - 'aha:name': 'aha', - 'aha:network': 'loop.vertex.link', - 'dns:name': 'aha.loop.vertex.link', - } - name = aconf.get('aha:name') - netw = aconf.get('aha:network') - aha = await s_aha.AhaCell.anit(aha00dirn, conf=aconf) - await cm.enter_context(aha) + aha = await cm.enter_context(self.getTestAha(dirn=aha00dirn)) - onetime = await aha.addAhaSvcProv('00.svc', provinfo=None) - sconf = {'aha:provision': onetime} - s_common.yamlsave(sconf, svc0dirn, 'cell.yaml') - svc0 = await s_cell.Cell.anit(svc0dirn, conf=sconf) - await cm.enter_context(svc0) + async with aha.waiter(2, 'aha:svcadd', timeout=6): + purl = await aha.addAhaSvcProv('00.svc') + svc0 = await s_cell.Cell.anit(svc0dirn, conf={'aha:provision': purl}) + await cm.enter_context(svc0) - onetime = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'}) - sconf = {'aha:provision': onetime} - s_common.yamlsave(sconf, svc1dirn, 'cell.yaml') - svc1 = await s_cell.Cell.anit(svc1dirn, conf=sconf) - await cm.enter_context(svc1) + async with aha.waiter(1, 'aha:svcadd', timeout=6): + purl = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'}) + svc1 = await s_cell.Cell.anit(svc1dirn, conf={'aha:provision': purl}) + await cm.enter_context(svc1) - # Ensure that services have connected await asyncio.wait_for(svc1.nexsroot._mirready.wait(), timeout=6) await svc1.sync() - # Get Aha services - snfo = await aha.getAhaSvc('01.svc.loop.vertex.link') - svcinfo = snfo.get('svcinfo') - ready = svcinfo.get('ready') - self.true(ready) - - await aha.fini() + snfo = self.nn(await aha.getAhaSvc('01.svc...')) + self.true(snfo['svcinfo']['ready']) # Now re-deploy the AHA Service and re-provision the two cells # with the same AHA configuration async with await s_base.Base.anit() as cm: - aconf = { - 'aha:name': 'aha', - 'aha:network': 'loop.vertex.link', - 'dns:name': 'aha.loop.vertex.link', - } - name = aconf.get('aha:name') - netw = aconf.get('aha:network') - aha = await s_aha.AhaCell.anit(aha01dirn, conf=aconf) - await cm.enter_context(aha) + aha = await cm.enter_context(self.getTestAha(dirn=aha01dirn)) - onetime = await aha.addAhaSvcProv('00.svc', provinfo=None) - sconf = {'aha:provision': onetime} - s_common.yamlsave(sconf, svc0dirn, 'cell.yaml') - svc0 = await s_cell.Cell.anit(svc0dirn, conf=sconf) - await cm.enter_context(svc0) + async with aha.waiter(2, 'aha:svcadd', timeout=6): + purl = await aha.addAhaSvcProv('00.svc') + svc0 = await s_cell.Cell.anit(svc0dirn, conf={'aha:provision': purl}) + await cm.enter_context(svc0) - onetime = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'}) - sconf = {'aha:provision': onetime} - s_common.yamlsave(sconf, svc1dirn, 'cell.yaml') - svc1 = await s_cell.Cell.anit(svc1dirn, conf=sconf) - await cm.enter_context(svc1) + async with aha.waiter(1, 'aha:svcadd', timeout=6): + purl = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'}) + svc1 = await s_cell.Cell.anit(svc1dirn, conf={'aha:provision': purl}) + await cm.enter_context(svc1) - # Ensure that services have connected await asyncio.wait_for(svc1.nexsroot._mirready.wait(), timeout=6) await svc1.sync() # Get Aha services - snfo = await aha.getAhaSvc('01.svc.loop.vertex.link') - svcinfo = snfo.get('svcinfo') - ready = svcinfo.get('ready') - self.true(ready) + snfo = self.nn(await aha.getAhaSvc('01.svc...')) + self.true(snfo['svcinfo']['ready']) async def test_aha_provision_longname(self): # Run a long network name and try provisioning with values that would exceed CSR diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index f1cf8019fc4..d346d59a482 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -1386,7 +1386,7 @@ async def getTestDmon(self): s_certdir.delCertPath(certpath) @contextlib.asynccontextmanager - async def getTestCell(self, ctor, conf=None, dirn=None): + async def getTestCell(self, ctor=s_cell.Cell, conf=None, dirn=None): ''' Get a test Cell. ''' @@ -2019,6 +2019,7 @@ def nn(self, x, msg=None): Assert X is not None ''' self.assertIsNotNone(x, msg=msg) + return x def none(self, x, msg=None): ''' From 08d3534a9633eb3074408f9a50091df84504f40f Mon Sep 17 00:00:00 2001 From: visi Date: Sun, 7 Jul 2024 15:08:40 -0400 Subject: [PATCH 25/86] wip --- synapse/lib/aha.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 9f9c03f2454..d3ce45e1d39 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -792,7 +792,6 @@ async def _clearInactiveSessions(self): async def _waitAhaSvcOnline(self, name, timeout=None): name = self._getAhaName(name) - print(f'WAITING FOR ONLINE {name}') while True: @@ -810,7 +809,6 @@ async def _waitAhaSvcOnline(self, name, timeout=None): async def _waitAhaSvcDown(self, name, timeout=None): name = self._getAhaName(name) - print(f'WAITING FOR DOWN {name}') while True: From 97509359c1ba3a98564bb7d851d99e00e00a5a28 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 8 Jul 2024 05:22:38 -0400 Subject: [PATCH 26/86] wip --- synapse/tests/test_lib_base.py | 5 +++++ synapse/tests/test_lib_nexus.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py index 742438d8f0b..90563f7ee3d 100644 --- a/synapse/tests/test_lib_base.py +++ b/synapse/tests/test_lib_base.py @@ -201,6 +201,7 @@ async def woot(mesg): self.eq(data['count'], 1) async def test_base_waiter(self): + base0 = await s_base.Base.anit() wait0 = base0.waiter(3, 'foo:bar') @@ -224,6 +225,10 @@ async def test_base_waiter(self): evts = await wait2.wait(1) self.len(2, evts) + with self.raises(s_exc.TimeOut): + async with base0.waiter(1, 'newp', timeout=0.01): + await asyncio.sleep(1) + async def test_baseref(self): bref = await s_base.BaseRef.anit() diff --git a/synapse/tests/test_lib_nexus.py b/synapse/tests/test_lib_nexus.py index e4b1521b155..8270e1a61c6 100644 --- a/synapse/tests/test_lib_nexus.py +++ b/synapse/tests/test_lib_nexus.py @@ -113,10 +113,10 @@ async def test_nexus(self): async def test_nexus_modroot(self): - async with self.getTestCell(s_cell.Cell) as cell: + async with self.getTestCell() as cell: await cell.sync() async with cell.nexslock: - cell.modNexsRoot(cell._ctorNexsRoot) + await cell.modNexsRoot(cell._ctorNexsRoot) await cell.sync() async def test_nexus_mixin(self): From 2ee4285d1994c04f07f9a797628e4ec15e11d369 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 8 Jul 2024 07:57:49 -0400 Subject: [PATCH 27/86] duh --- synapse/tests/test_lib_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py index 90563f7ee3d..3bd9b1be99b 100644 --- a/synapse/tests/test_lib_base.py +++ b/synapse/tests/test_lib_base.py @@ -227,7 +227,7 @@ async def test_base_waiter(self): with self.raises(s_exc.TimeOut): async with base0.waiter(1, 'newp', timeout=0.01): - await asyncio.sleep(1) + pass async def test_baseref(self): From 1e715c94adad73c580c9da1eaf309b2449ec39e6 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 8 Jul 2024 08:46:58 -0400 Subject: [PATCH 28/86] wip --- docs/synapse/deploymentguide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index c4bc8b8b54f..26b16a9ae12 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -455,7 +455,7 @@ Once they are enrolled, they will have a user certificate located in ``~/.syn/ce configuration located in ``~/.syn/telepath.yaml`` will be updated to reflect the use of the AHA server. From there the user should be able to use standard Synapse CLI tools using the ``aha://`` URL such as:: - python -m synapse.tools.storm aha://visi@cortex. + python -m synapse.tools.storm aha://visi@cortex... .. _deployment-guide-storm-pool: From b01d3bfb201a7a9f498993ccd1f16f7b66aa375b Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 8 Jul 2024 10:36:13 -0400 Subject: [PATCH 29/86] speed up tests --- docker/rmlist.txt | 1 + synapse/tests/files/aha/certs/cas/synapse.crt | 28 ++++++++++ synapse/tests/files/aha/certs/cas/synapse.key | 51 +++++++++++++++++++ .../certs/hosts/00.aha.loop.vertex.link.crt | 30 +++++++++++ .../certs/hosts/00.aha.loop.vertex.link.key | 51 +++++++++++++++++++ .../files/aha/certs/users/root@synapse.crt | 29 +++++++++++ .../files/aha/certs/users/root@synapse.key | 51 +++++++++++++++++++ synapse/tests/utils.py | 6 +++ 8 files changed, 247 insertions(+) create mode 100644 synapse/tests/files/aha/certs/cas/synapse.crt create mode 100644 synapse/tests/files/aha/certs/cas/synapse.key create mode 100644 synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt create mode 100644 synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key create mode 100644 synapse/tests/files/aha/certs/users/root@synapse.crt create mode 100644 synapse/tests/files/aha/certs/users/root@synapse.key diff --git a/docker/rmlist.txt b/docker/rmlist.txt index 88d30b7a22b..382394960ab 100644 --- a/docker/rmlist.txt +++ b/docker/rmlist.txt @@ -1 +1,2 @@ /usr/local/lib/python3.11/dist-packages/synapse/tests/files/certdir/ +/usr/local/lib/python3.11/dist-packages/synapse/tests/files/aha/certs/ diff --git a/synapse/tests/files/aha/certs/cas/synapse.crt b/synapse/tests/files/aha/certs/cas/synapse.crt new file mode 100644 index 00000000000..ee927854c7f --- /dev/null +++ b/synapse/tests/files/aha/certs/cas/synapse.crt @@ -0,0 +1,28 @@ +-----BEGIN CERTIFICATE----- +MIIEvzCCAqegAwIBAgIRAIdu+9lQ3Z2nydXmG1/p2BAwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEAwwHc3luYXBzZTAeFw0yNDA3MDgxMzAxNDBaFw0zNDA3MDYxMzAx +NDBaMBIxEDAOBgNVBAMMB3N5bmFwc2UwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDQ8AFyEbiRUsKdRFlm6WS5X2cp9339H3A0qsXLSRYsCZGc6oJtLX0/ +52xPGCrKlXde2jxHU8DMDiEzQ3iVJjzDbJWXgAoNVfwrNVS1zjlSZyExVfRrCbzo +XpGeuNAIY/WCpsbSxM7mCVPuEZtbX2rY4XmdyBWUpd4dqoDDIK2wU329daJEVNpd +y9wdbZ2lLNPmKGS6Kb7Th3mio+sHpHEi6gcfYzH+mOFdWFVAh602OFMUp+fLq3nG +MyFHHLDC2vvPqVWf+T7Z1E8k85Zqw43TSUTwdlG6u8egC6soKQsEze6oi4fMAt0l +a4bgMMWDMIKJuQ4f6THEq5vKwPj0E78aT6C4C7qSKeHZSf8IOj7l3ukdgl7Sbrga +JCjpL1V5+EB8LTpxjjK+TDbXUwU/eKIJp/I2i7dd//nMRs9Wr2k6aIvAt9PbopA2 +kG4tCS93TmzCza3TCtc4BruFHz92ToLfxvgcNq6N/BboDVjDIeAKtrWpe9VHPEIy +Eh23tAPcgPXmlOBZj4DKJ5YoGNbwYQaFDrLIHh1JzRGQbAh00JuaqDPH3kb1Q6sm +VjUx4QpX34qj231Bpfgu2eZX2kiLvtW+qPPfxWY2RCms1Yb+GMeKAE+9E8rak8Nq +oPvdqgdUUbneW1PRgBaalu7xDALH/DQm5R+baly5PEYi8gk/vsyaQwIDAQABoxAw +DjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQB4Asv4TuAgF2vcOdxl +mjPBwYueBDJrbbP2I6qjE9laO0xMyIN4Dp8ExbJyBKl/RlkSA/1cvJ7EyhlJnY1S +Dps8EgKS5lQw0D/bFg8JQNgNec6jCxjfytRgYkhoikabz8HR2gmUIb0On6X+dvht +L3Hv1/41Q4+Tig1JPRxP6C/hHcfLnCvmIsrMgKRbTGzb1AKfP8N93eZqDcA0nZo5 +Q0p+AdLQ6ilnh8895wQexoKs7mWImEzxUdTMZlf/YSQ9Nct3OKxtYg0IOk1KW9Fe +sMstTDHg+/NJr2297vCmeNydYRGOTidLefebaNq0YnTm9cI3kPi/9etfUGeWHbrW +aXXJCQ02uiWjrL9ykiifB6f0nve0m4ENWI11FypZ3nReMNDFIYSQfwlat6E/KTQ/ +5ih/NEgbY5KnMcUJqEKCBftfvfZyueE0frHClLZNi+YQNXD3ui6lmXzn8vmvaHeR +dDfCifQ5TGHOlNmk9LMOv1Szx9CNswUwKDpSLklFp2lzI0JHTXm+NYCqeE+3cnrN +vbsq55txmImj/Ekos5/kuv3Z2myJsDd2xT2eZ/QEkb7FOeO5RPrya3yXoSGNlkgs +XG8vvCDNTeD6BJPWHjbu0ksZ6nxBFRsEmA2Ni8Y9ROUXUuJtwX+sYqHfAAVMLbiR +wSujqDSopMoJSGr0hUT9kjbUSg== +-----END CERTIFICATE----- diff --git a/synapse/tests/files/aha/certs/cas/synapse.key b/synapse/tests/files/aha/certs/cas/synapse.key new file mode 100644 index 00000000000..c734ec8f972 --- /dev/null +++ b/synapse/tests/files/aha/certs/cas/synapse.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA0PABchG4kVLCnURZZulkuV9nKfd9/R9wNKrFy0kWLAmRnOqC +bS19P+dsTxgqypV3Xto8R1PAzA4hM0N4lSY8w2yVl4AKDVX8KzVUtc45UmchMVX0 +awm86F6RnrjQCGP1gqbG0sTO5glT7hGbW19q2OF5ncgVlKXeHaqAwyCtsFN9vXWi +RFTaXcvcHW2dpSzT5ihkuim+04d5oqPrB6RxIuoHH2Mx/pjhXVhVQIetNjhTFKfn +y6t5xjMhRxywwtr7z6lVn/k+2dRPJPOWasON00lE8HZRurvHoAurKCkLBM3uqIuH +zALdJWuG4DDFgzCCibkOH+kxxKubysD49BO/Gk+guAu6kinh2Un/CDo+5d7pHYJe +0m64GiQo6S9VefhAfC06cY4yvkw211MFP3iiCafyNou3Xf/5zEbPVq9pOmiLwLfT +26KQNpBuLQkvd05sws2t0wrXOAa7hR8/dk6C38b4HDaujfwW6A1YwyHgCra1qXvV +RzxCMhIdt7QD3ID15pTgWY+AyieWKBjW8GEGhQ6yyB4dSc0RkGwIdNCbmqgzx95G +9UOrJlY1MeEKV9+Ko9t9QaX4LtnmV9pIi77Vvqjz38VmNkQprNWG/hjHigBPvRPK +2pPDaqD73aoHVFG53ltT0YAWmpbu8QwCx/w0JuUfm2pcuTxGIvIJP77MmkMCAwEA +AQKCAgAxPqgwkQGt6tIoy//AVDkjwdsoVodQ3hSNrURiNe8uYPD7iYBFKEgRhEOQ +XtNTHShd6FT1wMU7swbbNMdabAE9VD3rz8dOvlnpey/ki98Rz3HQ1X/+rHRkVkm/ +HbMWjyzB5voMmktjh4ZLIcY6ooIl7PrDl/GSMAfqeRHRK8YUPZFw0qV0soUnP1G5 +c+kIkci9wf5/rDAoXhFqpnTSP81Um9Ei8jfJ2JGhdRze0TufgGYAg9SLufZBIzLw +NlBpFMDuAGzIgC/ymmou/OSSdFXcmzPO8ywvNWwHCkkEdav9rWXXPs+6Y2BpHe9T +rtsWoRvbRw0Ps2BCCOp2vsXOjUrohP4I2iKb5ExHAQoKmewro2b/txJw39rnaWa5 +/E9OG5hvbXEt0o5dfeLgtQLXcTsjTGqbOO3bi4dS42Zz9DdnnHf9oNes5odxW2lp +TCPJQozZuzY9J5toEltz2RBgX4ECWaXE1UnY6E4nm1zkDRc1xx7V1bXp6MsXWYx6 +MO62tprWb+zFz7M6yIz7fsB9339X5Ky+xtDPKBJk8qnOVFpMbHkefpiHs6tPlmyu +1wIjeL+3Om2pAd1nEPuk02pOhiurthiqI6wwHI8X03RnF5HU9tKUKLMCjnD1PJuM +Ysqj7pDdaezRic+mrQAyBqebLzEcZ+PRwOpPVCX3lo2G/BD1bQKCAQEA8u70QHiX +l/OlcxTG4nVn62ljsWCY3FYTyfDtWrP3uZQ7zaLF+wlGX7MXo416zVHPvW4YtCkG +1n6KdaQguuSbE+CgvzyCmCLtSYiXuA35dVa9a6TDG5nPvwPJeT99HPMg1xRXxCaC +f8ilJ6WILD9sG6383a0fWC7ZlXpIFB+ZXmNaj82vS421BrbciH+sLO9jk94Drxnw +aXe81G6Rkabw3w852y+0TgqOoiVis4rCBpCL1e7Oz+s6+kmpNvc/4NI9LZTk2B/l +M++9aTi11p4Z/1paDffOPinmgKXOhb1fud6+j7XbNmvU+mgDc+95H93zA2rp37CT +CU9jo5i/hNUDFQKCAQEA3Czy3iKpvDKtsq9cDStfg1Ysr1raqOKHr4cS01M3u5Xa +Ku3ORGkiVcQJAfgWik5nFfbzbiX2SNUFB9jLQvIGnAxRTen6empfCCds9+HNS4PI +E8Byyepr9H6LFpqLr/GDrTA4cXWez8r4BMZwsuYa/cOEZt9Uoq97DFlCEwL4YDD2 +ZDNLSLry7LhH4XN0oYU+7imj4fKebWDY2m/qc+OJ0+xMDg/dioFfWaIW5UnpteNY +6YZFTOygosSRrvEKGKnaszVpDRLPcHKcEzIKX8Naf03898R1m+dM1Atc/UZa9pet +6kpZpqpKoGHrXg7rNWWdPyJfm4bH1MoNqjABUIhd9wKCAQAV6xtcicTbr974oCJF +omQq6EpXYajJEHcenD8+FMjAFLDEn/AO80pHLihu2EABMGV26O0PrDfyuF4TuSg+ +1IttYrH+Lx51TYltPga6U4BzZs0WXjpATkNhL51I9EJ8jy8iWLKGfxb9IoRMLHI5 +08sUQEF1Wr5ePXPiObMxJZy32Gz+Vod/YJy5q1wAcMx/DWZFnB1m+gcn7Oa7n/JA +WviWl5AXx5kUBX3TAV6DZnyVDQug1LgSKF4c4PKEhBBeX3mnmCyBl3cdlX7YdIZr +g75CvMstQXN5RlyGtO8KQAjYA1HcM4NAyL/hi+rr1epuxp67azUIuqy5hVEvHIQD +Hxj1AoIBAQC8knrIKiP5neYKvgo29UjusaW/4i6YqrvPZ/6FpCZ9sRCT5+zbxrez +gRy95P9ZIWFE/KbtVfIj2t5eJB2ijqt+h0YzVwxCQEx4LVw0yd4MqSd5U0B9Exu2 +4ZK6n064OD+w2zXcZwLHsWzOmi736gCACy6g9PIGDAl1QBVJNygHKqg8lXoLJqLc +f9CAlWP02qxVSrCj2io6P9I689N3wg/Pw/g3qvrxn3BM0niNlMpoD/mcuHUuNxQ1 +k+m6TZN6IC/BgSMiIVQtWNu3zQn5jtU5Z1Ab3NVl26p/iePwwIsz3CEGIvu5tOwJ +hRQTEO/+YbNV2VjNWZhY9VzSwB7AHKttAoIBADaDQIZC+2JcOAZnLCwfToYb0Ga4 +Hs0NFcO8/EH7PNTTlhbD5bVPW/6IlTkSDemMp89zdYoB3EAfevWK5pMp5XOpQfsA +Puc2gueO/ICSb76YcabAdnOyB73Kb2PvqQ7yl6YP9U9IK3PBuZbDxiOcsM1h1yfL +XlMjDH08/e5efDBaxeVDTXS63haFXTgRKHYdnEm63AzeR0PXvpD/az++aaxoxkAV +SV/zudxSzx/O8zrVjl6lz1vb35jQ1s4X2T8EfGEjuXAGx7yV9MaEwLWH/RhHAUgD +vHoSEcB5Q3hbSj0QBqHiSkKcGn5VYP3kXSGMF2438flUahpdMpeP4QTa3tA= +-----END RSA PRIVATE KEY----- diff --git a/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt b/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt new file mode 100644 index 00000000000..b1fc57b2310 --- /dev/null +++ b/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt @@ -0,0 +1,30 @@ +-----BEGIN CERTIFICATE----- +MIIFJTCCAw2gAwIBAgIRANebBnGvTlRt7CIGrH6WJagwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEAwwHc3luYXBzZTAeFw0yNDA3MDgxMzAxNDFaFw0zNDA3MDYxMzAx +NDFaMCIxIDAeBgNVBAMMFzAwLmFoYS5sb29wLnZlcnRleC5saW5rMIICIjANBgkq +hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvXp4PAfmdzu5t6ADh/l3iVTRYbpPVnQz +7bySmQlhTtNQFRDOHgEyfttpd0zTWoQZBiQr9H4ZjXE5W3XHdptHcPE718rzBcP3 +G+DDIbp5nbkccUb7lvMVt39/4n9lK64K5YdnRop4SGsnps2WJvrLv/7qhgXPedjF +Yh/E7iR54C+wns4aKvD2gZV9vYudu6PcB3S/b/r0Vqykpsy6dFL1bKTvJCPyD4b3 +ZAXpHr8Cws9Chae8DPt3PkUFCMnRl4opG7i8w/aJABRiXS9whccP1H0uGMvRF69P +E1eSjXytwFcBUOI3rTwnAWBOfQ1zwRAQ6nCQsjH/CKZffrKro6M7kT+rfE0EiDPG +kqf0owa+av4Rv2D0qRXhOBsq/jWgCtznk4286JweaRW8Omk+ViG5JO0Pxczp0bpg +qiExqpJ/O8HTAEKbI9MB2+qRcUVnJanJ2KXmPUWdTtJQuEtCoLfDExudzOJgMwb8 +i2cmXTeVwas2QKA5pD+OOlGKciQc0T2xPIE/Gm/3CWRBs1wj9fqJ85NG9zbtCPpT +7yuruYyiuRJTAo9s8D4AGaoeQLhQ8JE4Xkfol9Pva/UTIaHoKux6aHYsXBUbEeZb +RqO3BGViV86Po+1+t2lBHHfD/s/YTnT7cjm+dYQdS9GqrwzZr7lp538lFkmFeX7t +wTZakkXWIGMCAwEAAaNmMGQwEQYJYIZIAYb4QgEBBAQDAgZAMAsGA1UdDwQEAwIF +oDATBgNVHSUEDDAKBggrBgEFBQcDATAJBgNVHRMEAjAAMCIGA1UdEQQbMBmCFzAw +LmFoYS5sb29wLnZlcnRleC5saW5rMA0GCSqGSIb3DQEBCwUAA4ICAQDFSKMqM/A9 +maLx5zMO917Ysw1odf1DsemYeDjoZPRMSC/PVIjJnaZqB7UH3/mUxX0uqk9NMjLD +aq0u8vJqNnjxBzFWfcb5vcH05WC1lBIJj9Bd/tR+0jXL31COqZ5usljv/uL74hII +MH9qiRqDE7eQq7Mcxz6qbXwGGqY5570iOBfvLConR8zCy0WEuDf3yB8/8XKaDsxA +6TiaY35OZgR6yNA7q+1rpBUGCsX14mJMOVWtIaY5V2F4RHc+7HzdkW0o8XDHk4lS +Tq/SAfjFsGt1bDLgmJ3HtI+lUBZtW+uKLZVHNnV4v7rAebu4TLUtbZXl7eeUBWpF +mSP2rIs0RbR7Y3jAXg4WMu4fIeZkP/jUvPpgUfkiF5jevwc0GvG7h8u+6ybg9nrS +09vlTXc08CRgtp2eYTnsSR7aJCr0kYo7ZTWbeo96gf/vJIVJcNnRZnvvfl1vyrZT +QK13S6TzUbF1oAVHMH5nSun200D2CSzyjOYDnGpTGIdfbamI2KsphyUS/sVCETDP +f8fC01rRBpjFc7w+ubYuRyiXEJpzbRJe2CWQZPxq0CZ7Yifgz7hIot5/F1EuNE5z +elh6vtjcudEoQjOgUI4HABQe9nI6TUDfAKPlo/ffB4iHMBmC49WaCdu/6gQs0s7j +KjhaMLwW7zaGUYFkdR1vcn3qeFnAM9cmBQ== +-----END CERTIFICATE----- diff --git a/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key b/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key new file mode 100644 index 00000000000..6db7bae2262 --- /dev/null +++ b/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEAvXp4PAfmdzu5t6ADh/l3iVTRYbpPVnQz7bySmQlhTtNQFRDO +HgEyfttpd0zTWoQZBiQr9H4ZjXE5W3XHdptHcPE718rzBcP3G+DDIbp5nbkccUb7 +lvMVt39/4n9lK64K5YdnRop4SGsnps2WJvrLv/7qhgXPedjFYh/E7iR54C+wns4a +KvD2gZV9vYudu6PcB3S/b/r0Vqykpsy6dFL1bKTvJCPyD4b3ZAXpHr8Cws9Chae8 +DPt3PkUFCMnRl4opG7i8w/aJABRiXS9whccP1H0uGMvRF69PE1eSjXytwFcBUOI3 +rTwnAWBOfQ1zwRAQ6nCQsjH/CKZffrKro6M7kT+rfE0EiDPGkqf0owa+av4Rv2D0 +qRXhOBsq/jWgCtznk4286JweaRW8Omk+ViG5JO0Pxczp0bpgqiExqpJ/O8HTAEKb +I9MB2+qRcUVnJanJ2KXmPUWdTtJQuEtCoLfDExudzOJgMwb8i2cmXTeVwas2QKA5 +pD+OOlGKciQc0T2xPIE/Gm/3CWRBs1wj9fqJ85NG9zbtCPpT7yuruYyiuRJTAo9s +8D4AGaoeQLhQ8JE4Xkfol9Pva/UTIaHoKux6aHYsXBUbEeZbRqO3BGViV86Po+1+ +t2lBHHfD/s/YTnT7cjm+dYQdS9GqrwzZr7lp538lFkmFeX7twTZakkXWIGMCAwEA +AQKCAgAgRonPk/ruiYZvoHqpgVWa149ZCc7055Nm5i3EksP4FOe5xuSNWN/cmwxi +jXwdGY5XrPatzYMVxFkkWrIw4m9vbjAm6IOwEjr4DTe/+Y84zizpoNE/W8XxvW6v +ysqVf66MfZ2adwDZOSOGdtOibSsi183kKX43f7TTq5y0ghMenJEF5A6yDNy4oxnJ +nUwvh9B1lq37abCQSRU88ne6U91Jdejka5kSiwd+CsG0go36WCq5MKLIRVeBDGm4 +nwQsP5UUC0pgSRD6Kf69Z9TPfOKV7ALbp3BFDBA4t7yXjErejhODzxzzzeDJC7oi +9BUpKE7xWF5VdE3Aj/KJVu8Ez0vYfTTr+IvhAOi6mOxmqYhcODzwehwiD5o0CiG8 +Igb3YzZjvgjjIhX0n5GsKSOWlxjXOMiuB221WvLIx/qlCXCJzR71XoX4YwPYn7GP +LjjTVrMiLqN8UNEZ1AYH1sP9xlL/Dzg164EMBsRUIgQJo7DHXtpcEEpioyDRfLmm +LsoT3MaNsd9EoNDKlTpNz1BrJtJR0GAQCvBOKOCIpXihiLRWj3d9gFe4ZO4JdNOd +B+m1eZzXltus3Rc/7tGGKAfNt212IGKOalZsIpYN40C7C/iFr2miST6SUyBLhVaj +3zbexSIbgj7QfT6IIMYZYrZBb2FfBALJa58ye/enNRGlcnFtwQKCAQEA8h7COxN8 +uotwC4T2PIFzjusGGCeAKx6PuRKfl4P8nzBic4BjvbKVrLc2W4hqqAu15uk5RkAk +sdjd2IFpQzPsdbUTmKWNKYDclpMctqTifdxyjOSxIx9Hg22UkZKbVKyrjHKvlKkx +o2E+p6XijAc8LvFLty7pSAe3uHv61bzEmEc+rJEoH+k5sQ1TQP43rbOLjvWNz4oS +gFv1/437+uFaMglsgfP14fBYwaTQZc3mPIEbeO3e2D95kZ6+ptOnUagS+LDNNcCk +F0YrFePPDP2GKZPLjfqoN4EGASSdXogcMDdQfVM/BmiZFYWAmSPKzwzm874mSRZA +fXyNxH+0FsMzowKCAQEAyFcqVXTbKTRzTgssjnRIrfgwp7bDT9xoMG4PWPqrNKax +f/Kg8CU/iJErkcriwcbheWKxKy6d8MgDcMZuDEM4gsfQxIeTLUPeiNvVCdtk8x0/ +twaX7R7KBRWR5Q4/SGZqoI5ZVZTNROpn1YwFlhJJXdJv0l0LxF2NGYSLXIqG99g2 +2kOlVBeAm4evoY6xHsp9dxVa41Bc3+KK/vrZezF9bQTq2NmtHyIUJ2jIMlgr+eAc +Bq+yJLLHQ3c1NR8jKNK76E59dUkawsxPnSEc0KWh3g3L2udxy/+1eE3r+jj0jCII +2MoiTFVCNnJ7LMdsGISkax5ZJX1blUflikIA72IsQQKCAQEAwLQsgRp8fni2f+Se +mv+pOsniOt1NjIQxferNrKk3Kng3E5jPSc9Wg3X6xJVp1kAj0ho0JK6uxgJGZ6hw +YDV2cSTi6O5y0OKoLwv9oXzQa75GSc9HER43K+rOgaJ/EMCxdQJeruKPCGtAk+xa +yHqFsxMH4U9sCpFh72p19SHeExk5T93kYqmc6kchySvMouqxG+JisRlCqnkG7RRT +xpUP1Z1ciH3kaKSD7/O+jhh3tBZKCFDCubijiHwhX+Q7Wql8GAWX/r1JnOCTMEP1 +qnAqFPN14pXqxuphHg3HVtLcJKAR5v2XvwEHPnLYLIqpQ2wQcVUZYbhdMcMtjoTZ +j/hjIwKCAQEApmGdyvMNwJ7K1Bn7myN/6NuirObgNkb6UJ5XKLKl1UhLSdObTVXh ++e12ndI9mGkvgLwyH4bLrNiv4s0pQA3jtNl1zII7/O/MtSS9PT50DGRSMhLLwiY7 +6RUM4Yp/jAVisI0ILEc0YvO54GQ1j3kIbV8Dd1XHHAIF2Rd3FhgGF3f9ti9P8xLB +wGljt2zmNIg+wtN9dCOdvmJKxZBXZjSn0g6vbAD8AksvKbuf6A/KFe/F1te7vzaq +vqEWE1QUwyag4EGvd+SK0RUVWY3SfIXSdLRIhTiKDb4EXDF6tYjvsCHj7weQjIyS +PN2+5mWIpKQkWMIPj08Y7FWVkMlYNXb3AQKCAQEAuXMzr9YD8r3kaF7vQmRiF3f7 +725mWjjBwcKrUW/ZgLMkfx2Pcn9Lt+1+ULVcEGwIuI8eoS/9LhI4WR6j6H8kkvOH +TQ//5ZFiOJS2MaS9kSEVs5RVu0MhaWshrCiYOIcq3z1v4AiSxh/jNGc3Uc3eEiMy +sxPA3YLKPahPTt3QEtwRagCsgFfBvj52RJg8TON3ZQoOACeUSRJlCgV7mxh+gRor +1D8eevuSd+7IRlWnM7UCQQTtNie5mbOgd3fRuNI+vGFIwQnHmkg55iFE5TCRg4oS +UeOhOVF+5BZp7qOn9j3FVWzkaGwfIredFWnfh9oENIW5orIGB/SbwEfWm8UtUw== +-----END RSA PRIVATE KEY----- diff --git a/synapse/tests/files/aha/certs/users/root@synapse.crt b/synapse/tests/files/aha/certs/users/root@synapse.crt new file mode 100644 index 00000000000..7b73a8def75 --- /dev/null +++ b/synapse/tests/files/aha/certs/users/root@synapse.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE9jCCAt6gAwIBAgIRAJGsran8ISfoaXuIMN7vAIAwDQYJKoZIhvcNAQELBQAw +EjEQMA4GA1UEAwwHc3luYXBzZTAeFw0yNDA3MDgxMzAxNDFaFw0zNDA3MDYxMzAx +NDFaMBcxFTATBgNVBAMMDHJvb3RAc3luYXBzZTCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAL42zY6E1q0RJIr9j+T6bPb7/f0RbwLJ2hza0/4qjAisNGQs +t1WkWvpNR/l0+JHmXw78KQ1k8/WP+niIXZmakZdrMNQOf+BBY+MoNqFHtA1KoP0p +b7f0hyQyRprYihSllA3uFZEagaylx9z5TliwclnCvB77whRZFU8f5PXH+wibWFvJ +n4ycdFVLg93Rb6IiliMytc3CA4TEemDiL5410nZNipWGcYRmTUruWZCADYLSh6D+ +VJvY/sfoteBX1cvhCdL6DORKaxZnXsPthzkDPJYgLn+1E9hJ4vhuqgXIJ7VjqrPZ +0wJtciZYb9YcJM6OqiWavW8AeLOEySEKO4NEGU0JduFpu4/9xnfhv/Atdv9YwK6j +mJmxl/bV9dPhjszNG/D+RbCeBdhWebL1zcnt8indigLxSI9UgulBnjCJg4xHrJPO +9tiXPi4ipFL4tIpiW4NYCWnJXAkCNgPBFGeFaEmzAxoiSh/iBIMfUgfMV26HzOUS +2MAzq04/2nH4MjsZYtIKNx8sNZvC3QnDVH1ncGC8JcbSVZPH2jBmMRJJjhxDT7wM +JbgJJomJ1mIP3a03FJpLTCxP8IbBTmq3TuOpqmhmiVHjBunACbNf6l6fhOJG3pMk ++VpWtIUiKyNMXvjxgM5bKi6sx1ivInxmPk0f2BPBTrYeLFCTj711A4XuHxSzAgMB +AAGjQjBAMBEGCWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMCB4AwEwYDVR0lBAww +CgYIKwYBBQUHAwIwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAgEANZtBnHeo +yatrA/s2BzOU0k4WGEUaFMrCi6cDyVAWQ6mhcQHqJ2C8Yi2xCd5Si2BEC0hiyJ4K +wOpi/bkoyAh67bX/RchzqLTwfUYyM17ZH5yiqbPufjpJQAitAVaywCI0jXDpaYlM +o2u6b3u8eUKhlfHSIzTJn1sVc0p0Ql0okhrHwwUfwN1lwkVDR/e+jF5GKAwjnkNc +yu3yYYz0oHAg6ZOcQaaFASX7oBBpYnHfDvou3uoJTGfKevRgI5NwQDRG3gipudSi +vIwWR2iyBjTZaICChmaBX7zVHLwOFN0LRv73tgEBcJwZ4oCvE64POpAL2pO+U52d +fIFlIOPSuILtTTtBRvfTuIjEvVCF44g4JkPgPpaN9yPcdAOnBQkP7RkrMlQ1pKZ3 +S6uZh9Ar0jNx59ijYq2Y8o/yyEVveXIKbp3HLogfcLeEWUqjL3syP72WkfBWRWx9 +QELaYrFFxgtT4bKZSmyK4C4qwgSknm89Xp/JZFyKWAjYEmu9BAHDI29IBVbLzQAS +ezrtG2CmyhejnGhwSMYNmv+dC+ahSeP5FQiixrrxaTJ0s3Ax5mvbQQqv9lBvkn9Z +WpVNN5MHdHEWnvbK4kfFnVQIBfv3lvPl0C0vk3VSWZexXIOjNtkUK6/ALHV2n5AA +BEJmC3OPyT3eNrlr7gCxuMoV6uDMeTCRpTY= +-----END CERTIFICATE----- diff --git a/synapse/tests/files/aha/certs/users/root@synapse.key b/synapse/tests/files/aha/certs/users/root@synapse.key new file mode 100644 index 00000000000..f426d5dac89 --- /dev/null +++ b/synapse/tests/files/aha/certs/users/root@synapse.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKgIBAAKCAgEAvjbNjoTWrREkiv2P5Pps9vv9/RFvAsnaHNrT/iqMCKw0ZCy3 +VaRa+k1H+XT4keZfDvwpDWTz9Y/6eIhdmZqRl2sw1A5/4EFj4yg2oUe0DUqg/Slv +t/SHJDJGmtiKFKWUDe4VkRqBrKXH3PlOWLByWcK8HvvCFFkVTx/k9cf7CJtYW8mf +jJx0VUuD3dFvoiKWIzK1zcIDhMR6YOIvnjXSdk2KlYZxhGZNSu5ZkIANgtKHoP5U +m9j+x+i14FfVy+EJ0voM5EprFmdew+2HOQM8liAuf7UT2Eni+G6qBcgntWOqs9nT +Am1yJlhv1hwkzo6qJZq9bwB4s4TJIQo7g0QZTQl24Wm7j/3Gd+G/8C12/1jArqOY +mbGX9tX10+GOzM0b8P5FsJ4F2FZ5svXNye3yKd2KAvFIj1SC6UGeMImDjEesk872 +2Jc+LiKkUvi0imJbg1gJaclcCQI2A8EUZ4VoSbMDGiJKH+IEgx9SB8xXbofM5RLY +wDOrTj/acfgyOxli0go3Hyw1m8LdCcNUfWdwYLwlxtJVk8faMGYxEkmOHENPvAwl +uAkmiYnWYg/drTcUmktMLE/whsFOardO46mqaGaJUeMG6cAJs1/qXp+E4kbekyT5 +Wla0hSIrI0xe+PGAzlsqLqzHWK8ifGY+TR/YE8FOth4sUJOPvXUDhe4fFLMCAwEA +AQKCAgAO2f6QuyR73NPKmmOq0DbDzgcVxE+zmwkRqbBVrwLCBpgMnkUuRafo1THi +c1SZJ7CSXfPinNhDextmX9dXD++AMBle5Ubsvo5LBS/Gxe9z/ji1q4+SgGAw3lAO +9QtK82as88LxUm1/im4hfcG1QQmcoozHqoaLyizUwbvByPx7vo1WlVaExw56Pmws +XJbnxOWOF/6D8zsNGb4oZGCtbKxjGkjQxe+FE+vDBvstQiJL7Z4zXshCZt31w+C3 +hIQn6Ak5VNa+2GSmGioXCCu/stX/LsISrJPBFnvHQ/S4gNlA22hPpJlKHjuEbPFG +eWoSSqdUSlGyIgpKkfqWPtNIu39yFtj2GKQPotwnGA9CVk4HjjEpBEYBSz8cGGvc +jw9/YjUUEpna8Ab+RJcUJ0Oc+RdMZvnmEtGs2w2dHHXFWU/0nMe7oBeSR1DLYRW3 +X++je8IqLDvHVZ6+LgcdQOcMM5Myw9rPsu89+YOf76wtOwb8T014XmoLyNC0sulN +WbEdx49yQSkSX0lFq9fvSmfaptl+2PXIAqoRmK5Vkg2f/OQBmFX9GM6mLLXR/eMH +kurXGrnZ/RRqb3TIO1Gy+v7vTCSL0nZC4krQj1AWlUA3lhTEIo9qJtlL2D+e1pkR +Ujhcz0U73SAFbkVOMzv/loCsSIRHdfhh4vRNEGZRW6lx6xdxiQKCAQEA7KMqiDJt ++xn0hQjWOBL8nnbNiXKKNU+ENwedUXnmZnkpaWbXEh5+gbmUsSNwTvVCncQbjh/a +sNwvVi2yalSEaQIji7PUmOJV4Pw/UXhEaGYhsQqs/snnKpnfzqdHXcZhhqmVXALm +eVJPYjvmt49obd91lC+sCc/iH+wkn1Anj8UvAlU8bXoUNEwR5ZcNG9Rc8QgfpMTE +CM0lgjYqcxiJ65YUUoEuz8QeAteRPdSNgIDy3A1q4PbPyFwjwAQYAxW4BhEyqnJx +EAf8BMZw4t4qbaN/j5Nc+KzyRmA0vcQVzReZj3cRo+qx9gkiDDsD0zz/Nx2iYFCl +XrQwGquq0VMZXwKCAQEAzcc1uaeJbMvvEI2OphNroMmxmkXM++M4qQnhXeZAHkU1 +9lTKvmMCM1arVW/anIXb4PPUCkT9C5a1adHCW43KPzR/x9Rh2q8oqNy8Eb9HdRkn +5gjQogsQJLC9CSyuwdpSrtjqZZMSv533vRSu53GoCCDIM9iL8MyWhMo3yOzouoh7 +09Xm7fpeU7OU8PGqmCWcT1ykNDcyKp4Qn+FjfAGt8RX81dJp8Bd9s97fu1oFK1jv +FHjDiofdFveqVucFuxnEydOohhcnyKOEKLiGHH1iFP0BKNzGTswiboiMCPFJUgkX +3F18Rpx9eM0qzHeD38OOGTu9kwYK7XJaRiQlh33BLQKCAQEAkWtL7dqfv5mZrE9b +5aW1XGRBtt+Ok2hEJdUmFjXFIt/+VOl/7YCT2YCEIb9Xew45W95x2Is2x0zoQhte +8vzxSd6onWdrlHAyukoJVzWDRGzO849N9F17E7Dv2nzt3HDW6fw47wxROekdI48Q +H2mfkTWred855+W6Xket0cFeLnnMg7CBq/1DgUfmD6MdySZd3zlQyN6qYYZAMJ68 +w3Im/4GBB5qfe8pXbwOuG8MNGhUkKZqNPaYu1j/ZMFzTrMGiDwhujFQ7qtBdTUnt +gV1p0WqSL1ct4RvW3uysPJnNk/WWxpUzX/oQVtQR+lYh6aQoamRprqqglWxBy4IH +PQ/GHQKCAQEAgGfBl6L6tyAegobCr9FIhOG26JyLx9ZG4fxfXBe0hO8NODQ6+0iO +8/guG7cbhDZnOV0NQIdCG7wkYe7ZSAYXYQ9ieRCHK03Hom9zD+P7NA/JfNqdTbB3 +l12N6C6wshiJJcAHq/B1b4qkL6G7boLWppPGOcvzrwQLHLuOydtDcjp+gnZkjrV/ +rN1PoHY8zIoTmtYlRMv4iu1tVEbxrEzj/J+K0OZm4sHV16rIk4Ed3qm1LmIZKtWS +2aUb7vRj+BehW4uvClVipumaK0rRrO90h8JEufnh6QOqrKyaLW5bIUy5TfIr/Wfj +R8FG5qXhbXz6q7ZCmn7XzMlhmYkuvfhz1QKCAQEAvCLx84l0WONH1jq0XTXssQev +CVj0k3izQ4/KRsEJT8elWU/94dMqS2vLfhaF2/ySJkH7/Dr+UpmUbLZa6xlO1Rp0 +Vw0EuYlWgoSvD11YGC+8LoSjNgcePyQFE0Kv+i7UHBGibI23ooyOGbap/Nq94Fhk +mI9JNKgiQ1G4sqkDpfvNEb5a95OdWDCKwZWdaRttYmu5fb1xxpTur+r0QrsJR81y +FgFbV9OU1IBI8HirMO5JpTicenatszi+fCif0ldDAnkiM6apzuqOuM7y9mlWt6cg +8HQuTwBKqTNbj35Red79YcdBnPbLkbwPWi6NfPdQpC3icEDlwOzVjiMFOuex+Q== +-----END RSA PRIVATE KEY----- diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index d346d59a482..7ed11691b00 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -1465,6 +1465,12 @@ async def getTestAha(self, conf=None, dirn=None, ctor=None): with self.mayTestDir(dirn) as dirn: + if conf.get('aha:network') == 'synapse': + dstpath = os.path.join(dirn, 'certs') + if not os.path.isdir(dstpath): + srcpath = self.getTestFilePath('aha/certs') + shutil.copytree(srcpath, os.path.join(dirn, 'certs')) + async with await ctor(dirn, conf=conf) as aha: mods = {} From 15a57d9f3d8eaeec73ceadaca7427f65c1c6aed2 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 8 Jul 2024 18:59:10 -0400 Subject: [PATCH 30/86] AHA Gather APIs --- synapse/lib/aha.py | 106 +++++++++++++++++++++++++++++++++- synapse/lib/cell.py | 8 +-- synapse/lib/nexus.py | 5 +- synapse/tests/test_lib_aha.py | 51 ++++++++++++++++ 4 files changed, 164 insertions(+), 6 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index d3ce45e1d39..8b327ad983c 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -136,6 +136,16 @@ async def getAhaUrls(self, user='root'): return () return ahaurls + @s_cell.adminapi() + async def runGatherCall(self, iden, todo, timeout=None): + async for item in self.cell.runGatherCall(iden, todo, timeout=timeout): + yield item + + @s_cell.adminapi() + async def runGatherGenr(self, iden, todo, timeout=None): + async for item in self.cell.runGatherGenr(iden, todo, timeout=timeout): + yield item + async def getAhaSvc(self, name, filters=None): ''' Return an AHA service description dictionary for a service name. @@ -563,7 +573,6 @@ async def _initCellBoot(self): async def initServiceStorage(self): - # TODO plumb using a remote jsonstor? dirn = s_common.gendir(self.dirn, 'slabs', 'jsonstor') slab = await s_lmdbslab.Slab.anit(dirn) @@ -675,6 +684,101 @@ def _initCellHttpApis(self): self.addHttpApi('/api/v1/aha/services', AhaServicesV1, {'cell': self}) self.addHttpApi('/api/v1/aha/provision/service', AhaProvisionServiceV1, {'cell': self}) + async def getAhaSvcsByIden(self, iden, online=True): + + runs = set() + async for svcdef in self.getAhaSvcs(): + + # TODO services by iden indexes! + if svcdef['svcinfo'].get('iden') != iden: + continue + + if online and svcdef['svcinfo'].get('online') is None: + continue + + svcrun = svcdef['svcinfo'].get('run') + if svcrun in runs: + continue + + runs.add(svcrun) + yield svcdef + + async def runGatherCall(self, iden, todo, timeout=None): + + queue = asyncio.Queue() + network = self.conf.get('aha:network') + + async def call(svcdef): + + svcname = svcdef.get('svcname') + svcnetw = svcdef.get('svcnetw') + svcfull = f'{svcname}.{svcnetw}' + + try: + + host = svcdef['svcinfo']['urlinfo']['host'] + port = svcdef['svcinfo']['urlinfo']['port'] + + svcurl = f'ssl://{host}:{port}?hostname={svcfull}&certname=root@{svcnetw}' + + async with await s_telepath.openurl(svcurl) as proxy: + valu = await asyncio.wait_for(proxy.taskv2(todo), timeout=timeout) + await queue.put((svcfull, (True, valu))) + + except Exception as e: + await queue.put((svcfull, (False, s_common.excinfo(e)))) + + count = 0 + async for svcdef in self.getAhaSvcsByIden(iden): + count += 1 + self.schedCoro(call(svcdef)) + + for i in range(count): + yield await queue.get() + + async def runGatherGenr(self, iden, todo, timeout=None): + + queue = asyncio.Queue() + network = self.conf.get('aha:network') + + async def call(svcdef): + + svcname = svcdef.get('svcname') + svcnetw = svcdef.get('svcnetw') + svcfull = f'{svcname}.{svcnetw}' + + try: + + host = svcdef['svcinfo']['urlinfo']['host'] + port = svcdef['svcinfo']['urlinfo']['port'] + + svcurl = f'ssl://{host}:{port}?hostname={svcfull}&certname=root@{svcnetw}' + + async with await s_telepath.openurl(svcurl) as proxy: + async for item in await proxy.taskv2(todo): + await queue.put((svcfull, (True, item))) + + except Exception as e: + await queue.put((svcfull, (False, s_exc.excinfo(e)))) + + finally: + await queue.put((svcfull, None)) + + count = 0 + async for svcdef in self.getAhaSvcsByIden(iden): + count += 1 + self.schedCoro(call(svcdef)) + + while count > 0: + + item = await queue.get() + + yield item + + if item[1] is None: + count -= 1 + continue + async def initServiceRuntime(self): self.addActiveCoro(self._clearInactiveSessions) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index c32898ab894..ea1aee0a927 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -716,8 +716,8 @@ async def saveHiveTree(self, path=()): return await self.cell.saveHiveTree(path=path) @adminapi() - async def getNexusChanges(self, offs, tellready=False): - async for item in self.cell.getNexusChanges(offs, tellready=tellready): + async def getNexusChanges(self, offs, tellready=False, wait=True): + async for item in self.cell.getNexusChanges(offs, tellready=tellready, wait=wait): yield item @adminapi() @@ -2007,8 +2007,8 @@ async def initServiceActive(self): # pragma: no cover async def initServicePassive(self): # pragma: no cover pass - async def getNexusChanges(self, offs, tellready=False): - async for item in self.nexsroot.iter(offs, tellready=tellready): + async def getNexusChanges(self, offs, tellready=False, wait=True): + async for item in self.nexsroot.iter(offs, tellready=tellready, wait=wait): yield item def _reqBackDirn(self, name): diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py index 8418232e8a5..f3b3813702c 100644 --- a/synapse/lib/nexus.py +++ b/synapse/lib/nexus.py @@ -403,7 +403,7 @@ async def _apply(self, indx, mesg): return await func(nexus, *args, **kwargs) - async def iter(self, offs: int, tellready=False) -> AsyncIterator[Any]: + async def iter(self, offs: int, tellready=False, wait=True) -> AsyncIterator[Any]: ''' Returns an iterator of change entries in the log ''' @@ -424,6 +424,9 @@ async def iter(self, offs: int, tellready=False) -> AsyncIterator[Any]: if tellready: yield None + if not wait: + return + async with self.getChangeDist(maxoffs) as dist: async for item in dist: if self.isfini: diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 24f82a28598..00aae028331 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1188,3 +1188,54 @@ async def test_aha_provision_longname(self): with self.raises(s_exc.CryptoErr) as errcm: await s_aha.AhaCell.anit(aha00dirn, conf=aconf) self.isin('Certificate name values must be between 1-64 characters', errcm.exception.get('mesg')) + + async def test_aha_gather(self): + + async with self.getTestAha() as aha: + + async with aha.waiter(3, 'aha:svcadd', timeout=10): + + conf = {'aha:provision': await aha.addAhaSvcProv('00.cell')} + cell00 = await aha.enter_context(self.getTestCell(conf=conf)) + + conf = {'aha:provision': await aha.addAhaSvcProv('01.cell', {'mirror': 'cell'})} + cell01 = await aha.enter_context(self.getTestCell(conf=conf)) + + await cell01.sync() + + nexsindx = await cell00.getNexsIndx() + + # test the call endpoint + todo = s_common.todo('getCellInfo') + items = dict([item async for item in aha.runGatherCall(cell00.iden, todo, timeout=3)]) + self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) + self.true(all(item[0] for item in items.values())) + self.eq(cell00.runid, items['00.cell.synapse'][1]['cell']['run']) + self.eq(cell01.runid, items['01.cell.synapse'][1]['cell']['run']) + + todo = s_common.todo('newp') + items = dict([item async for item in aha.runGatherCall(cell00.iden, todo, timeout=3)]) + self.false(any(item[0] for item in items.values())) + self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) + + # test the genr endpoint + todo = s_common.todo('getNexusChanges', 0, wait=False) + items = [item async for item in aha.runGatherGenr(cell00.iden, todo, timeout=3) if item[1]] + self.len(nexsindx * 2, items) + + # ensure we handle down services correctly + async with aha.waiter(1, 'aha:svcdown', timeout=10): + await cell01.fini() + + # test the call endpoint + todo = s_common.todo('getCellInfo') + items = dict([item async for item in aha.runGatherCall(cell00.iden, todo, timeout=3)]) + self.sorteq(items.keys(), ('00.cell.synapse',)) + self.true(all(item[0] for item in items.values())) + self.eq(cell00.runid, items['00.cell.synapse'][1]['cell']['run']) + + # test the genr endpoint + todo = s_common.todo('getNexusChanges', 0, wait=False) + items = [item async for item in aha.runGatherGenr(cell00.iden, todo, timeout=3) if item[1]] + self.len(nexsindx, items) + self.true(all(item[1][0] for item in items)) From e18708eec99a8edfade4203896f567e953688319 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 8 Jul 2024 19:33:43 -0400 Subject: [PATCH 31/86] wip --- synapse/lib/aha.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 8b327ad983c..9f6b179a7fa 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -3,6 +3,7 @@ import random import asyncio import logging +import contextlib import collections import cryptography.x509 as c_x509 @@ -703,11 +704,24 @@ async def getAhaSvcsByIden(self, iden, online=True): runs.add(svcrun) yield svcdef + @contextlib.asynccontextmanager + async def getAhaSvcProxy(self, svcdef): + + svcname = svcdef.get('svcname') + svcnetw = svcdef.get('svcnetw') + svcfull = f'{svcname}.{svcnetw}' + + host = svcdef['svcinfo']['urlinfo']['host'] + port = svcdef['svcinfo']['urlinfo']['port'] + + svcurl = f'ssl://{host}:{port}?hostname={svcfull}&certname=root@{svcnetw}' + + async with await s_telepath.openurl(svcurl) as proxy: + yield proxy + async def runGatherCall(self, iden, todo, timeout=None): queue = asyncio.Queue() - network = self.conf.get('aha:network') - async def call(svcdef): svcname = svcdef.get('svcname') @@ -715,13 +729,7 @@ async def call(svcdef): svcfull = f'{svcname}.{svcnetw}' try: - - host = svcdef['svcinfo']['urlinfo']['host'] - port = svcdef['svcinfo']['urlinfo']['port'] - - svcurl = f'ssl://{host}:{port}?hostname={svcfull}&certname=root@{svcnetw}' - - async with await s_telepath.openurl(svcurl) as proxy: + async with self.getAhaSvcProxy(svcdef) as proxy: valu = await asyncio.wait_for(proxy.taskv2(todo), timeout=timeout) await queue.put((svcfull, (True, valu))) @@ -739,7 +747,6 @@ async def call(svcdef): async def runGatherGenr(self, iden, todo, timeout=None): queue = asyncio.Queue() - network = self.conf.get('aha:network') async def call(svcdef): @@ -748,13 +755,7 @@ async def call(svcdef): svcfull = f'{svcname}.{svcnetw}' try: - - host = svcdef['svcinfo']['urlinfo']['host'] - port = svcdef['svcinfo']['urlinfo']['port'] - - svcurl = f'ssl://{host}:{port}?hostname={svcfull}&certname=root@{svcnetw}' - - async with await s_telepath.openurl(svcurl) as proxy: + async with self.getAhaSvcProxy(svcdef) as proxy: async for item in await proxy.taskv2(todo): await queue.put((svcfull, (True, item))) From be760d2a8286b78d1cb38b23a86b39e74d3065ca Mon Sep 17 00:00:00 2001 From: invisig0th Date: Tue, 9 Jul 2024 12:50:12 -0400 Subject: [PATCH 32/86] Update docs/synapse/deploymentguide.rst Co-authored-by: Cisphyx --- docs/synapse/deploymentguide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index 26b16a9ae12..e92774e9830 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -83,7 +83,7 @@ Choose an AHA Network Name -------------------------- Your AHA network name is separate from DNS and is used by services within the AHA service deployment. It is -generally a good idea to chose a name that aligns with the use case for the synapse deployment. For example, +generally a good idea to chose a name that aligns with the use case for the Synapse deployment. For example, if you plan to have a test/dev/prod deployment, choosing a name like ``prod.synapse`` will make it clear which deployment is which. Changing this name later is difficult, so choose carefully! Throughout the examples, we will be using ``prod.synapse`` as the AHA network name which is also used as the common-name (CN) for the CA From a7c5e3d6896297e8df4218ccfd5d947b663fada5 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 9 Jul 2024 15:18:19 -0400 Subject: [PATCH 33/86] wip --- synapse/tests/test_lib_agenda.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/tests/test_lib_agenda.py b/synapse/tests/test_lib_agenda.py index 7497319ace1..5280cec22e4 100644 --- a/synapse/tests/test_lib_agenda.py +++ b/synapse/tests/test_lib_agenda.py @@ -850,7 +850,7 @@ async def task(): async def test_cron_kill_pool(self): - async with self.getTestAhaProv() as aha: + async with self.getTestAha() as aha: import synapse.cortex as s_cortex import synapse.lib.base as s_base @@ -872,11 +872,11 @@ async def test_cron_kill_pool(self): msgs = await core00.stormlist('aha.pool.add pool00...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('Created AHA service pool: pool00.loop.vertex.link', msgs) + self.stormIsInPrint('Created AHA service pool: pool00.synapse', msgs) msgs = await core00.stormlist('aha.pool.svc.add pool00... 01.core...') self.stormHasNoWarnErr(msgs) - self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.loop.vertex.link)', msgs) + self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.synapse)', msgs) msgs = await core00.stormlist('cortex.storm.pool.set --connection-timeout 1 --sync-timeout 1 aha://pool00...') self.stormHasNoWarnErr(msgs) From 90b7959bc167e09312b6e4e73400b1fb71606bed Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 9 Jul 2024 16:51:29 -0400 Subject: [PATCH 34/86] wip --- synapse/lib/aha.py | 4 +-- synapse/lib/cell.py | 53 +++++++++++++++++------------------ synapse/tests/test_lib_aha.py | 9 ++++++ 3 files changed, 36 insertions(+), 30 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index d3ce45e1d39..a5a406d4b17 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -1272,7 +1272,7 @@ async def getAhaUrls(self, user='root'): return urls - def _getAhaUrl(self, user='root'): + def getMyUrl(self, user='root'): port = self.sockaddr[1] host = self._getDnsName() network = self.conf.req('aha:network') @@ -1291,7 +1291,7 @@ async def addAhaClone(self, host, port=27492, conf=None): network = self.conf.req('aha:network') - conf['mirror'] = self._getAhaUrl() + conf['mirror'] = self.getMyUrl() conf['dns:name'] = host conf['aha:network'] = network diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index c32898ab894..3de3adf57c5 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1759,6 +1759,11 @@ async def waitFor(turl_sani, prox_): async def setNexsIndx(self, indx): return await self.nexsroot.setindex(indx) + def getMyUrl(self, user='root'): + host = self.conf.req('aha:name') + network = self.conf.req('aha:network') + return f'aha://{host}.{network}' + async def promote(self, graceful=False): ''' Transform this cell from a passive follower to @@ -1769,45 +1774,36 @@ async def promote(self, graceful=False): mesg = 'promote() called on non-mirror' raise s_exc.BadConfValu(mesg=mesg) - ahaname = self.conf.get('aha:name') - logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful} ahaname={ahaname}') + myurl = self.getMyUrl() + logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}.') if graceful: - if ahaname is None: # pragma: no cover - mesg = 'Cannot gracefully promote without aha:name configured.' - raise s_exc.BadArg(mesg=mesg) - - ahanetw = self.conf.req('aha:network') - - myurl = f'aha://{ahaname}.{ahanetw}' - logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}') + logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff.') async with await s_telepath.openurl(mirurl) as lead: - logger.debug(f'PROMOTION: Requesting leadership handoff to ahaname={ahaname}') await lead.handoff(myurl) - logger.warning(f'PROMOTION: Completed leadership handoff to ahaname={ahaname}') + logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}') return - logger.debug(f'PROMOTION: Clearing mirror configuration for ahaname={ahaname}') + logger.debug(f'PROMOTION: Clearing mirror configuration.') self.modCellConf({'mirror': None}) - logger.debug(f'PROMOTION: Promoting the nexus root for ahaname={ahaname}') + logger.debug(f'PROMOTION: Promoting the nexus root.') await self.nexsroot.promote() - logger.debug(f'PROMOTION: Setting the cell as active ahaname={ahaname}') + logger.debug(f'PROMOTION: Setting the cell as active.') await self.setCellActive(True) - logger.warning(f'PROMOTION: Finished leadership promotion ahaname={ahaname}') + logger.warning(f'PROMOTION: Finished leadership promotion!') async def handoff(self, turl, timeout=30): ''' Hand off leadership to a mirror in a transactional fashion. ''' - ahaname = self.conf.get("aha:name") - logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}') + logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}.') async with await s_telepath.openurl(turl) as cell: - logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}') + logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)}.') if self.iden != await cell.getCellIden(): # pragma: no cover mesg = 'Mirror handoff remote cell iden does not match!' @@ -1817,33 +1813,34 @@ async def handoff(self, turl, timeout=30): mesg = 'Cannot handoff mirror leadership to myself!' raise s_exc.BadArg(mesg=mesg) - logger.debug(f'HANDOFF: Obtaining nexus lock ahaname={ahaname}') + logger.debug(f'HANDOFF: Obtaining nexus lock.') async with self.nexslock: - logger.debug(f'HANDOFF: Obtained nexus lock ahaname={ahaname}') + logger.debug(f'HANDOFF: Obtained nexus lock.') indx = await self.getNexsIndx() - logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}, ahaname={ahaname}') + logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}.') if not await cell.waitNexsOffs(indx - 1, timeout=timeout): # pragma: no cover mndx = await cell.getNexsIndx() mesg = f'Remote mirror did not catch up in time: {mndx}/{indx}.' raise s_exc.NotReady(mesg=mesg) - logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion ahaname={ahaname}') + logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion.') await cell.promote() - logger.debug(f'HANDOFF: Setting the service as inactive ahaname={ahaname}') + logger.debug(f'HANDOFF: Setting the service as inactive.') await self.setCellActive(False) - logger.debug(f'HANDOFF: Configuring service to use the new leader as its mirror ahaname={ahaname}') + logger.debug(f'HANDOFF: Configuring service to sync from new leader.') self.modCellConf({'mirror': turl}) - logger.debug(f'HANDOFF: Restarting the nexus ahaname={ahaname}') + logger.debug(f'HANDOFF: Restarting the nexus.') await self.nexsroot.startup() - logger.debug(f'HANDOFF: Released nexus lock ahaname={ahaname}') - logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)} ahaname={self.conf.get("aha:name")}') + logger.debug(f'HANDOFF: Released nexus lock.') + + logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}.') async def reqAhaProxy(self, timeout=None): if self.ahaclient is None: diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 24f82a28598..a66edc2b131 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -104,6 +104,15 @@ async def test_lib_aha_clone(self): mnfo = await aha1.getAhaSvc('test.example.net') self.none(mnfo) + self.true(aha0.isactive) + self.false(aha1.isactive) + + async with aha1.getLocalProxy() as proxy: + await proxy.promote(graceful=True) + + self.false(aha0.isactive) + self.true(aha1.isactive) + async def test_lib_aha_offon(self): with self.getTestDir() as dirn: cryo0_dirn = s_common.gendir(dirn, 'cryo0') From 25af8b63b8836af54702bf3db9b9d3a2b366d3e1 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 9 Jul 2024 17:34:23 -0400 Subject: [PATCH 35/86] wip --- synapse/lib/cell.py | 3 ++- synapse/tests/test_lib_aha.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 3de3adf57c5..ddc6d563269 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1774,11 +1774,12 @@ async def promote(self, graceful=False): mesg = 'promote() called on non-mirror' raise s_exc.BadConfValu(mesg=mesg) - myurl = self.getMyUrl() logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}.') if graceful: + myurl = self.getMyUrl() + logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff.') async with await s_telepath.openurl(mirurl) as lead: await lead.handoff(myurl) diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index a66edc2b131..c37bfe6f621 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -884,7 +884,7 @@ async def test_aha_connect_back(self): async with self.getTestAha() as aha: # type: s_aha.AhaCell async with self.addSvcToAha(aha, '00.exec', ExecTeleCaller) as conn: - ahaurl = aha._getAhaUrl() + ahaurl = aha.getMyUrl() await conn.exectelecall(ahaurl, 'getNexsIndx') self.true(conn.ahaclient.isfini) From e4dea7bd398b5da2fb3050764074b2a6306949bb Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 10 Jul 2024 08:41:17 -0400 Subject: [PATCH 36/86] make timeout message a bit prettier --- synapse/lib/base.py | 3 ++- synapse/tests/test_lib_base.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/lib/base.py b/synapse/lib/base.py index 8fe9d6bb7a0..8040fe60c75 100644 --- a/synapse/lib/base.py +++ b/synapse/lib/base.py @@ -686,7 +686,8 @@ async def __aenter__(self): async def __aexit__(self, exc, cls, tb): if exc is None: if await self.wait() is None: - mesg = f'timeout waiting for {self.count} events {self.names}' + events = ','.join(self.names) + mesg = f'timeout waiting for {self.count} event(s): {events}' raise s_exc.TimeOut(mesg=mesg) class BaseRef(Base): diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py index 3bd9b1be99b..f7d746b35fa 100644 --- a/synapse/tests/test_lib_base.py +++ b/synapse/tests/test_lib_base.py @@ -226,7 +226,7 @@ async def test_base_waiter(self): self.len(2, evts) with self.raises(s_exc.TimeOut): - async with base0.waiter(1, 'newp', timeout=0.01): + async with base0.waiter(1, 'newp', 'nuuh', timeout=0.01): pass async def test_baseref(self): From c7f1973091411248755f89340d49903ae922aaef Mon Sep 17 00:00:00 2001 From: invisig0th Date: Wed, 10 Jul 2024 09:06:17 -0400 Subject: [PATCH 37/86] Update synapse/lib/aha.py Co-authored-by: Cisphyx --- synapse/lib/aha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index a5a406d4b17..665e791375a 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -540,7 +540,7 @@ async def _initCellBoot(self): path = s_common.genpath(self.dirn, 'cell.guid') if os.path.isfile(path): - logger.info('Cloneing AHA: cell.guid detected. Skipping.') + logger.info('Cloning AHA: cell.guid detected. Skipping.') return logger.warning(f'Cloning AHA: {curl}') From f62ee425ea10a6d263a174ee74d0b91487857248 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 10 Jul 2024 13:14:00 +0000 Subject: [PATCH 38/86] Do container branch build --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index bf7ce9db25d..0234a01bf6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -603,6 +603,7 @@ workflows: branches: only: - master + - visi-aha-defnet - build_docker_tag: requires: From 5954fa4f3ee91744cc9c2c00a854c4f856b12e60 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Fri, 12 Jul 2024 14:41:59 +0000 Subject: [PATCH 39/86] Update docker test script for required value. --- docker/scripts/test_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/test_all.sh b/docker/scripts/test_all.sh index 2a3b809803e..b72d8616996 100755 --- a/docker/scripts/test_all.sh +++ b/docker/scripts/test_all.sh @@ -28,7 +28,7 @@ echo "Spinning up images" docker run --rm -it --entrypoint python vertexproject/synapse:${TAG} -m synapse.servers.cortex --help dstatus00=$? if [ $dstatus00 != "0" ]; then exit 1; fi -docker run --rm -d --name test-aha vertexproject/synapse-aha:${TAG} +docker run --rm -d --name test-aha -e "SYN_AHA_AHA_NETWORK=synapse.ci" vertexproject/synapse-aha:${TAG} docker run --rm -d --name test-axon vertexproject/synapse-axon:${TAG} docker run --rm -d --name test-cortex vertexproject/synapse-cortex:${TAG} docker run --rm -d --name test-cryotank vertexproject/synapse-cryotank:${TAG} From c7145e3bed4f01797ee3d17813d46fc945350681 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 12 Jul 2024 13:07:29 -0400 Subject: [PATCH 40/86] wip --- synapse/lib/aha.py | 4 ++-- synapse/lib/config.py | 4 ++++ synapse/telepath.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 665e791375a..7e677f298d7 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -611,7 +611,6 @@ async def _addAhaServer(self, server): host = server.get('host') port = server.get('port') - oldv = None lkey = s_msgpack.en((host, port)) byts = self.slab.get(lkey, db='aha:servers') @@ -751,7 +750,6 @@ async def initServiceNetwork(self): # bootstrap CA/host certs first network = self.conf.req('aha:network') - await self._genCaCert(network) hostname = self._getDnsName() if hostname is not None and network is not None: @@ -1155,6 +1153,8 @@ async def genCaCert(self, network): async def _genCaCert(self, network): + # generate a CA cert if one does not exist + # ( but don't read it in if it does ) if os.path.isfile(os.path.join(self.dirn, 'certs', 'cas', f'{network}.crt')): return diff --git a/synapse/lib/config.py b/synapse/lib/config.py index 5295a4bac7a..d03a950e0bf 100644 --- a/synapse/lib/config.py +++ b/synapse/lib/config.py @@ -392,6 +392,10 @@ def reqConfValid(self): else: return + def reqConfValu(self, name): + s_common.deprecated('reqConfValu()') + return self.req(name) + def req(self, name): ''' Return a configuration value or raise NeedConfValu if it is unset. diff --git a/synapse/telepath.py b/synapse/telepath.py index 388bc2b108c..ba45546d406 100644 --- a/synapse/telepath.py +++ b/synapse/telepath.py @@ -1292,7 +1292,7 @@ async def _teleLinkLoop(self): now = time.monotonic() if now > lastlog + 60.0: # don't logspam the disconnect message more than 1/min url = s_urlhelp.sanitizeUrl(zipurl(urlinfo)) - logger.warning(f'telepath client ({url}) encountered an error: {e}') + logger.warning(f'telepath client ({url}) encountered an error: {e}', exc_info=e) lastlog = now await self.waitfini(timeout=self._t_conf.get('retrysleep', 0.2)) From f2c4faeb5e652d98ac2dbcf60feab54c6b8393b3 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 15 Jul 2024 09:10:57 -0400 Subject: [PATCH 41/86] wip --- synapse/lib/cell.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 32c41be28cc..5be7acb6932 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -3849,15 +3849,12 @@ async def initFromArgv(cls, argv, outp=None): try: - if 'dmon:listen' not in cell.conf: - await cell.dmon.listen(opts.telepath) - logger.info(f'...{cell.getCellType()} API (telepath): {opts.telepath}') - else: - lisn = cell.conf.get('dmon:listen') - if lisn is None: - lisn = cell.getLocalUrl() + turl = cell._getDmonListen() + if turl is None: + turl = opts.telepath + await cell.dmon.listen(turl) - logger.info(f'...{cell.getCellType()} API (telepath): {lisn}') + logger.info(f'...{cell.getCellType()} API (telepath): {turl}') if 'https:port' not in cell.conf: await cell.addHttpsPort(opts.https) From 3badfdc57cfd5628fa090de6244db4a75ff9c088 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 15 Jul 2024 15:06:01 -0400 Subject: [PATCH 42/86] wip --- synapse/tests/test_lib_cell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index 025397f635c..a46b1e9ad73 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -913,7 +913,7 @@ async def test_cell_confprint(self): pass stream.seek(0) buf = stream.read() - self.isin(f'...cell API (telepath): cell://root@{dirn}:*', buf) + self.isin(f'...cell API (telepath): tcp://0.0.0.0:27492', buf) self.isin('...cell API (https): disabled', buf) async def test_cell_initargv_conf(self): From cbc39e3ef37ab839063b14e4eddba0d52dba2f32 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jul 2024 20:50:49 +0000 Subject: [PATCH 43/86] Fix fstrings --- synapse/lib/aha.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 7e677f298d7..79bd5438ab1 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -1165,7 +1165,7 @@ async def _genHostCert(self, hostname, signas=None): if signas is not None: await self._genCaCert(signas) - if os.path.isfile(os.path.join(self.dirn, 'certs', 'hosts', '{hostname}.crt')): + if os.path.isfile(os.path.join(self.dirn, 'certs', 'hosts', f'{hostname}.crt')): return pkey, cert = await s_coro.executor(self.certdir.genHostCert, hostname, signas=signas, save=False) @@ -1175,7 +1175,7 @@ async def _genHostCert(self, hostname, signas=None): async def _genUserCert(self, username, signas=None): - if os.path.isfile(os.path.join(self.dirn, 'certs', 'users', '{username}.crt')): + if os.path.isfile(os.path.join(self.dirn, 'certs', 'users', f'{username}.crt')): return logger.info(f'Adding user certificate for {username}') From 7fb5d584cf3ce678bdd7fc2264b4f4ef74ed6596 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Mon, 15 Jul 2024 21:13:43 +0000 Subject: [PATCH 44/86] Fix fstring for clone; add --only-url --- synapse/tools/aha/clone.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/tools/aha/clone.py b/synapse/tools/aha/clone.py index 878b08f7425..be0c7b03938 100644 --- a/synapse/tools/aha/clone.py +++ b/synapse/tools/aha/clone.py @@ -22,7 +22,8 @@ async def main(argv, outp=s_output.stdout): pars.add_argument('dnsname', help='The DNS name of the new AHA server.') pars.add_argument('--port', type=int, default=27492, help='The port that the new AHA server should listen on.') pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL to connect to the AHA service.') - + pars.add_argument('--only-url', help='Only output the URL upon successful execution', + action='store_true', default=False) opts = pars.parse_args(argv) async with s_telepath.withTeleEnv(): @@ -30,7 +31,11 @@ async def main(argv, outp=s_output.stdout): try: async with await s_telepath.openurl(opts.url) as aha: curl = await aha.addAhaClone(opts.dnsname, port=opts.port) - outp.printf('one-time use URL: {curl}') + + if opts.only_url: + outp.printf(curl) + else: + outp.printf(f'one-time use URL: {curl}') return 0 except Exception as e: From 760331e010292e067180a355314d9e05bb6d9571 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 17 Jul 2024 10:58:24 -0400 Subject: [PATCH 45/86] wip --- synapse/lib/aha.py | 82 +++++++++++++++++++++++---------------------- synapse/lib/cell.py | 25 ++++++++++++++ 2 files changed, 67 insertions(+), 40 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 04dd3d1081b..74d8fdbbb58 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -704,7 +704,7 @@ async def getAhaSvcsByIden(self, iden, online=True): yield svcdef @contextlib.asynccontextmanager - async def getAhaSvcProxy(self, svcdef): + async def getAhaSvcProxy(self, svcdef, user='root'): svcname = svcdef.get('svcname') svcnetw = svcdef.get('svcnetw') @@ -713,7 +713,7 @@ async def getAhaSvcProxy(self, svcdef): host = svcdef['svcinfo']['urlinfo']['host'] port = svcdef['svcinfo']['urlinfo']['port'] - svcurl = f'ssl://{host}:{port}?hostname={svcfull}&certname=root@{svcnetw}' + svcurl = f'ssl://{host}:{port}?hostname={svcfull}&certname={user}@{svcnetw}' async with await s_telepath.openurl(svcurl) as proxy: yield proxy @@ -721,63 +721,65 @@ async def getAhaSvcProxy(self, svcdef): async def runGatherCall(self, iden, todo, timeout=None): queue = asyncio.Queue() - async def call(svcdef): + async with await s_base.Base.anit() as base: + async def call(svcdef): - svcname = svcdef.get('svcname') - svcnetw = svcdef.get('svcnetw') - svcfull = f'{svcname}.{svcnetw}' + svcname = svcdef.get('svcname') + svcnetw = svcdef.get('svcnetw') + svcfull = f'{svcname}.{svcnetw}' - try: - async with self.getAhaSvcProxy(svcdef) as proxy: - valu = await asyncio.wait_for(proxy.taskv2(todo), timeout=timeout) - await queue.put((svcfull, (True, valu))) + try: + async with self.getAhaSvcProxy(svcdef) as proxy: + valu = await asyncio.wait_for(proxy.taskv2(todo), timeout=timeout) + await queue.put((svcfull, (True, valu))) - except Exception as e: - await queue.put((svcfull, (False, s_common.excinfo(e)))) + except Exception as e: + await queue.put((svcfull, (False, s_common.excinfo(e)))) - count = 0 - async for svcdef in self.getAhaSvcsByIden(iden): - count += 1 - self.schedCoro(call(svcdef)) + count = 0 + async for svcdef in self.getAhaSvcsByIden(iden): + count += 1 + self.base(call(svcdef)) - for i in range(count): - yield await queue.get() + for i in range(count): + yield await queue.get() async def runGatherGenr(self, iden, todo, timeout=None): queue = asyncio.Queue() - async def call(svcdef): + async with await s_base.Base.anit() as base: - svcname = svcdef.get('svcname') - svcnetw = svcdef.get('svcnetw') - svcfull = f'{svcname}.{svcnetw}' + async def call(svcdef): - try: - async with self.getAhaSvcProxy(svcdef) as proxy: - async for item in await proxy.taskv2(todo): - await queue.put((svcfull, (True, item))) + svcname = svcdef.get('svcname') + svcnetw = svcdef.get('svcnetw') + svcfull = f'{svcname}.{svcnetw}' - except Exception as e: - await queue.put((svcfull, (False, s_exc.excinfo(e)))) + try: + async with self.getAhaSvcProxy(svcdef) as proxy: + async for item in await proxy.taskv2(todo): + await queue.put((svcfull, (True, item))) - finally: - await queue.put((svcfull, None)) + except Exception as e: + await queue.put((svcfull, (False, s_common.excinfo(e)))) - count = 0 - async for svcdef in self.getAhaSvcsByIden(iden): - count += 1 - self.schedCoro(call(svcdef)) + finally: + await queue.put((svcfull, None)) - while count > 0: + count = 0 + async for svcdef in self.getAhaSvcsByIden(iden): + count += 1 + base.schedCoro(call(svcdef)) - item = await queue.get() + while count > 0: - yield item + item = await asyncio.wait_for(queue.get(), timeout=timeout) + if item[1] is None: + count -= 1 + continue - if item[1] is None: - count -= 1 - continue + yield item async def initServiceRuntime(self): diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index fd7fbea68ab..94994b02c57 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -4043,6 +4043,31 @@ async def ps(self, user): return retn + async def ahaGatherPs(self, user): + + if self.ahaclient is None: + return await self.ps() + + retn = [] + todo = s_common.todo('ps') + + isallowed = await self.isUserAllowed(user.iden, ('task', 'get')) + + try: + proxy = self.ahaclient.proxy(timeout=4) + async for svcfull, (ok, tasks) in proxy.runGatherCall(todo, timeout=4): + if ok: + for task in tasks: + if isallowed or task['user'] == user.name: + retn.append(task) + else: + logger.warning(f'AHA gather ps() from {svcfull}: {tasks}') + + except Exception as e: + logger.warning(f'AHA gather ps() on {self.ahasvcname}: {e}') + + # async def _ahaGatherKill(self, user, iden): + async def kill(self, user, iden): perm = ('task', 'del') isallowed = await self.isUserAllowed(user.iden, perm) From efa7d025bd586ba4bc07fe87de039a5bf5666713 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 23 Jul 2024 14:35:06 -0400 Subject: [PATCH 46/86] wip --- docs/synapse/deploymentguide.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index e92774e9830..70a703a4f10 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -77,7 +77,14 @@ using AHA, the only host that needs DNS or other external name resolution is the .. note:: - FIXME AHA FLAT NETWORK STUFF + The AHA service resolver requires that registered services connect directly using IP addresses which + must also be reachable to AHA clients. Using an ``aha://`` telepath URL requires direct routes to the + service via it's AHA facing network address. If you need to provide telepath access from outside the + Synapse service network via any network address translation (NAT) method such as an inbound TCP proxy + or docker/kubernetes port mapping, you will need to use ``ssl://`` based URIs and specify ``hostname``, + ``certname``, and ``ca`` parameters which match the service's AHA registration info. You will also need + to specify a specific ``--dmon-port`` option when using the ``synapse.tools.aha.provision.service`` + command to provision the service. Choose an AHA Network Name -------------------------- From fe2e58382f8c367258a841db4efe59d65e442d69 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 24 Jul 2024 10:36:24 -0400 Subject: [PATCH 47/86] wip --- synapse/lib/aha.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 79bd5438ab1..5c445f28dc7 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -1154,8 +1154,7 @@ async def genCaCert(self, network): async def _genCaCert(self, network): # generate a CA cert if one does not exist - # ( but don't read it in if it does ) - if os.path.isfile(os.path.join(self.dirn, 'certs', 'cas', f'{network}.crt')): + if self.certdir.getCaCertPath(network) is not None: return await self.genCaCert(network) @@ -1165,7 +1164,7 @@ async def _genHostCert(self, hostname, signas=None): if signas is not None: await self._genCaCert(signas) - if os.path.isfile(os.path.join(self.dirn, 'certs', 'hosts', f'{hostname}.crt')): + if self.certdir.getHostCertPath(hostname) is not None: return pkey, cert = await s_coro.executor(self.certdir.genHostCert, hostname, signas=signas, save=False) @@ -1175,7 +1174,7 @@ async def _genHostCert(self, hostname, signas=None): async def _genUserCert(self, username, signas=None): - if os.path.isfile(os.path.join(self.dirn, 'certs', 'users', f'{username}.crt')): + if self.certdir.getUserCertPath(username) is not None: return logger.info(f'Adding user certificate for {username}') From 21ab32717eb0f6d7941414b930569b187bb47cf5 Mon Sep 17 00:00:00 2001 From: vEpiphyte Date: Fri, 26 Jul 2024 16:20:48 -0400 Subject: [PATCH 48/86] Visi aha defnet epiphyte (#3823) --- changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml | 5 + synapse/lib/aha.py | 30 +++++- synapse/lib/cell.py | 20 +++- synapse/lib/config.py | 28 ++++-- synapse/tests/test_lib_aha.py | 95 ++++++++++++++++++- synapse/tests/test_lib_config.py | 3 +- synapse/tests/test_tools_aha.py | 7 ++ synapse/tests/utils.py | 12 ++- 8 files changed, 179 insertions(+), 21 deletions(-) create mode 100644 changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml diff --git a/changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml b/changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml new file mode 100644 index 00000000000..6b565c84e01 --- /dev/null +++ b/changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml @@ -0,0 +1,5 @@ +--- +desc: Deprecate the ``Cell.conf.reqConfValu()`` API. This has been replaced with ``Cell.conf.req()``. +prs: [] +type: deprecation +... diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 5c445f28dc7..5a87a036c34 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -153,6 +153,7 @@ async def getAhaSvc(self, name, filters=None): username = self.user.name.split('@')[0] if svcinfo.get('svcinfo'): + logger.info(f'Hinting {self.user.name} for {name}') svcinfo['svcinfo']['urlinfo']['user'] = username return svcinfo @@ -359,6 +360,14 @@ async def clearAhaUserEnrolls(self): ''' return await self.cell.clearAhaUserEnrolls() + @s_cell.adminapi() + async def clearAhaClones(self): + ''' + Remove all unused AHA clone provisioning values. + ''' + return await self.cell.clearAhaClones() + + class ProvDmon(s_daemon.Daemon): async def __anit__(self, aha): @@ -386,8 +395,8 @@ async def _getSharedItem(self, name): clone = await self.aha.getAhaClone(name) if clone is not None: - mesg = f'Retrieved AHA clone info for {name}' host = clone.get('host') + mesg = f'Retrieved AHA clone info for {host} iden {name}' logger.info(mesg, extra=await self.aha.getLogExtra(iden=name, host=host)) return CloneApi(self.aha, clone) @@ -1221,8 +1230,8 @@ async def signHostCsr(self, csrtext, signas=None, sans=None): hostname = xcsr.subject.get_attributes_for_oid(c_x509.NameOID.COMMON_NAME)[0].value - hostpath = s_common.genpath(self.dirn, 'certs', 'hosts', f'{hostname}.crt') - if os.path.isfile(hostpath): + hostpath = self.certdir.getHostCertPath(hostname) + if hostpath is not None: os.unlink(hostpath) if signas is None: @@ -1240,8 +1249,8 @@ async def signUserCsr(self, csrtext, signas=None): username = xcsr.subject.get_attributes_for_oid(c_x509.NameOID.COMMON_NAME)[0].value - userpath = s_common.genpath(self.dirn, 'certs', 'users', f'{username}.crt') - if os.path.isfile(userpath): + userpath = self.certdir.getUserCertPath(username) + if userpath is not None: os.unlink(userpath) if signas is None: @@ -1304,6 +1313,10 @@ async def addAhaClone(self, host, port=27492, conf=None): 'conf': conf, } await self._push('aha:clone:add', clone) + + logger.info(f'Created AHA clone provisioning for {host} with iden {iden}', + extra=await self.getLogExtra(iden=iden, name=host, netw=network)) + return self._getProvClientUrl(iden) @s_nexus.Pusher.onPush('aha:clone:add') @@ -1443,6 +1456,13 @@ async def clearAhaUserEnrolls(self): userinfo = s_msgpack.un(byts) logger.info(f'Deleted user enrollment username={userinfo.get("name")}, iden={iden.decode()}') + @s_nexus.Pusher.onPushAuto('aha:clone:clear') + async def clearAhaClones(self): + for lkey, byts in self.slab.scanByFull(db='aha:clones'): + self.slab.delete(lkey, db='aha:clones') + cloninfo = s_msgpack.un(byts) + logger.info(f'Deleted AHA clone enrollment username={cloninfo.get("host")}, iden={s_common.ehex(lkey)}') + @s_nexus.Pusher.onPushAuto('aha:svc:prov:del') async def delAhaSvcProv(self, iden): self.slab.delete(iden.encode(), db='aha:provs') diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 3cedf26a2ea..93f04a57456 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1673,15 +1673,24 @@ async def _initAhaRegistry(self): ahaurls = self.conf.get('aha:registry') if ahaurls is not None: - info = await s_telepath.addAhaUrl(ahaurls) + await s_telepath.addAhaUrl(ahaurls) if self.ahaclient is not None: await self.ahaclient.fini() async def onlink(proxy): ahauser = self.conf.get('aha:user', 'root') newurls = await proxy.getAhaUrls(user=ahauser) - oldurls = tuple(self.conf.get('aha:registry')) + oldurls = self.conf.get('aha:registry') + if isinstance(oldurls, str): + oldurls = (oldurls,) + elif isinstance(oldurls, list): + oldurls = tuple(oldurls) if newurls and newurls != oldurls: + if oldurls[0].startswith('tcp://'): + s_common.deprecated('aha:registry: tcp:// client values.') + logger.warning('tcp:// based aha:registry options will be deprecated in Synapse v3.0.0') + return + self.modCellConf({'aha:registry': newurls}) self.ahaclient.setBootUrls(newurls) @@ -1694,9 +1703,14 @@ async def fini(): self.ahaclient.onfini(fini) ahaadmin = self.conf.get('aha:admin') + ahauser = self.conf.get('aha:user') + if ahaadmin is not None: await self._addAdminUser(ahaadmin) + if ahauser is not None: + await self._addAdminUser(ahauser) + def _getDmonListen(self): lisn = self.conf.get('dmon:listen', s_common.novalu) @@ -1741,7 +1755,7 @@ async def initNexusSubsystem(self): async def _bindDmonListen(self): - # functionalized so Raft code can bind early... + # functionalized so downstream code can bind early. if self.sockaddr is not None: return diff --git a/synapse/lib/config.py b/synapse/lib/config.py index d03a950e0bf..ed46d6476bf 100644 --- a/synapse/lib/config.py +++ b/synapse/lib/config.py @@ -392,20 +392,32 @@ def reqConfValid(self): else: return - def reqConfValu(self, name): - s_common.deprecated('reqConfValu()') - return self.req(name) + def reqConfValu(self, key): # pragma: no cover + ''' + Deprecated. Use ``req(key)`` API instead. + ''' + s_common.deprecated('Config.reqConfValu(), use req() instead.') + return self.req(key) - def req(self, name): + def req(self, key): ''' - Return a configuration value or raise NeedConfValu if it is unset. + Get a configuration value. If that value is not present in the schema or is not set, then raise an exception. + + Args: + key (str): The key to require. + + Returns: + The requested value. ''' - valu = self.conf.get(name, s_common.novalu) + if key not in self.json_schema.get('properties', {}): + raise s_exc.BadArg(mesg=f'The {key} configuration option is not present in the configuration schema.', + name=key) + + valu = self.conf.get(key, s_common.novalu) if valu is not s_common.novalu: return valu - mesg = f'The {name} configuration option is required.' - raise s_exc.NeedConfValu(mesg=mesg, name=name) + raise s_exc.NeedConfValu(mesg=f'The {key} configuration option is required.', name=key) def reqKeyValid(self, key, value): ''' diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index c37bfe6f621..50b6a7a921e 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -14,7 +14,6 @@ import synapse.lib.cell as s_cell import synapse.tools.aha.list as s_a_list -import synapse.tools.backup as s_tools_backup import synapse.tools.aha.enroll as s_tools_enroll import synapse.tools.aha.provision.user as s_tools_provision_user @@ -46,6 +45,8 @@ class AhaTest(s_test.SynTest): async def test_lib_aha_clone(self): + zoinks = 'zoinks.aha.loop.vertex.link' + with self.getTestDir() as dirn: dir0 = s_common.gendir(dirn, 'aha0') @@ -58,7 +59,7 @@ async def test_lib_aha_clone(self): self.len(ahacount, await proxy0.getAhaUrls()) self.len(ahacount, await proxy0.getAhaServers()) - purl = await proxy0.addAhaClone('zoinks.aha.loop.vertex.link') + purl = await proxy0.addAhaClone(zoinks) conf1 = {'clone': purl} async with self.getTestAha(conf=conf1, dirn=dir1) as aha1: @@ -113,6 +114,21 @@ async def test_lib_aha_clone(self): self.false(aha0.isactive) self.true(aha1.isactive) + # Remove 00.aha.loop.vertex.link since we're done with him + coverage + async with self.getTestAha(conf={'dns:name': zoinks}, dirn=dir1) as aha1: + async with aha1.getLocalProxy() as proxy1: + srvs = await proxy1.getAhaServers() + self.len(2, srvs) + aha00 = [info for info in srvs if info.get('host') == '00.aha.loop.vertex.link'][0] + data = await proxy1.delAhaServer(aha00.get('host'), aha00.get('port')) + self.eq(data.get('host'), aha00.get('host')) + self.eq(data.get('port'), aha00.get('port')) + + srvs = await proxy1.getAhaServers() + self.len(1, srvs) + urls = await proxy1.getAhaUrls() + self.len(1, urls) + async def test_lib_aha_offon(self): with self.getTestDir() as dirn: cryo0_dirn = s_common.gendir(dirn, 'cryo0') @@ -348,6 +364,13 @@ async def test_lib_aha_basics(self): await cell.ahaclient.proxy() self.len(ahacount + 1, cell.conf.get('aha:registry')) + self.nn(await aha.delAhaServer('zoinks.aha.loop.vertex.link', 27492)) + self.len(ahacount, await aha.getAhaServers()) + + async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell: + await cell.ahaclient.proxy() + self.len(ahacount, cell.conf.get('aha:registry')) + async def test_lib_aha_loadenv(self): with self.getTestDir() as dirn: @@ -775,14 +798,18 @@ async def test_lib_aha_provision(self): # We can generate urls and then drop them en-mass. They will not usable. provurls = [] enrlursl = [] + clonurls = [] async with aha.getLocalProxy() as proxy: provurls.append(await proxy.addAhaSvcProv('00.cell')) provurls.append(await proxy.addAhaSvcProv('01.cell', {'mirror': 'cell'})) enrlursl.append(await proxy.addAhaUserEnroll('bob')) enrlursl.append(await proxy.addAhaUserEnroll('alice')) + clonurls.append(await proxy.addAhaClone('hehe.haha.com')) + clonurls.append(await proxy.addAhaClone('wow.haha.com', port='12345')) await proxy.clearAhaSvcProvs() await proxy.clearAhaUserEnrolls() + await proxy.clearAhaClones() for url in provurls: with self.raises(s_exc.NoSuchName) as cm: @@ -792,6 +819,10 @@ async def test_lib_aha_provision(self): with self.raises(s_exc.NoSuchName) as cm: async with await s_telepath.openurl(url) as prox: self.fail(f'Connected to an expired enrollment URL {url}') # pragma: no cover + for url in clonurls: + with self.raises(s_exc.NoSuchName) as cm: + async with await s_telepath.openurl(url) as prox: + self.fail(f'Connected to an expired clone URL {url}') # pragma: no cover async def test_aha_httpapi(self): @@ -1197,3 +1228,63 @@ async def test_aha_provision_longname(self): with self.raises(s_exc.CryptoErr) as errcm: await s_aha.AhaCell.anit(aha00dirn, conf=aconf) self.isin('Certificate name values must be between 1-64 characters', errcm.exception.get('mesg')) + + async def test_aha_prov_with_user(self): + + async with self.getTestAha() as aha: + async with await s_base.Base.anit() as base: + with self.getTestDir() as dirn: + user = 'synuser' + dirn00 = s_common.genpath(dirn, 'cell00') + dirn01 = s_common.genpath(dirn, 'cell01') + + axon00 = await base.enter_context(self.addSvcToAha(aha, '00.axon', s_axon.Axon, dirn=dirn00, + provinfo={'conf': {'aha:user': user}})) + self.eq(axon00.conf.get('aha:user'), user) + core00 = await base.enter_context(self.addSvcToAha(aha, '00.core', s_cortex.Cortex, dirn=dirn01, + conf={'axon': 'aha://axon...'}, + provinfo={'conf': {'aha:user': user}})) + self.eq(core00.conf.get('aha:user'), user) + # Svc to svc connections use the hinted aha:user value + prox = self.nn(await asyncio.wait_for(core00.axon.proxy(), timeout=12)) + unfo = await prox.getCellUser() + self.eq(unfo.get('name'), user) + + async def test_aha_cell_with_tcp(self): + # It's an older code, sir, but it checks out. + # This should be removed in Synapse v3.0.0 + + with self.getTestDir() as dirn: + ahadir = s_common.gendir(dirn, 'aha') + clldir = s_common.gendir(dirn, 'cell') + ahaconf = { + 'aha:name': '00.aha', + 'aha:network': 'loop.vertex.link', + 'dmon:listen': 'tcp://127.0.0.1:0/', + 'auth:passwd': 'secret', + } + async with await s_aha.AhaCell.anit(dirn=ahadir, conf=ahaconf) as aha: + urls = await aha.getAhaUrls() + self.len(1, urls) + self.true(urls[0].startswith('ssl://')) + ahaurl = f'tcp://root:secret@127.0.0.1:{aha.sockaddr[1]}/' + cllconf = { + 'aha:name': '00.cell', + 'aha:network': 'loop.vertex.link', + 'aha:registry': ahaurl, + 'dmon:listen': None, + } + async with await s_cell.Cell.anit(dirn=clldir, conf=cllconf) as cell: + self.none(await cell.ahaclient.waitready(timeout=12)) + self.eq(cell.conf.get('aha:registry'), ahaurl) + + prox = await cell.ahaclient.proxy() + await prox.fini() + self.false(cell.ahaclient._t_ready.is_set()) + + self.none(await cell.ahaclient.waitready(timeout=12)) + + # No change when restarting + async with await s_cell.Cell.anit(dirn=clldir, conf=cllconf) as cell: + self.none(await cell.ahaclient.waitready(timeout=12)) + self.eq(cell.conf.get('aha:registry'), ahaurl) diff --git a/synapse/tests/test_lib_config.py b/synapse/tests/test_lib_config.py index f04de15b829..bc2972a8352 100644 --- a/synapse/tests/test_lib_config.py +++ b/synapse/tests/test_lib_config.py @@ -164,7 +164,8 @@ async def test_config_basics(self): # We can ensure that certain vars are loaded self.eq('Funky string time!', conf.req('key:string')) # And throw if they are not, or if the requested key isn't even schema valid - self.raises(s_exc.NeedConfValu, conf.req, 'key:bool:nodefval') + self.raises(s_exc.NeedConfValu, conf.reqConfValu, 'key:bool:nodefval') + self.raises(s_exc.BadArg, conf.reqConfValu, 'key:newp') # Since we're an Mutable mapping, we have some dict methods available to us self.len(8, conf) # __len__ diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py index e3ca70d4ba0..8938a276a38 100644 --- a/synapse/tests/test_tools_aha.py +++ b/synapse/tests/test_tools_aha.py @@ -96,10 +96,17 @@ async def test_aha_enroll(self): argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link'] retn, outp = await self.execToolMain(s_a_clone.main, argv) + self.eq(retn, 0) self.isin('one-time use URL:', str(outp)) + argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link', '--only-url'] + retn, outp = await self.execToolMain(s_a_clone.main, argv) + self.eq(retn, 0) + self.notin('one-time use URL:', str(outp)) + argv = ['--url', 'newp://1.2.3.4', '01.aha.loop.vertex.link'] retn, outp = await self.execToolMain(s_a_clone.main, argv) + self.eq(retn, 1) self.isin('ERROR: Invalid URL scheme: newp', str(outp)) argv = ['--url', aha.getLocalUrl(), 'visi'] diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py index 4cb1881fe52..4608cdd884f 100644 --- a/synapse/tests/utils.py +++ b/synapse/tests/utils.py @@ -1429,8 +1429,16 @@ async def getTestCoreProxSvc(self, ssvc, ssvc_conf=None, core_conf=None): yield core, prox, testsvc @contextlib.contextmanager - def mayTestDir(self, dirn): + def mayTestDir(self, dirn: str | None) -> contextlib.AbstractContextManager[str, None, None]: + ''' + Convenience method to make a temporary directory or use an existing one. + + Args: + dirn: Directory to use, or None. + Returns: + The directory to use. + ''' if dirn is not None: yield dirn return @@ -1559,7 +1567,7 @@ def getTestProxy(self, dmon, name, **kwargs): return s_telepath.openurl(f'tcp:///{name}', **kwargs) @contextlib.contextmanager - def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None): + def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None) -> contextlib.AbstractContextManager[str, None, None]: ''' Get a temporary directory for test purposes. This destroys the directory afterwards. From 7a91debd2eecbfe0a585d663a1f131ee1c5e4fa3 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 30 Jul 2024 15:33:20 -0400 Subject: [PATCH 49/86] wip --- synapse/lib/base.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/synapse/lib/base.py b/synapse/lib/base.py index 8040fe60c75..1fc4a19d13b 100644 --- a/synapse/lib/base.py +++ b/synapse/lib/base.py @@ -579,7 +579,7 @@ def sigint(): loop.add_signal_handler(signal.SIGINT, sigint) loop.add_signal_handler(signal.SIGTERM, sigterm) - async def main(self): + async def main(self): # pragma: no cover ''' Helper function to setup signal handlers for this base as the main object. ( use base.waitfini() to block ) @@ -791,9 +791,6 @@ async def genrtask(base): await q.put((False, None)) - except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only - raise - except Exception: if not base.isfini: await q.put((False, None)) From 728da2856f1c6556f2c13283635bfdd6ec14b455 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 30 Jul 2024 15:46:30 -0400 Subject: [PATCH 50/86] wip --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ca8091c6cdb..fc0e859976e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -603,7 +603,6 @@ workflows: branches: only: - master - - visi-aha-defnet - build_docker_tag: requires: From 37983b2437103c9f0062bc26dfcf28a091848b66 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 30 Jul 2024 15:47:58 -0400 Subject: [PATCH 51/86] wip --- synapse/lib/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/lib/base.py b/synapse/lib/base.py index 1fc4a19d13b..f67b73137c7 100644 --- a/synapse/lib/base.py +++ b/synapse/lib/base.py @@ -408,8 +408,6 @@ async def fini(self): for fini in self._fini_funcs: try: await s_coro.ornot(fini) - except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only - raise except Exception: logger.exception(f'{self} - fini function failed: {fini}') @@ -685,7 +683,9 @@ async def __aenter__(self): async def __aexit__(self, exc, cls, tb): if exc is None: - if await self.wait() is None: + if await self.wait() is None: # pragma: no cover + # these lines are 100% covered by the tests but + # the coverage plugin cannot seem to see them... events = ','.join(self.names) mesg = f'timeout waiting for {self.count} event(s): {events}' raise s_exc.TimeOut(mesg=mesg) From 8d0793030d3a32bfaeac6fa47523d6af04925e3e Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 31 Jul 2024 00:17:41 +0000 Subject: [PATCH 52/86] Remove logger statement about username hinting on aha lookup --- synapse/lib/aha.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 5a87a036c34..1bf7cdf49dd 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -153,7 +153,6 @@ async def getAhaSvc(self, name, filters=None): username = self.user.name.split('@')[0] if svcinfo.get('svcinfo'): - logger.info(f'Hinting {self.user.name} for {name}') svcinfo['svcinfo']['urlinfo']['user'] = username return svcinfo From 891fc225cd373a94f998ec695de3857756a65af4 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 31 Jul 2024 00:18:16 +0000 Subject: [PATCH 53/86] Restore ahaname logging during promote/handoff API calls when it is available. --- synapse/lib/cell.py | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 93f04a57456..7e2c60450df 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1965,37 +1965,39 @@ async def promote(self, graceful=False): mesg = 'promote() called on non-mirror' raise s_exc.BadConfValu(mesg=mesg) - logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}.') + _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else '' + logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}{_dispname}.') if graceful: myurl = self.getMyUrl() - logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff.') + logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff{_dispname}.') async with await s_telepath.openurl(mirurl) as lead: await lead.handoff(myurl) - logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}') + logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}{_dispname}') return - logger.debug(f'PROMOTION: Clearing mirror configuration.') + logger.debug(f'PROMOTION: Clearing mirror configuration{_dispname}.') self.modCellConf({'mirror': None}) - logger.debug(f'PROMOTION: Promoting the nexus root.') + logger.debug(f'PROMOTION: Promoting the nexus root{_dispname}.') await self.nexsroot.promote() - logger.debug(f'PROMOTION: Setting the cell as active.') + logger.debug(f'PROMOTION: Setting the cell as active{_dispname}.') await self.setCellActive(True) - logger.warning(f'PROMOTION: Finished leadership promotion!') + logger.warning(f'PROMOTION: Finished leadership promotion{_dispname}.') async def handoff(self, turl, timeout=30): ''' Hand off leadership to a mirror in a transactional fashion. ''' - logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}.') + _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else '' + logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.') async with await s_telepath.openurl(turl) as cell: - logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)}.') + logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.') if self.iden != await cell.getCellIden(): # pragma: no cover mesg = 'Mirror handoff remote cell iden does not match!' @@ -2005,34 +2007,34 @@ async def handoff(self, turl, timeout=30): mesg = 'Cannot handoff mirror leadership to myself!' raise s_exc.BadArg(mesg=mesg) - logger.debug(f'HANDOFF: Obtaining nexus lock.') + logger.debug(f'HANDOFF: Obtaining nexus lock{_dispname}.') async with self.nexslock: - logger.debug(f'HANDOFF: Obtained nexus lock.') + logger.debug(f'HANDOFF: Obtained nexus lock{_dispname}.') indx = await self.getNexsIndx() - logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}.') + logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}{_dispname}.') if not await cell.waitNexsOffs(indx - 1, timeout=timeout): # pragma: no cover mndx = await cell.getNexsIndx() mesg = f'Remote mirror did not catch up in time: {mndx}/{indx}.' raise s_exc.NotReady(mesg=mesg) - logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion.') + logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion{_dispname}.') await cell.promote() - logger.debug(f'HANDOFF: Setting the service as inactive.') + logger.debug(f'HANDOFF: Setting the service as inactive{_dispname}.') await self.setCellActive(False) - logger.debug(f'HANDOFF: Configuring service to sync from new leader.') + logger.debug(f'HANDOFF: Configuring service to sync from new leader{_dispname}.') self.modCellConf({'mirror': turl}) - logger.debug(f'HANDOFF: Restarting the nexus.') + logger.debug(f'HANDOFF: Restarting the nexus{_dispname}.') await self.nexsroot.startup() - logger.debug(f'HANDOFF: Released nexus lock.') + logger.debug(f'HANDOFF: Released nexus lock{_dispname}.') - logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}.') + logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}{_dispname}.') async def reqAhaProxy(self, timeout=None): if self.ahaclient is None: From a2d5d0b507a39fcd43935e1023baf7a05f1eb7f8 Mon Sep 17 00:00:00 2001 From: epiphyte Date: Wed, 31 Jul 2024 00:25:23 +0000 Subject: [PATCH 54/86] Tweak flat network docs slightly --- docs/synapse/deploymentguide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index 70a703a4f10..e6fd5a0f358 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -84,7 +84,7 @@ using AHA, the only host that needs DNS or other external name resolution is the or docker/kubernetes port mapping, you will need to use ``ssl://`` based URIs and specify ``hostname``, ``certname``, and ``ca`` parameters which match the service's AHA registration info. You will also need to specify a specific ``--dmon-port`` option when using the ``synapse.tools.aha.provision.service`` - command to provision the service. + command to provision the services with static ports that you can provide mappings too. Choose an AHA Network Name -------------------------- From 252599d77a48a8f5de5ad70d6b67d983250d7efb Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 31 Jul 2024 11:28:00 -0400 Subject: [PATCH 55/86] changelog entries --- changes/06bd1eabe999e90b2b3301ee041f7017.yaml | 5 +++++ changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml | 5 +++++ changes/6ebc22454e67c26ce57ea4533441c9fc.yaml | 5 +++++ changes/b652abf9758a82bb616f0037740c35e7.yaml | 5 +++++ changes/e3b7e45e107d35ac377e867f1a8ac725.yaml | 5 +++++ 5 files changed, 25 insertions(+) create mode 100644 changes/06bd1eabe999e90b2b3301ee041f7017.yaml create mode 100644 changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml create mode 100644 changes/6ebc22454e67c26ce57ea4533441c9fc.yaml create mode 100644 changes/b652abf9758a82bb616f0037740c35e7.yaml create mode 100644 changes/e3b7e45e107d35ac377e867f1a8ac725.yaml diff --git a/changes/06bd1eabe999e90b2b3301ee041f7017.yaml b/changes/06bd1eabe999e90b2b3301ee041f7017.yaml new file mode 100644 index 00000000000..6ac21a8e68a --- /dev/null +++ b/changes/06bd1eabe999e90b2b3301ee041f7017.yaml @@ -0,0 +1,5 @@ +--- +desc: Update deployment guide to include optional steps to deploy AHA mirrors. +prs: [] +type: doc +... diff --git a/changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml b/changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml new file mode 100644 index 00000000000..bd5d105ba76 --- /dev/null +++ b/changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml @@ -0,0 +1,5 @@ +--- +desc: Updated service base class to retrieve updated AHA servers on startup. +prs: [] +type: feat +... diff --git a/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml b/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml new file mode 100644 index 00000000000..826209b9c79 --- /dev/null +++ b/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml @@ -0,0 +1,5 @@ +--- +desc: Added ``synapse.tools.aha.clone`` command to make it easy to boostrap AHA mirrors. +prs: [] +type: feat +... diff --git a/changes/b652abf9758a82bb616f0037740c35e7.yaml b/changes/b652abf9758a82bb616f0037740c35e7.yaml new file mode 100644 index 00000000000..2340aa113eb --- /dev/null +++ b/changes/b652abf9758a82bb616f0037740c35e7.yaml @@ -0,0 +1,5 @@ +--- +desc: Added support for dynamically registered AHA mirrors. +prs: [] +type: feat +... diff --git a/changes/e3b7e45e107d35ac377e867f1a8ac725.yaml b/changes/e3b7e45e107d35ac377e867f1a8ac725.yaml new file mode 100644 index 00000000000..be05f4352cf --- /dev/null +++ b/changes/e3b7e45e107d35ac377e867f1a8ac725.yaml @@ -0,0 +1,5 @@ +--- +desc: Update deployment guide to clarify aha:network selection vs dns:name selection. +prs: [] +type: doc +... From 26c51276aa23aef99d89b6597195250cc5baf0ff Mon Sep 17 00:00:00 2001 From: invisig0th Date: Wed, 31 Jul 2024 11:46:46 -0400 Subject: [PATCH 56/86] Update changes/6ebc22454e67c26ce57ea4533441c9fc.yaml Co-authored-by: Cisphyx --- changes/6ebc22454e67c26ce57ea4533441c9fc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml b/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml index 826209b9c79..04f677f49f9 100644 --- a/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml +++ b/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml @@ -1,5 +1,5 @@ --- -desc: Added ``synapse.tools.aha.clone`` command to make it easy to boostrap AHA mirrors. +desc: Added ``synapse.tools.aha.clone`` command to make it easy to bootstrap AHA mirrors. prs: [] type: feat ... From 55d8394a41b8533dd9be67f8cb6367ef77108c2a Mon Sep 17 00:00:00 2001 From: invisig0th Date: Wed, 31 Jul 2024 12:31:05 -0400 Subject: [PATCH 57/86] Update docs/synapse/deploymentguide.rst Co-authored-by: Cisphyx --- docs/synapse/deploymentguide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index e6fd5a0f358..03b47ac38ea 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -79,7 +79,7 @@ using AHA, the only host that needs DNS or other external name resolution is the The AHA service resolver requires that registered services connect directly using IP addresses which must also be reachable to AHA clients. Using an ``aha://`` telepath URL requires direct routes to the - service via it's AHA facing network address. If you need to provide telepath access from outside the + service via its AHA facing network address. If you need to provide telepath access from outside the Synapse service network via any network address translation (NAT) method such as an inbound TCP proxy or docker/kubernetes port mapping, you will need to use ``ssl://`` based URIs and specify ``hostname``, ``certname``, and ``ca`` parameters which match the service's AHA registration info. You will also need From 7da0c6e6d9cea9fcf01ff28f182766ff5b7d8711 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Wed, 31 Jul 2024 12:31:17 -0400 Subject: [PATCH 58/86] Update docs/synapse/deploymentguide.rst Co-authored-by: Cisphyx --- docs/synapse/deploymentguide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index 03b47ac38ea..8c169de32a1 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -84,7 +84,7 @@ using AHA, the only host that needs DNS or other external name resolution is the or docker/kubernetes port mapping, you will need to use ``ssl://`` based URIs and specify ``hostname``, ``certname``, and ``ca`` parameters which match the service's AHA registration info. You will also need to specify a specific ``--dmon-port`` option when using the ``synapse.tools.aha.provision.service`` - command to provision the services with static ports that you can provide mappings too. + command to provision the services with static ports that you can provide mappings to. Choose an AHA Network Name -------------------------- From e9af9a338c38890c1e832cdc1d7b6db405335b36 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Wed, 31 Jul 2024 12:31:40 -0400 Subject: [PATCH 59/86] Update synapse/lib/cell.py Co-authored-by: Cisphyx --- synapse/lib/cell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 7e2c60450df..fa3ce5b401b 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1688,7 +1688,7 @@ async def onlink(proxy): if newurls and newurls != oldurls: if oldurls[0].startswith('tcp://'): s_common.deprecated('aha:registry: tcp:// client values.') - logger.warning('tcp:// based aha:registry options will be deprecated in Synapse v3.0.0') + logger.warning('tcp:// based aha:registry options are deprecated and will be removed in Synapse v3.0.0') return self.modCellConf({'aha:registry': newurls}) From 502d23d53c6e7149c3e74434b6dafb1b255101d3 Mon Sep 17 00:00:00 2001 From: visi Date: Thu, 1 Aug 2024 07:29:02 -0400 Subject: [PATCH 60/86] wip --- docs/synapse/deploymentguide.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index 8c169de32a1..08d60f411f4 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -175,8 +175,9 @@ For this example, we will assume you chose a DNS name for your primary AHA serve listed above. If so, you can simply replace ``00`` with sequential numbers and repeat this step to deploy however many AHA mirrors you deem appropriate. -NOTE: AHA uses two default ports ETC. The following steps assume you will be running each of your AHA servers -on a different host. The use of ``network_mode; host`` ETC +By default, AHA uses port ``27492`` to listen for RPC connections from other Synapse services and port ``27272`` +for the provisioning listener. The following example steps assume you will be running each AHA server on separate +hosts or in a containerized to avoid port collisions. **Inside the AHA container** From 37a7a8505eced1f5b4489be90e76c4f312cdf5b6 Mon Sep 17 00:00:00 2001 From: invisig0th Date: Fri, 2 Aug 2024 15:38:18 -0400 Subject: [PATCH 61/86] Update docs/synapse/deploymentguide.rst Co-authored-by: Cisphyx --- docs/synapse/deploymentguide.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst index 08d60f411f4..e8a9b9d0ce0 100644 --- a/docs/synapse/deploymentguide.rst +++ b/docs/synapse/deploymentguide.rst @@ -177,7 +177,7 @@ however many AHA mirrors you deem appropriate. By default, AHA uses port ``27492`` to listen for RPC connections from other Synapse services and port ``27272`` for the provisioning listener. The following example steps assume you will be running each AHA server on separate -hosts or in a containerized to avoid port collisions. +hosts or in a containerized deployment to avoid port collisions. **Inside the AHA container** From 221c8237bc21ebbe70a5a696c5a22869aa01bf4c Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 5 Aug 2024 10:05:26 -0400 Subject: [PATCH 62/86] wip --- synapse/lib/aha.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 1bf7cdf49dd..be454b04011 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -1159,19 +1159,8 @@ async def genCaCert(self, network): return cacert - async def _genCaCert(self, network): - - # generate a CA cert if one does not exist - if self.certdir.getCaCertPath(network) is not None: - return - - await self.genCaCert(network) - async def _genHostCert(self, hostname, signas=None): - if signas is not None: - await self._genCaCert(signas) - if self.certdir.getHostCertPath(hostname) is not None: return From 23a946347aa1e41fe041db33e916759fba3b3868 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 7 Aug 2024 11:37:11 -0400 Subject: [PATCH 63/86] wip --- synapse/lib/aha.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index ba81194bb76..95cfac79db1 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -147,6 +147,25 @@ async def runGatherGenr(self, iden, todo, timeout=None): async for item in self.cell.runGatherGenr(iden, todo, timeout=timeout): yield item + async def addNexsTracker(self, name, iden, network=None): + ''' + Add an explicit nexus tracker which consumes the transactions + produced by the iden leader. + ''' + return await self.cell.addNexsTracker(name, iden, network=network) + + async def delNexsTracker(self, name, iden, network=None): + ''' + Remove a nexus tracker entry. + ''' + return await self.cell.delNexsTracker(name, iden, network=network) + + async def getNexsTrackers(self, iden): + # services which have the same iden are all automatically trackers + # explicitly registered trackers are also yielded here + async for item in self.cell.getNexsTrackers(iden): + yield item + async def getAhaSvc(self, name, filters=None): ''' Return an AHA service description dictionary for a service name. From 301813ed56e9b07d9dd2422c95eae1c54c757232 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 20 Aug 2024 09:52:01 -0400 Subject: [PATCH 64/86] wip --- synapse/lib/cell.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 6efd9968447..6ca48e823a1 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -4284,14 +4284,14 @@ async def ahaGatherPs(self, user): retn = [] todo = s_common.todo('ps') - isallowed = await self.isUserAllowed(user.iden, ('task', 'get')) + allowed = user.allowed(('task', 'get')) try: proxy = self.ahaclient.proxy(timeout=4) async for svcfull, (ok, tasks) in proxy.runGatherCall(todo, timeout=4): if ok: for task in tasks: - if isallowed or task['user'] == user.name: + if allowed or task['user'] == user.name: retn.append(task) else: logger.warning(f'AHA gather ps() from {svcfull}: {tasks}') From fb6a92f84587650f1e9dd830405d61c127480c10 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 21 Aug 2024 12:26:38 -0400 Subject: [PATCH 65/86] wip --- synapse/common.py | 25 ++++++++ synapse/daemon.py | 1 + synapse/lib/aha.py | 148 ++++++++++++++++++++++++++++++++------------ synapse/lib/cell.py | 46 ++++++++++++-- synapse/telepath.py | 18 ++++++ 5 files changed, 193 insertions(+), 45 deletions(-) diff --git a/synapse/common.py b/synapse/common.py index acb1c3b81b8..f3c8f97f6d2 100644 --- a/synapse/common.py +++ b/synapse/common.py @@ -1237,6 +1237,31 @@ async def wait_for(fut, timeout): async with _timeout(timeout): return await fut +async def waitretn(futu, timeout): + try: + valu = await wait_for(futu, timeout) + return (True, valu) + except Exception as e: + return (False, excinfo(e)) + +async def waitgenr(genr, timeout): + + genr = genr.__aiter__() + while True: + try: + retn = await waitretn(genr.__anext__(), timeout) + + if not retn[0] and retn[1]['err'] == 'StopAsyncIteration': + return + + yield retn + + if not retn[0]: + return + + finally: + await genr.aclose() + def _release_waiter(waiter, *args): if not waiter.done(): waiter.set_result(None) diff --git a/synapse/daemon.py b/synapse/daemon.py index 822eae06eec..b212f9f4aba 100644 --- a/synapse/daemon.py +++ b/synapse/daemon.py @@ -462,6 +462,7 @@ async def sessfini(): link.set('sess', sess) if isinstance(item, s_telepath.Aware): + reply[1]['features'] = await item.getTeleFeats() item = await s_coro.ornot(item.getTeleApi, link, mesg, path) if isinstance(item, s_base.Base): link.onfini(item) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 95cfac79db1..2580e956897 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -138,13 +138,13 @@ async def getAhaUrls(self, user='root'): return ahaurls @s_cell.adminapi() - async def runGatherCall(self, iden, todo, timeout=None): - async for item in self.cell.runGatherCall(iden, todo, timeout=timeout): + async def runGatherCall(self, iden, todo, timeout=None, skiprun=None): + async for item in self.cell.runGatherCall(iden, todo, timeout=timeout, skiprun=skiprun): yield item @s_cell.adminapi() - async def runGatherGenr(self, iden, todo, timeout=None): - async for item in self.cell.runGatherGenr(iden, todo, timeout=timeout): + async def runGatherGenr(self, iden, todo, timeout=None, skiprun=None): + async for item in self.cell.runGatherGenr(iden, todo, timeout=timeout, skiprun=skiprun): yield item async def addNexsTracker(self, name, iden, network=None): @@ -711,7 +711,25 @@ def _initCellHttpApis(self): self.addHttpApi('/api/v1/aha/services', AhaServicesV1, {'cell': self}) self.addHttpApi('/api/v1/aha/provision/service', AhaProvisionServiceV1, {'cell': self}) - async def getAhaSvcsByIden(self, iden, online=True): + async def callAhaSvcApi(self, name, todo, timeout=None): + name = self._getAhaName(name) + svcdef = await self._getAhaSvc(name) + proxy = await self.getAhaSvcProxy(svcdef) + return await s_common.waitretn(proxy.taskv2(todo), timeout=timeout) + + async def callAhaSvcGenr(self, name, todo, timeout=None): + name = self._getAhaName(name) + svcdef = await self._getAhaSvc(name) + proxy = await self.getAhaSvcProxy(svcdef) + async for item in s_common.waitgenr(proxy.taskv2(todo), timeout=timeout): + yield item + + async def getAhaSvcPeerTasks(self, iden, timeout=None, skiprun=None): + todo = s_common.todo('getTasks') + async for item in self.runGatherGenr(iden, todo, timeout=timeout, skiprun=skiprun): + yield item + + async def getAhaSvcsByIden(self, iden, online=True, skiprun=None): runs = set() async for svcdef in self.getAhaSvcs(): @@ -727,89 +745,95 @@ async def getAhaSvcsByIden(self, iden, online=True): if svcrun in runs: continue + if skiprun == svcrun: + continue + runs.add(svcrun) yield svcdef - @contextlib.asynccontextmanager - async def getAhaSvcProxy(self, svcdef, user='root'): - - svcname = svcdef.get('svcname') + def getAhaSvcUrl(self, svcdef, user='root'): + svcfull = svcdef.get('name') svcnetw = svcdef.get('svcnetw') - svcfull = f'{svcname}.{svcnetw}' - host = svcdef['svcinfo']['urlinfo']['host'] port = svcdef['svcinfo']['urlinfo']['port'] + return f'ssl://{host}:{port}?hostname={svcfull}&certname={user}@{svcnetw}' - svcurl = f'ssl://{host}:{port}?hostname={svcfull}&certname={user}@{svcnetw}' + async def runGatherCall(self, iden, todo, timeout=None, skiprun=None): - async with await s_telepath.openurl(svcurl) as proxy: - yield proxy - - async def runGatherCall(self, iden, todo, timeout=None): + if not self.isactive: + proxy = await self.nexsroot.client.proxy(timeout=timeout) + async for item in proxy.runGatherCall(iden, todo, timeout=timeout, skiprun=skiprun): + yield item queue = asyncio.Queue() async with await s_base.Base.anit() as base: - async def call(svcdef): - svcname = svcdef.get('svcname') - svcnetw = svcdef.get('svcnetw') - svcfull = f'{svcname}.{svcnetw}' + async def call(svcdef): + svcfull = svcdef.get('name') try: - async with self.getAhaSvcProxy(svcdef) as proxy: - valu = await asyncio.wait_for(proxy.taskv2(todo), timeout=timeout) - await queue.put((svcfull, (True, valu))) + proxy = await self.reqAhaSvcProxy(svcdef) + retn = await s_common.waitretn(proxy.taskv2(todo), timeout=timeout) + await queue.put((svcfull, retn)) except Exception as e: await queue.put((svcfull, (False, s_common.excinfo(e)))) count = 0 - async for svcdef in self.getAhaSvcsByIden(iden): + async for svcdef in self.getAhaSvcsByIden(iden, skiprun=skiprun): count += 1 - self.base(call(svcdef)) + base.schedCoro(call(svcdef)) for i in range(count): yield await queue.get() - async def runGatherGenr(self, iden, todo, timeout=None): + async def runGatherGenr(self, iden, todo, timeout=None, skiprun=None): - queue = asyncio.Queue() + if not self.isactive: + proxy = await self.nexsroot.client.proxy(timeout=timeout) + async for item in proxy.runGatherGenr(iden, todo, timeout=timeout, skiprun=skiprun): + yield item + queue = asyncio.Queue() async with await s_base.Base.anit() as base: async def call(svcdef): - - svcname = svcdef.get('svcname') - svcnetw = svcdef.get('svcnetw') - svcfull = f'{svcname}.{svcnetw}' - + svcfull = svcdef.get('name') try: - async with self.getAhaSvcProxy(svcdef) as proxy: - async for item in await proxy.taskv2(todo): - await queue.put((svcfull, (True, item))) + proxy = await self.getAhaSvcProxy(svcdef) - except Exception as e: - await queue.put((svcfull, (False, s_common.excinfo(e)))) + genr = await proxy.taskv2(todo) + async for item in s_common.waitgenr(genr, timeout=timeout): + await queue.put((svcfull, item)) finally: await queue.put((svcfull, None)) count = 0 - async for svcdef in self.getAhaSvcsByIden(iden): + async for svcdef in self.getAhaSvcsByIden(iden, skiprun=skiprun): count += 1 base.schedCoro(call(svcdef)) while count > 0: - item = await asyncio.wait_for(queue.get(), timeout=timeout) + item = await queue.get() if item[1] is None: count -= 1 - continue - yield item + yield item[1] + + async def _finiSvcClients(self): + for client in list(self.clients.values()): + await client.fini() + + async def initServicePassive(self): + await self._finiSvcClients() async def initServiceRuntime(self): + self.clients = {} + self.onfini(self._finiSvcClients) + self.addActiveCoro(self._clearInactiveSessions) if self.isactive: @@ -1022,6 +1046,44 @@ async def addAhaSvc(self, name, info, network=None): await self.fire('aha:svcadd', svcinfo=svcinfo) await self.fire(f'aha:svcadd:{svcfull}', svcinfo=svcinfo) + async def reqAhaSvcProxy(self, svcdef, timeout=None): + + proxy = await self.getAhaSvcProxy(svcdef, timeout=timeout) + if proxy is not None: + return proxy + + mesg = f'The service is not ready {svcfull}.' + raise s_exc.NotReady(mesg=mesg) + + async def getAhaSvcProxy(self, svcdef, timeout=None): + + assert self.isactive + client = await self.getAhaSvcClient(svcdef) + if client is None: + return None + + return await client.proxy(timeout=timeout) + + async def getAhaSvcClient(self, svcdef): + + assert self.isactive + + svcfull = svcdef.get('name') + + client = self.clients.get(svcfull) + if client is not None: + return client + + svcurl = self.getAhaSvcUrl(svcdef) + print(f'AHA SVC CLIENT {svcfull} {svcurl}') + + client = self.clients[svcfull] = await s_telepath.ClientV2.anit(svcurl) + async def fini(): + self.clients.pop(svcfull, None) + + client.onfini(fini) + return client + def _getAhaName(self, name): # the modern version of names is absolute or ... if name.endswith('...'): @@ -1191,6 +1253,10 @@ async def _setAhaSvcDown(self, name, linkiden, network=None): logger.info(f'Set [{svcfull}] offline.', extra=await self.getLogExtra(name=svcname, netw=svcnetw)) + client = self.clients.pop(svcfull, None) + if client is not None: + await client.fini() + async def getAhaSvc(self, name, filters=None): name = self._getAhaName(name) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 6ca48e823a1..e1259836bbd 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -425,6 +425,17 @@ async def ps(self): async def kill(self, iden): return await self.cell.kill(self.user, iden) + @adminapi() + async def getTasks(self): + async for task in self.cell.getTasks(): + yield task + + async def getFeatures(self): + return await self.cell.getFeatures() + + async def hasFeature(self, name): + return self.cell.hasFeature(name) + @adminapi(log=True) async def behold(self): ''' @@ -1115,6 +1126,10 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): self.netready = asyncio.Event() self.conf = self._initCellConf(conf) + self.features = { + 'tellready': True, + 'dynmirror': True, + } self.minfree = self.conf.get('limit:disk:free') if self.minfree is not None: @@ -4276,6 +4291,29 @@ async def ps(self, user): return retn + async def getPeerTasks(self, timeout=None): + + for task in self.boss.ps(): + yield task + + if self.ahaclient: + + proxy = await self.ahaclient.proxy(timeout=timeout) + if proxy is None: + logger.warning('AHA client connection timed out for getPeerTasks()') + return + + if not proxy._hasTeleMeth('getAhaSvcTasks'): + logger.warning('AHA server does not implement getAhaSvcTasks(). Please update.') + return + + async for (ok, valu) in proxy.getAhaPeerTasks(self.iden, timeout=5, skiprun=self.runid): + if ok: yield valu + + async def getTasks(self): + for task in self.boss.ps(): + yield task + async def ahaGatherPs(self, user): if self.ahaclient is None: @@ -4368,13 +4406,13 @@ async def getCellInfo(self): 'https': self.https_listeners, } }, - 'features': { - 'tellready': True, - 'dynmirror': True, - }, + 'features': self.features, } return ret + async def getTeleFeats(self): + return dict(self.features) + async def getSystemInfo(self): ''' Get info about the system in which the cell is running diff --git a/synapse/telepath.py b/synapse/telepath.py index ea9efe107fe..bfc8dd1ccc2 100644 --- a/synapse/telepath.py +++ b/synapse/telepath.py @@ -321,6 +321,9 @@ async def getTeleApi(self, link, mesg, path): ''' return self + async def getTeleFeats(self): + return {} + def onTeleShare(self, dmon, name): pass @@ -599,6 +602,8 @@ async def __anit__(self, link, name): self.shares = {} self._ahainfo = {} + self._features = {} + self.sharinfo = {} self.methinfo = {} @@ -634,6 +639,18 @@ async def fini(): self.onfini(fini) self.link.onfini(self.fini) + def _hasTeleMeth(self, name): + return self.methinfo.get(name) is not None + + def _hasTeleGenr(self, name): + info = self.methinfo.get(name) + if info is None: + return False + return info.get('genr', False) + + def _hasTeleFeat(self, name): + return self.features.get(name, False) + def _getSynVers(self): ''' Helper method to retrieve the remote Synapse version from Proxy. @@ -906,6 +923,7 @@ async def handshake(self, auth=None): self.sess = self.synack[1].get('sess') self._ahainfo = self.synack[1].get('ahainfo', {}) + self.features = self.synack[1].get('features', {}) self.sharinfo = self.synack[1].get('sharinfo', {}) self.methinfo = self.sharinfo.get('meths', {}) From e6248eb0835be6312091a5ea6e7595d75d13a87c Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 23 Aug 2024 12:35:41 -0400 Subject: [PATCH 66/86] wip --- synapse/common.py | 9 +++---- synapse/lib/aha.py | 47 +++++++++++++++++++---------------- synapse/lib/cell.py | 12 ++++++--- synapse/tests/test_lib_aha.py | 2 ++ 4 files changed, 40 insertions(+), 30 deletions(-) diff --git a/synapse/common.py b/synapse/common.py index f3c8f97f6d2..e16827dba11 100644 --- a/synapse/common.py +++ b/synapse/common.py @@ -1247,8 +1247,8 @@ async def waitretn(futu, timeout): async def waitgenr(genr, timeout): genr = genr.__aiter__() - while True: - try: + try: + while True: retn = await waitretn(genr.__anext__(), timeout) if not retn[0] and retn[1]['err'] == 'StopAsyncIteration': @@ -1258,9 +1258,8 @@ async def waitgenr(genr, timeout): if not retn[0]: return - - finally: - await genr.aclose() + finally: + await genr.aclose() def _release_waiter(waiter, *args): if not waiter.done(): diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 2580e956897..d61c657051c 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -714,15 +714,30 @@ def _initCellHttpApis(self): async def callAhaSvcApi(self, name, todo, timeout=None): name = self._getAhaName(name) svcdef = await self._getAhaSvc(name) - proxy = await self.getAhaSvcProxy(svcdef) - return await s_common.waitretn(proxy.taskv2(todo), timeout=timeout) + return self._callAhaSvcApi(svcdef, todo, timeout=timeout) + + async def _callAhaSvcApi(self, svcdef, todo, timeout=None): + try: + proxy = await self.getAhaSvcProxy(svcdef, timeout=timeout) + meth = getattr(proxy, todo[0]) + return await s_common.waitretn(meth(*todo[1], **todo[2]), timeout=timeout) + except Exception as e: + # in case proxy construction fails + return (False, s_common.excinfo(e)) async def callAhaSvcGenr(self, name, todo, timeout=None): name = self._getAhaName(name) svcdef = await self._getAhaSvc(name) - proxy = await self.getAhaSvcProxy(svcdef) - async for item in s_common.waitgenr(proxy.taskv2(todo), timeout=timeout): - yield item + + async def _callAhaSvcGenr(self, svcdef, todo, timeout=None): + try: + proxy = await self.getAhaSvcProxy(svcdef, timeout=timeout) + meth = getattr(proxy, todo[0]) + async for item in s_common.waitgenr(meth(*todo[1], **todo[2]), timeout=timeout): + yield item + except Exception as e: + # in case proxy construction fails + yield (False, s_common.excinfo(e)) async def getAhaSvcPeerTasks(self, iden, timeout=None, skiprun=None): todo = s_common.todo('getTasks') @@ -769,15 +784,8 @@ async def runGatherCall(self, iden, todo, timeout=None, skiprun=None): async with await s_base.Base.anit() as base: async def call(svcdef): - svcfull = svcdef.get('name') - try: - proxy = await self.reqAhaSvcProxy(svcdef) - retn = await s_common.waitretn(proxy.taskv2(todo), timeout=timeout) - await queue.put((svcfull, retn)) - - except Exception as e: - await queue.put((svcfull, (False, s_common.excinfo(e)))) + await queue.put((svcfull, await self._callAhaSvcApi(svcdef, todo, timeout=timeout))) count = 0 async for svcdef in self.getAhaSvcsByIden(iden, skiprun=skiprun): @@ -800,14 +808,10 @@ async def runGatherGenr(self, iden, todo, timeout=None, skiprun=None): async def call(svcdef): svcfull = svcdef.get('name') try: - proxy = await self.getAhaSvcProxy(svcdef) - - genr = await proxy.taskv2(todo) - async for item in s_common.waitgenr(genr, timeout=timeout): + async for item in self._callAhaSvcGenr(svcdef, todo, timeout=timeout): await queue.put((svcfull, item)) - finally: - await queue.put((svcfull, None)) + await queue.put(None) count = 0 async for svcdef in self.getAhaSvcsByIden(iden, skiprun=skiprun): @@ -817,10 +821,11 @@ async def call(svcdef): while count > 0: item = await queue.get() - if item[1] is None: + if item is None: count -= 1 + continue - yield item[1] + yield item async def _finiSvcClients(self): for client in list(self.clients.values()): diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 7feea66d089..a03871f0123 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -4414,7 +4414,7 @@ async def ps(self, user): async def getPeerTasks(self, timeout=None): for task in self.boss.ps(): - yield task + yield task.pack() if self.ahaclient: @@ -4427,12 +4427,16 @@ async def getPeerTasks(self, timeout=None): logger.warning('AHA server does not implement getAhaSvcTasks(). Please update.') return - async for (ok, valu) in proxy.getAhaPeerTasks(self.iden, timeout=5, skiprun=self.runid): - if ok: yield valu + async for (name, (ok, valu)) in proxy.getAhaPeerTasks(self.iden, timeout=5, skiprun=self.runid): + if not ok: + logger.warning(f'getPeerTasks peer error: {name} {valu}') + await asyncio.sleep(0) + valu['aha:name'] = name + yield valu async def getTasks(self): for task in self.boss.ps(): - yield task + yield task.pack() async def ahaGatherPs(self, user): diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index a6430cca6d0..b97e361ed24 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1323,6 +1323,8 @@ async def test_aha_gather(self): self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) # test the genr endpoint + print(repr(nexsindx)) + reals = [item async for item in cell00.getNexusChanges(0, wait=False)] todo = s_common.todo('getNexusChanges', 0, wait=False) items = [item async for item in aha.runGatherGenr(cell00.iden, todo, timeout=3) if item[1]] self.len(nexsindx * 2, items) From 2af006b8b21c043cf785e95c1ea6947af856a855 Mon Sep 17 00:00:00 2001 From: visi Date: Mon, 26 Aug 2024 11:37:38 -0400 Subject: [PATCH 67/86] wip --- synapse/lib/aha.py | 5 ++++- synapse/lib/cell.py | 13 +++++++----- synapse/tests/test_lib_aha.py | 37 ++++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 3439a84424b..a238db32232 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -396,6 +396,10 @@ async def clearAhaClones(self): ''' return await self.cell.clearAhaClones() + @s_cell.adminapi() + async def getAhaSvcPeerTasks(self, iden, timeout=None, skiprun=None): + async for task in self.cell.getAhaSvcPeerTasks(iden, timeout=timeout, skiprun=skiprun): + yield task class ProvDmon(s_daemon.Daemon): @@ -1081,7 +1085,6 @@ async def getAhaSvcClient(self, svcdef): return client svcurl = self.getAhaSvcUrl(svcdef) - print(f'AHA SVC CLIENT {svcfull} {svcurl}') client = self.clients[svcfull] = await s_telepath.ClientV2.anit(svcurl) async def fini(): diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 94447e23c32..b913f4ec24b 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -4420,20 +4420,23 @@ async def ps(self, user): async def getPeerTasks(self, timeout=None): for task in self.boss.ps(): - yield task.pack() + info = task.pack() + if self.ahasvcname is not None: + info['aha:name'] = self.ahasvcname + yield info - if self.ahaclient: + if self.ahaclient is not None: proxy = await self.ahaclient.proxy(timeout=timeout) if proxy is None: logger.warning('AHA client connection timed out for getPeerTasks()') return - if not proxy._hasTeleMeth('getAhaSvcTasks'): - logger.warning('AHA server does not implement getAhaSvcTasks(). Please update.') + if not proxy._hasTeleMeth('getAhaSvcPeerTasks'): + logger.warning('AHA server does not implement getAhaSvcPeerTasks(). Please update.') return - async for (name, (ok, valu)) in proxy.getAhaPeerTasks(self.iden, timeout=5, skiprun=self.runid): + async for (name, (ok, valu)) in proxy.getAhaSvcPeerTasks(self.iden, timeout=timeout, skiprun=self.runid): if not ok: logger.warning(f'getPeerTasks peer error: {name} {valu}') await asyncio.sleep(0) diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index e34e1303cdb..f98e2e70a5e 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1340,7 +1340,6 @@ async def test_aha_gather(self): self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) # test the genr endpoint - print(repr(nexsindx)) reals = [item async for item in cell00.getNexusChanges(0, wait=False)] todo = s_common.todo('getNexusChanges', 0, wait=False) items = [item async for item in aha.runGatherGenr(cell00.iden, todo, timeout=3) if item[1]] @@ -1362,3 +1361,39 @@ async def test_aha_gather(self): items = [item async for item in aha.runGatherGenr(cell00.iden, todo, timeout=3) if item[1]] self.len(nexsindx, items) self.true(all(item[1][0] for item in items)) + + # test some of the gather API implementations... + purl00 = await aha.addAhaSvcProv('0.cell') + purl01 = await aha.addAhaSvcProv('1.cell', provinfo={'mirror': '0.cell'}) + + cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00})) + cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01})) + + await cell01.sync() + + async def sleep99(cell): + await cell.boss.promote('sleep99', cell.auth.rootuser) + await cell00.fire('sleep99') + await asyncio.sleep(99) + + async with cell00.waiter(2, 'sleep99', timeout=2): + task00 = cell00.schedCoro(sleep99(cell00)) + task01 = cell01.schedCoro(sleep99(cell01)) + + proxy = await aha.enter_context(aha.getLocalProxy()) + tasks = [task async for task in proxy.getAhaSvcPeerTasks(cell00.iden, timeout=3)] + tasks.sort() + self.len(2, tasks) + self.eq(tasks[0][0], '0.cell.synapse') + self.true(tasks[0][1][0]) + self.eq(tasks[1][0], '1.cell.synapse') + self.true(tasks[1][1][0]) + + tasks = [task async for task in proxy.getAhaSvcPeerTasks(cell00.iden, timeout=3, skiprun=cell00.runid)] + tasks.sort() + self.len(1, tasks) + self.eq(tasks[0][0], '1.cell.synapse') + + tasks = [task async for task in cell00.getPeerTasks(timeout=3)] + print(repr(tasks)) + self.len(2, tasks) From c9f3db359c20ddea21bcfedcbe817a2f3ff3d7f9 Mon Sep 17 00:00:00 2001 From: visi Date: Thu, 5 Sep 2024 07:42:23 -0400 Subject: [PATCH 68/86] wip --- synapse/tests/test_lib_aha.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 68ba735b27a..0b2dbadb04c 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1396,5 +1396,4 @@ async def sleep99(cell): self.eq(tasks[0][0], '1.cell.synapse') tasks = [task async for task in cell00.getPeerTasks(timeout=3)] - print(repr(tasks)) self.len(2, tasks) From 770a98056620d5d24876f35a776b6b77164257d9 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 10 Sep 2024 15:26:35 -0400 Subject: [PATCH 69/86] wip --- synapse/lib/aha.py | 1 + synapse/lib/cell.py | 129 ++++++++++++++++++++++++---------- synapse/tests/test_lib_aha.py | 10 +++ 3 files changed, 102 insertions(+), 38 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index a238db32232..999018f511b 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -752,6 +752,7 @@ async def getAhaSvcsByIden(self, iden, online=True, skiprun=None): runs = set() async for svcdef in self.getAhaSvcs(): + await asyncio.sleep(0) # TODO services by iden indexes! if svcdef['svcinfo'].get('iden') != iden: diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index b913f4ec24b..6a5e2e6cb2d 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -431,6 +431,14 @@ async def getTasks(self): async for task in self.cell.getTasks(): yield task + @adminapi() + async def getTask(self, iden): + return await self.cell.getTask(iden) + + @adminapi() + async def killTask(self, iden): + return await self.cell.killTask(iden) + async def getFeatures(self): return await self.cell.getFeatures() @@ -4417,60 +4425,105 @@ async def ps(self, user): return retn - async def getPeerTasks(self, timeout=None): + async def getAhaProxy(self, timeout=None, methname=None): - for task in self.boss.ps(): - info = task.pack() - if self.ahasvcname is not None: - info['aha:name'] = self.ahasvcname - yield info + if self.ahaclient is None: + return - if self.ahaclient is not None: + proxy = await self.ahaclient.proxy(timeout=timeout) + if proxy is None: + logger.warning('AHA client connection failed.') + return - proxy = await self.ahaclient.proxy(timeout=timeout) - if proxy is None: - logger.warning('AHA client connection timed out for getPeerTasks()') - return + if methname is not None and not proxy._hasTeleMeth(methname): + logger.warning(f'AHA server does not implement {methname}.') + return - if not proxy._hasTeleMeth('getAhaSvcPeerTasks'): - logger.warning('AHA server does not implement getAhaSvcPeerTasks(). Please update.') - return + return proxy + + async def callPeerApi(self, todo, timeout=None): + + proxy = await self.getAhaProxy(timeout=timeout, methname='runGatherCall') + if proxy is None: + return + + async for item in proxy.runGatherCall(self.iden, todo, timeout=timeout, skiprun=self.runid): + yield item - async for (name, (ok, valu)) in proxy.getAhaSvcPeerTasks(self.iden, timeout=timeout, skiprun=self.runid): - if not ok: - logger.warning(f'getPeerTasks peer error: {name} {valu}') - await asyncio.sleep(0) - valu['aha:name'] = name - yield valu + async def callPeerGenr(self, todo, timeout=None): + + proxy = await self.getAhaProxy(timeout=timeout, methname='runGatherGenr') + if proxy is None: + return + + async for item in proxy.runGatherGenr(self.iden, todo, timeout=timeout, skiprun=self.runid): + yield item async def getTasks(self): for task in self.boss.ps(): yield task.pack() - async def ahaGatherPs(self, user): + async def getTask(self, iden): + task = self.boss.get(iden) + if task is not None: + return task.pack() - if self.ahaclient is None: - return await self.ps() + async def killTask(self, iden): + task = self.boss.get(iden) + if task is None: + return False - retn = [] - todo = s_common.todo('ps') + await task.kill() + return True - allowed = user.allowed(('task', 'get')) + async def getPeerTask(self, iden, timeout=None): + # return a task which may be running on a peer + task = await self.getTask(iden) + if task is not None: + task['aha:name'] = self.ahasvcname + return task - try: - proxy = self.ahaclient.proxy(timeout=4) - async for svcfull, (ok, tasks) in proxy.runGatherCall(todo, timeout=4): - if ok: - for task in tasks: - if allowed or task['user'] == user.name: - retn.append(task) - else: - logger.warning(f'AHA gather ps() from {svcfull}: {tasks}') + todo = s_common.todo('getTask', iden) + async for (name, (ok, task)) in self.callPeerApi(todo, timeout=timeout): + if ok and task is not None: + task['aha:name'] = name + return task - except Exception as e: - logger.warning(f'AHA gather ps() on {self.ahasvcname}: {e}') + async def getPeerTasks(self, timeout=None): + + for task in self.boss.ps(): + info = task.pack() + if self.ahasvcname is not None: + info['aha:name'] = self.ahasvcname + yield info + + todo = s_common.todo('getTasks') + async for (name, (ok, task)) in self.callPeerGenr(todo, timeout=timeout): + + if not ok: + logger.warning(f'getPeerTasks() error from peer {name}: {retn}') + continue + + task['aha:name'] = name + yield task + + async def killPeerTask(self, iden, timeout=None): + + # kill a task which may be running on a peer + if await self.killTask(iden): + return (True, self.ahasvcname) + + todo = s_common.todo('killTask', iden) + async for (name, (ok, done)) in self.callPeerApi(todo, timeout=timeout): + + if not ok: + logger.warning(f'killPeerTask() error from peer {name}: {retn}') + continue + + if ok and done: + return (True, name) - # async def _ahaGatherKill(self, user, iden): + return (False, None) async def kill(self, user, iden): perm = ('task', 'del') diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 0b2dbadb04c..363ded398ad 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1397,3 +1397,13 @@ async def sleep99(cell): tasks = [task async for task in cell00.getPeerTasks(timeout=3)] self.len(2, tasks) + + self.eq(tasks[0], await cell00.getPeerTask(tasks[0].get('iden'))) + self.eq(tasks[1], await cell00.getPeerTask(tasks[1].get('iden'))) + + retn00 = await cell00.killPeerTask(tasks[0].get('iden')) + retn01 = await cell00.killPeerTask(tasks[1].get('iden')) + + self.eq((True, tasks[0]['aha:name']), retn00) + self.eq((True, tasks[1]['aha:name']), retn01) + self.eq((False, None), await cell00.killPeerTask('newp')) From d4f0aa0e89827f1543f09a6791797e16d441bcff Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 27 Nov 2024 11:54:21 -0500 Subject: [PATCH 70/86] wip --- synapse/lib/aha.py | 6 +-- synapse/lib/cell.py | 106 +++++++++++++++++--------------------- synapse/lib/stormtypes.py | 37 +++++++++++++ 3 files changed, 87 insertions(+), 62 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 999018f511b..860830ad5dc 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -397,8 +397,8 @@ async def clearAhaClones(self): return await self.cell.clearAhaClones() @s_cell.adminapi() - async def getAhaSvcPeerTasks(self, iden, timeout=None, skiprun=None): - async for task in self.cell.getAhaSvcPeerTasks(iden, timeout=timeout, skiprun=skiprun): + async def getSvcPeerTasks(self, iden, timeout=None, skiprun=None): + async for task in self.cell.getSvcPeerTasks(iden, timeout=timeout, skiprun=skiprun): yield task class ProvDmon(s_daemon.Daemon): @@ -743,7 +743,7 @@ async def _callAhaSvcGenr(self, svcdef, todo, timeout=None): # in case proxy construction fails yield (False, s_common.excinfo(e)) - async def getAhaSvcPeerTasks(self, iden, timeout=None, skiprun=None): + async def getSvcPeerTasks(self, iden, timeout=None, skiprun=None): todo = s_common.todo('getTasks') async for item in self.runGatherGenr(iden, todo, timeout=timeout, skiprun=skiprun): yield item diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 02979aa9107..a6c5901a3ab 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -436,17 +436,17 @@ async def kill(self, iden): return await self.cell.kill(self.user, iden) @adminapi() - async def getTasks(self): - async for task in self.cell.getTasks(): + async def getTasks(self, peers=True, timeout=None): + async for task in self.cell.getTasks(peers=peers, timeout=timeout): yield task @adminapi() - async def getTask(self, iden): - return await self.cell.getTask(iden) + async def getTask(self, iden, peers=True, timeout=None): + return await self.cell.getTask(iden, peers=peers, timeout=timeout) @adminapi() - async def killTask(self, iden): - return await self.cell.killTask(iden) + async def killTask(self, iden, peers=True, timeout=None): + return await self.cell.killTask(iden, peers=peers, timeout=timeout) async def getFeatures(self): return await self.cell.getFeatures() @@ -4460,7 +4460,9 @@ async def getAhaProxy(self, timeout=None, methname=None): return proxy async def callPeerApi(self, todo, timeout=None): - + ''' + Yield responses from our peers via the AHA gather call API. + ''' proxy = await self.getAhaProxy(timeout=timeout, methname='runGatherCall') if proxy is None: return @@ -4469,7 +4471,9 @@ async def callPeerApi(self, todo, timeout=None): yield item async def callPeerGenr(self, todo, timeout=None): - + ''' + Yield responses from invoking a generator via the AHA gather API. + ''' proxy = await self.getAhaProxy(timeout=timeout, methname='runGatherGenr') if proxy is None: return @@ -4477,71 +4481,55 @@ async def callPeerGenr(self, todo, timeout=None): async for item in proxy.runGatherGenr(self.iden, todo, timeout=timeout, skiprun=self.runid): yield item - async def getTasks(self): - for task in self.boss.ps(): - yield task.pack() + async def getTasks(self, peers=True, timeout=None): - async def getTask(self, iden): - task = self.boss.get(iden) - if task is not None: - return task.pack() - - async def killTask(self, iden): - task = self.boss.get(iden) - if task is None: - return False - - await task.kill() - return True + for task in self.boss.ps(): - async def getPeerTask(self, iden, timeout=None): - # return a task which may be running on a peer - task = await self.getTask(iden) - if task is not None: - task['aha:name'] = self.ahasvcname - return task + item = task.pack() + item['aha:service'] = self.ahasvcname - todo = s_common.todo('getTask', iden) - async for (name, (ok, task)) in self.callPeerApi(todo, timeout=timeout): - if ok and task is not None: - task['aha:name'] = name - return task + yield item - async def getPeerTasks(self, timeout=None): + if not peers: + return - for task in self.boss.ps(): - info = task.pack() - if self.ahasvcname is not None: - info['aha:name'] = self.ahasvcname - yield info + todo = s_common.todo('getTasks', peers=False) + # we can ignore the yielded aha names because we embed it in the task + async for (svcname, (ok, retn)) in self.callPeerGenr(todo, timeout=timeout): + yield retn - todo = s_common.todo('getTasks') - async for (name, (ok, task)) in self.callPeerGenr(todo, timeout=timeout): + async def getTask(self, iden, peers=True, timeout=None): - if not ok: - logger.warning(f'getPeerTasks() error from peer {name}: {retn}') - continue + task = self.boss.get(iden) + if task is not None: + item = task.pack() + item['aha:service'] = self.ahasvcname + return item - task['aha:name'] = name - yield task + if not peers: + return - async def killPeerTask(self, iden, timeout=None): + todo = s_common.todo('getTask', peers=False, timeout=timeout) + async for ahasvc, retn in self.callPeerApi(todo, timeout=timeout) + if retn is not None: + return retn - # kill a task which may be running on a peer - if await self.killTask(iden): - return (True, self.ahasvcname) + async def killTask(self, iden, peers=True, timeout=None): - todo = s_common.todo('killTask', iden) - async for (name, (ok, done)) in self.callPeerApi(todo, timeout=timeout): + task = self.boss.get(iden) + if task is not None: + await task.kill() + return True - if not ok: - logger.warning(f'killPeerTask() error from peer {name}: {retn}') - continue + if not peers: + return False - if ok and done: - return (True, name) + todo = s_common.todo('killTask', peers=False, timeout=timeout) + async for ahasvc, retn in self.callPeerApi(todo, timeout=timeout): + if retn: + return True - return (False, None) + return False async def kill(self, user, iden): perm = ('task', 'del') diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index beab1afc24a..97dacea6f89 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -1850,6 +1850,43 @@ async def __call__(self, **kwargs): await self.runt.snap.warnonce('$lib.dict() is deprecated. Use ({}) instead.') return Dict(kwargs) +Task +@registry.registerType +class Task(StormType): + ''' + A Storm Pipe provides fast ephemeral queues. + ''' + _storm_locals = ( + {'name': 'put', 'desc': 'Add a single item to the Pipe.', + 'type': {'type': 'function', '_funcname': '_methPipePut', + 'args': ( + {'name': 'item', 'type': 'any', 'desc': 'An object to add to the Pipe.', }, + ), + 'returns': {'type': 'null', }}}, + {'name': 'svcname', 'desc': 'The AHA service name of the service running the task.' + 'type': 'str'}, + ) + def __init__(self, task): + self. + +@registry.registerLib +class LibTask(Lib): + ''' + A Storm library for interacting with running tasks. + ''' + _storm_locals = ( + {'name': 'get', 'desc': 'Retrieve a specific task by iden.', + 'type': {'type': 'function', '_funcname': '_methPipePut', + 'args': ( + {'name': 'item', 'type': 'any', 'desc': 'An object to add to the Pipe.', }, + ), + 'returns': {'type': 'null', }}}, + {'name': 'svcname', 'desc': 'The AHA service name of the service running the task.' + 'type': 'str'}, + ) + def get(): + def list(): + @registry.registerLib class LibPs(Lib): ''' From 1b28ac210104554dfb19674be25dce6fdf669910 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 27 Nov 2024 12:27:35 -0500 Subject: [PATCH 71/86] wip --- synapse/lib/cell.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index a6c5901a3ab..f93b57d40b2 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -435,6 +435,16 @@ async def ps(self): async def kill(self, iden): return await self.cell.kill(self.user, iden) + @adminapi() + async def callPeerApi(self, todo, timeout=None): + async for item in self.cell.callPeerApi(todo, timeout=timeout): + yield item + + @adminapi() + async def callPeerGenr(self, todo, timeout=None): + async for item in self.cell.callPeerGenr(todo, timeout=timeout): + yield item + @adminapi() async def getTasks(self, peers=True, timeout=None): async for task in self.cell.getTasks(peers=peers, timeout=timeout): From fa61327742745a35c19bfcaab17a05e087eec72b Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 27 Nov 2024 12:51:29 -0500 Subject: [PATCH 72/86] wip --- synapse/lib/aha.py | 26 +++++++---------------- synapse/lib/cell.py | 17 ++++++++------- synapse/lib/stormtypes.py | 37 --------------------------------- synapse/tests/test_lib_aha.py | 39 +++++++++++------------------------ 4 files changed, 30 insertions(+), 89 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 860830ad5dc..a6b4ec6ff21 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -138,13 +138,13 @@ async def getAhaUrls(self, user='root'): return ahaurls @s_cell.adminapi() - async def runGatherCall(self, iden, todo, timeout=None, skiprun=None): - async for item in self.cell.runGatherCall(iden, todo, timeout=timeout, skiprun=skiprun): + async def callAhaPeerApi(self, iden, todo, timeout=None, skiprun=None): + async for item in self.cell.callAhaPeerApi(iden, todo, timeout=timeout, skiprun=skiprun): yield item @s_cell.adminapi() - async def runGatherGenr(self, iden, todo, timeout=None, skiprun=None): - async for item in self.cell.runGatherGenr(iden, todo, timeout=timeout, skiprun=skiprun): + async def callAhaPeerGenr(self, iden, todo, timeout=None, skiprun=None): + async for item in self.cell.callAhaPeerGenr(iden, todo, timeout=timeout, skiprun=skiprun): yield item async def addNexsTracker(self, name, iden, network=None): @@ -396,11 +396,6 @@ async def clearAhaClones(self): ''' return await self.cell.clearAhaClones() - @s_cell.adminapi() - async def getSvcPeerTasks(self, iden, timeout=None, skiprun=None): - async for task in self.cell.getSvcPeerTasks(iden, timeout=timeout, skiprun=skiprun): - yield task - class ProvDmon(s_daemon.Daemon): async def __anit__(self, aha): @@ -743,11 +738,6 @@ async def _callAhaSvcGenr(self, svcdef, todo, timeout=None): # in case proxy construction fails yield (False, s_common.excinfo(e)) - async def getSvcPeerTasks(self, iden, timeout=None, skiprun=None): - todo = s_common.todo('getTasks') - async for item in self.runGatherGenr(iden, todo, timeout=timeout, skiprun=skiprun): - yield item - async def getAhaSvcsByIden(self, iden, online=True, skiprun=None): runs = set() @@ -778,11 +768,11 @@ def getAhaSvcUrl(self, svcdef, user='root'): port = svcdef['svcinfo']['urlinfo']['port'] return f'ssl://{host}:{port}?hostname={svcfull}&certname={user}@{svcnetw}' - async def runGatherCall(self, iden, todo, timeout=None, skiprun=None): + async def callAhaPeerApi(self, iden, todo, timeout=None, skiprun=None): if not self.isactive: proxy = await self.nexsroot.client.proxy(timeout=timeout) - async for item in proxy.runGatherCall(iden, todo, timeout=timeout, skiprun=skiprun): + async for item in proxy.callAhaPeerApi(iden, todo, timeout=timeout, skiprun=skiprun): yield item queue = asyncio.Queue() @@ -800,11 +790,11 @@ async def call(svcdef): for i in range(count): yield await queue.get() - async def runGatherGenr(self, iden, todo, timeout=None, skiprun=None): + async def callAhaPeerGenr(self, iden, todo, timeout=None, skiprun=None): if not self.isactive: proxy = await self.nexsroot.client.proxy(timeout=timeout) - async for item in proxy.runGatherGenr(iden, todo, timeout=timeout, skiprun=skiprun): + async for item in proxy.callAhaPeerGenr(iden, todo, timeout=timeout, skiprun=skiprun): yield item queue = asyncio.Queue() diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index f93b57d40b2..cde0991fb87 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -4473,22 +4473,22 @@ async def callPeerApi(self, todo, timeout=None): ''' Yield responses from our peers via the AHA gather call API. ''' - proxy = await self.getAhaProxy(timeout=timeout, methname='runGatherCall') + proxy = await self.getAhaProxy(timeout=timeout, methname='callAhaPeerApi') if proxy is None: return - async for item in proxy.runGatherCall(self.iden, todo, timeout=timeout, skiprun=self.runid): + async for item in proxy.callAhaPeerApi(self.iden, todo, timeout=timeout, skiprun=self.runid): yield item async def callPeerGenr(self, todo, timeout=None): ''' Yield responses from invoking a generator via the AHA gather API. ''' - proxy = await self.getAhaProxy(timeout=timeout, methname='runGatherGenr') + proxy = await self.getAhaProxy(timeout=timeout, methname='callAhaPeerGenr') if proxy is None: return - async for item in proxy.runGatherGenr(self.iden, todo, timeout=timeout, skiprun=self.runid): + async for item in proxy.callAhaPeerGenr(self.iden, todo, timeout=timeout, skiprun=self.runid): yield item async def getTasks(self, peers=True, timeout=None): @@ -4519,11 +4519,14 @@ async def getTask(self, iden, peers=True, timeout=None): if not peers: return - todo = s_common.todo('getTask', peers=False, timeout=timeout) - async for ahasvc, retn in self.callPeerApi(todo, timeout=timeout) - if retn is not None: + todo = s_common.todo('getTask', iden, peers=False, timeout=timeout) + async for ahasvc, (ok, retn) in self.callPeerApi(todo, timeout=timeout): + + if ok: return retn + logger.warning(f'getTask() on {ahasvc} failed: {retn}') + async def killTask(self, iden, peers=True, timeout=None): task = self.boss.get(iden) diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 60332131a9b..1db0f34967c 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -1863,43 +1863,6 @@ async def __call__(self, **kwargs): await self.runt.snap.warnonce('$lib.dict() is deprecated. Use ({}) instead.') return Dict(kwargs) -Task -@registry.registerType -class Task(StormType): - ''' - A Storm Pipe provides fast ephemeral queues. - ''' - _storm_locals = ( - {'name': 'put', 'desc': 'Add a single item to the Pipe.', - 'type': {'type': 'function', '_funcname': '_methPipePut', - 'args': ( - {'name': 'item', 'type': 'any', 'desc': 'An object to add to the Pipe.', }, - ), - 'returns': {'type': 'null', }}}, - {'name': 'svcname', 'desc': 'The AHA service name of the service running the task.' - 'type': 'str'}, - ) - def __init__(self, task): - self. - -@registry.registerLib -class LibTask(Lib): - ''' - A Storm library for interacting with running tasks. - ''' - _storm_locals = ( - {'name': 'get', 'desc': 'Retrieve a specific task by iden.', - 'type': {'type': 'function', '_funcname': '_methPipePut', - 'args': ( - {'name': 'item', 'type': 'any', 'desc': 'An object to add to the Pipe.', }, - ), - 'returns': {'type': 'null', }}}, - {'name': 'svcname', 'desc': 'The AHA service name of the service running the task.' - 'type': 'str'}, - ) - def get(): - def list(): - @registry.registerLib class LibPs(Lib): ''' diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 363ded398ad..a1290fc1c47 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1329,21 +1329,21 @@ async def test_aha_gather(self): # test the call endpoint todo = s_common.todo('getCellInfo') - items = dict([item async for item in aha.runGatherCall(cell00.iden, todo, timeout=3)]) + items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)]) self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) self.true(all(item[0] for item in items.values())) self.eq(cell00.runid, items['00.cell.synapse'][1]['cell']['run']) self.eq(cell01.runid, items['01.cell.synapse'][1]['cell']['run']) todo = s_common.todo('newp') - items = dict([item async for item in aha.runGatherCall(cell00.iden, todo, timeout=3)]) + items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)]) self.false(any(item[0] for item in items.values())) self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) # test the genr endpoint reals = [item async for item in cell00.getNexusChanges(0, wait=False)] todo = s_common.todo('getNexusChanges', 0, wait=False) - items = [item async for item in aha.runGatherGenr(cell00.iden, todo, timeout=3) if item[1]] + items = [item async for item in aha.callAhaPeerGenr(cell00.iden, todo, timeout=3) if item[1]] self.len(nexsindx * 2, items) # ensure we handle down services correctly @@ -1352,14 +1352,14 @@ async def test_aha_gather(self): # test the call endpoint todo = s_common.todo('getCellInfo') - items = dict([item async for item in aha.runGatherCall(cell00.iden, todo, timeout=3)]) + items = dict([item async for item in aha.callAhaPeerApi(cell00.iden, todo, timeout=3)]) self.sorteq(items.keys(), ('00.cell.synapse',)) self.true(all(item[0] for item in items.values())) self.eq(cell00.runid, items['00.cell.synapse'][1]['cell']['run']) # test the genr endpoint todo = s_common.todo('getNexusChanges', 0, wait=False) - items = [item async for item in aha.runGatherGenr(cell00.iden, todo, timeout=3) if item[1]] + items = [item async for item in aha.callAhaPeerGenr(cell00.iden, todo, timeout=3) if item[1]] self.len(nexsindx, items) self.true(all(item[1][0] for item in items)) @@ -1382,28 +1382,13 @@ async def sleep99(cell): task01 = cell01.schedCoro(sleep99(cell01)) proxy = await aha.enter_context(aha.getLocalProxy()) - tasks = [task async for task in proxy.getAhaSvcPeerTasks(cell00.iden, timeout=3)] - tasks.sort() + tasks = [task async for task in cell00.getTasks(timeout=3)] self.len(2, tasks) - self.eq(tasks[0][0], '0.cell.synapse') - self.true(tasks[0][1][0]) - self.eq(tasks[1][0], '1.cell.synapse') - self.true(tasks[1][1][0]) + self.eq(tasks[0]['aha:service'], '0.cell.synapse') + self.eq(tasks[1]['aha:service'], '1.cell.synapse') - tasks = [task async for task in proxy.getAhaSvcPeerTasks(cell00.iden, timeout=3, skiprun=cell00.runid)] - tasks.sort() - self.len(1, tasks) - self.eq(tasks[0][0], '1.cell.synapse') + self.eq(tasks[0], await cell00.getTask(tasks[0].get('iden'))) + self.eq(tasks[1], await cell00.getTask(tasks[1].get('iden'))) - tasks = [task async for task in cell00.getPeerTasks(timeout=3)] - self.len(2, tasks) - - self.eq(tasks[0], await cell00.getPeerTask(tasks[0].get('iden'))) - self.eq(tasks[1], await cell00.getPeerTask(tasks[1].get('iden'))) - - retn00 = await cell00.killPeerTask(tasks[0].get('iden')) - retn01 = await cell00.killPeerTask(tasks[1].get('iden')) - - self.eq((True, tasks[0]['aha:name']), retn00) - self.eq((True, tasks[1]['aha:name']), retn01) - self.eq((False, None), await cell00.killPeerTask('newp')) + self.true(await cell00.killTask(tasks[0].get('iden'))) + self.true(await cell00.killTask(tasks[1].get('iden'))) From fda40c7043be57ecb1920153087fac1e63090107 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 3 Dec 2024 09:28:41 -0500 Subject: [PATCH 73/86] wip --- synapse/lib/cell.py | 4 ++-- synapse/lib/stormtypes.py | 5 +++++ synapse/tests/test_lib_aha.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index cde0991fb87..8a08e10611c 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -4496,7 +4496,7 @@ async def getTasks(self, peers=True, timeout=None): for task in self.boss.ps(): item = task.pack() - item['aha:service'] = self.ahasvcname + item['service'] = self.ahasvcname yield item @@ -4513,7 +4513,7 @@ async def getTask(self, iden, peers=True, timeout=None): task = self.boss.get(iden) if task is not None: item = task.pack() - item['aha:service'] = self.ahasvcname + item['service'] = self.ahasvcname return item if not peers: diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 1db0f34967c..257e233774d 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -1892,6 +1892,9 @@ def getObjLocals(self): async def _kill(self, prefix): idens = [] + s_common.deprecated('$lib.ps.kill()') + self.runt.snap.warnonce('$lib.ps.kill() is deprecated. Please use $lib.task.get().kill()') + todo = s_common.todo('ps', self.runt.user) tasks = await self.dyncall('cell', todo) for task in tasks: @@ -1912,6 +1915,8 @@ async def _kill(self, prefix): @stormfunc(readonly=True) async def _list(self): + s_common.deprecated('$lib.ps.list()') + self.runt.snap.warnonce('$lib.ps.list() is deprecated. Please use $lib.task.list()') todo = s_common.todo('ps', self.runt.user) return await self.dyncall('cell', todo) diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index a1290fc1c47..f6f7bcbbb76 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1384,8 +1384,8 @@ async def sleep99(cell): proxy = await aha.enter_context(aha.getLocalProxy()) tasks = [task async for task in cell00.getTasks(timeout=3)] self.len(2, tasks) - self.eq(tasks[0]['aha:service'], '0.cell.synapse') - self.eq(tasks[1]['aha:service'], '1.cell.synapse') + self.eq(tasks[0]['service'], '0.cell.synapse') + self.eq(tasks[1]['service'], '1.cell.synapse') self.eq(tasks[0], await cell00.getTask(tasks[0].get('iden'))) self.eq(tasks[1], await cell00.getTask(tasks[1].get('iden'))) From a62e7fda8b3ef692b8d4f00e5b60cb2d76380691 Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 3 Dec 2024 14:24:56 -0500 Subject: [PATCH 74/86] wip --- synapse/lib/aha.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index a6b4ec6ff21..7671e24e129 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -147,25 +147,6 @@ async def callAhaPeerGenr(self, iden, todo, timeout=None, skiprun=None): async for item in self.cell.callAhaPeerGenr(iden, todo, timeout=timeout, skiprun=skiprun): yield item - async def addNexsTracker(self, name, iden, network=None): - ''' - Add an explicit nexus tracker which consumes the transactions - produced by the iden leader. - ''' - return await self.cell.addNexsTracker(name, iden, network=network) - - async def delNexsTracker(self, name, iden, network=None): - ''' - Remove a nexus tracker entry. - ''' - return await self.cell.delNexsTracker(name, iden, network=network) - - async def getNexsTrackers(self, iden): - # services which have the same iden are all automatically trackers - # explicitly registered trackers are also yielded here - async for item in self.cell.getNexsTrackers(iden): - yield item - async def getAhaSvc(self, name, filters=None): ''' Return an AHA service description dictionary for a service name. From 86d0b92e03c7e3a9f88012344ecb2e882e3a4cae Mon Sep 17 00:00:00 2001 From: visi Date: Tue, 3 Dec 2024 14:29:48 -0500 Subject: [PATCH 75/86] wip --- synapse/lib/stormtypes.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index 257e233774d..1db0f34967c 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -1892,9 +1892,6 @@ def getObjLocals(self): async def _kill(self, prefix): idens = [] - s_common.deprecated('$lib.ps.kill()') - self.runt.snap.warnonce('$lib.ps.kill() is deprecated. Please use $lib.task.get().kill()') - todo = s_common.todo('ps', self.runt.user) tasks = await self.dyncall('cell', todo) for task in tasks: @@ -1915,8 +1912,6 @@ async def _kill(self, prefix): @stormfunc(readonly=True) async def _list(self): - s_common.deprecated('$lib.ps.list()') - self.runt.snap.warnonce('$lib.ps.list() is deprecated. Please use $lib.task.list()') todo = s_common.todo('ps', self.runt.user) return await self.dyncall('cell', todo) From 13e80c6b34b346e2724aea49debab218e2481c51 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 4 Dec 2024 10:34:05 -0500 Subject: [PATCH 76/86] wip --- synapse/lib/aha.py | 2 ++ synapse/lib/cell.py | 27 +++++++++++++-------------- synapse/telepath.py | 13 ++----------- synapse/tests/test_lib_aha.py | 3 +++ 4 files changed, 20 insertions(+), 25 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 7671e24e129..a275f1a9043 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -581,6 +581,8 @@ async def _initCellBoot(self): async def initServiceStorage(self): + self.features['callpeers'] = 1 + dirn = s_common.gendir(self.dirn, 'slabs', 'jsonstor') slab = await s_lmdbslab.Slab.anit(dirn) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 5f9b4b7425d..a814120b330 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -83,6 +83,8 @@ PERM_ADMIN: 'admin', } +feat_aha_callpeers_v1 = ('callpeers', 1) + diskspace = "Insufficient free space on disk." def adminapi(log=False): @@ -458,12 +460,6 @@ async def getTask(self, iden, peers=True, timeout=None): async def killTask(self, iden, peers=True, timeout=None): return await self.cell.killTask(iden, peers=peers, timeout=timeout) - async def getFeatures(self): - return await self.cell.getFeatures() - - async def hasFeature(self, name): - return self.cell.hasFeature(name) - @adminapi(log=True) async def behold(self): ''' @@ -1156,8 +1152,9 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): self.conf = self._initCellConf(conf) self.features = { - 'tellready': True, - 'dynmirror': True, + 'tellready': 1, + 'dynmirror': 1, + 'tasks': 1, } self.minfree = self.conf.get('limit:disk:free') @@ -4453,7 +4450,7 @@ async def ps(self, user): return retn - async def getAhaProxy(self, timeout=None, methname=None): + async def getAhaProxy(self, timeout=None, feats=None): if self.ahaclient is None: return @@ -4463,9 +4460,11 @@ async def getAhaProxy(self, timeout=None, methname=None): logger.warning('AHA client connection failed.') return - if methname is not None and not proxy._hasTeleMeth(methname): - logger.warning(f'AHA server does not implement {methname}.') - return + if feats is not None: + for name, vers in feats: + if not proxy._hasTeleFeat(name, vers): + logger.warning(f'AHA server does not support feature: {name} >= {vers}') + return None return proxy @@ -4473,7 +4472,7 @@ async def callPeerApi(self, todo, timeout=None): ''' Yield responses from our peers via the AHA gather call API. ''' - proxy = await self.getAhaProxy(timeout=timeout, methname='callAhaPeerApi') + proxy = await self.getAhaProxy(timeout=timeout, feats=(feat_aha_callpeers_v1,)) if proxy is None: return @@ -4484,7 +4483,7 @@ async def callPeerGenr(self, todo, timeout=None): ''' Yield responses from invoking a generator via the AHA gather API. ''' - proxy = await self.getAhaProxy(timeout=timeout, methname='callAhaPeerGenr') + proxy = await self.getAhaProxy(timeout=timeout, feats=(feat_aha_callpeers_v1,)) if proxy is None: return diff --git a/synapse/telepath.py b/synapse/telepath.py index bfc8dd1ccc2..7a8997af584 100644 --- a/synapse/telepath.py +++ b/synapse/telepath.py @@ -639,17 +639,8 @@ async def fini(): self.onfini(fini) self.link.onfini(self.fini) - def _hasTeleMeth(self, name): - return self.methinfo.get(name) is not None - - def _hasTeleGenr(self, name): - info = self.methinfo.get(name) - if info is None: - return False - return info.get('genr', False) - - def _hasTeleFeat(self, name): - return self.features.get(name, False) + def _hasTeleFeat(self, name, vers=1): + return self.features.get(name, 0) >= vers def _getSynVers(self): ''' diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index f6f7bcbbb76..f31f1b238c3 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1325,6 +1325,9 @@ async def test_aha_gather(self): await cell01.sync() + async with cell01.getLocalProxy() as proxy: + self.true(proxy._hasTeleFeat('dynmirror')) + nexsindx = await cell00.getNexsIndx() # test the call endpoint From dbe453e62f90a5b71025b31922a1facbe496401c Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 4 Dec 2024 10:41:30 -0500 Subject: [PATCH 77/86] wip --- synapse/telepath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/telepath.py b/synapse/telepath.py index 7a8997af584..343eb13b91f 100644 --- a/synapse/telepath.py +++ b/synapse/telepath.py @@ -640,7 +640,7 @@ async def fini(): self.link.onfini(self.fini) def _hasTeleFeat(self, name, vers=1): - return self.features.get(name, 0) >= vers + return self._features.get(name, 0) >= vers def _getSynVers(self): ''' @@ -914,7 +914,7 @@ async def handshake(self, auth=None): self.sess = self.synack[1].get('sess') self._ahainfo = self.synack[1].get('ahainfo', {}) - self.features = self.synack[1].get('features', {}) + self._features = self.synack[1].get('features', {}) self.sharinfo = self.synack[1].get('sharinfo', {}) self.methinfo = self.sharinfo.get('meths', {}) From 53b6e51ae49e7bfa77cd8778cbe1d74205407fa8 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 4 Dec 2024 13:10:19 -0500 Subject: [PATCH 78/86] wip --- synapse/lib/reflect.py | 9 ++++----- synapse/telepath.py | 3 +++ synapse/tests/test_lib_aha.py | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/synapse/lib/reflect.py b/synapse/lib/reflect.py index 2897bc5a414..dfc00fd12de 100644 --- a/synapse/lib/reflect.py +++ b/synapse/lib/reflect.py @@ -87,17 +87,16 @@ def getShareInfo(item): if not callable(attr): continue + meths[name] = meth = {} + # We know we can cleanly unwrap these functions # for asyncgenerator inspection. wrapped = getattr(attr, '__syn_wrapped__', None) if wrapped in unwraps: - real = inspect.unwrap(attr) - if inspect.isasyncgenfunction(real): - meths[name] = {'genr': True} - continue + attr = inspect.unwrap(attr) if inspect.isasyncgenfunction(attr): - meths[name] = {'genr': True} + meth['genr'] = True try: setattr(item, key, info) diff --git a/synapse/telepath.py b/synapse/telepath.py index 343eb13b91f..7b355058b8b 100644 --- a/synapse/telepath.py +++ b/synapse/telepath.py @@ -642,6 +642,9 @@ async def fini(): def _hasTeleFeat(self, name, vers=1): return self._features.get(name, 0) >= vers + def _hasTeleMeth(self, name): + return self.methinfo.get(name) is not None + def _getSynVers(self): ''' Helper method to retrieve the remote Synapse version from Proxy. diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index f31f1b238c3..1a5cb3c004d 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1327,6 +1327,7 @@ async def test_aha_gather(self): async with cell01.getLocalProxy() as proxy: self.true(proxy._hasTeleFeat('dynmirror')) + self.true(proxy._hasTeleMeth('getNexsIndx')) nexsindx = await cell00.getNexsIndx() From c1dd0d6f95575028ec5559e31dab9845bc4a47c2 Mon Sep 17 00:00:00 2001 From: visi Date: Wed, 4 Dec 2024 13:36:26 -0500 Subject: [PATCH 79/86] wip --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 047f0322710..5f1b36913f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -85,7 +85,7 @@ commands: - restore_cache: keys: - - v5-docvenv-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "pyproject.toml" }} + - v6-docvenv-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "pyproject.toml" }} - run: name: setup venv @@ -103,7 +103,7 @@ commands: - save_cache: paths: - ./venv - key: v5-docvenv-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "pyproject.toml" }} + key: v6-docvenv-{{ .Environment.CIRCLE_JOB }}-{{ .Branch }}-{{ checksum "pyproject.toml" }} - run: name: executing docs jupyter notebooks From f1ac2eccb1313572f5286d3ddf7f3550128e9e62 Mon Sep 17 00:00:00 2001 From: visi Date: Thu, 5 Dec 2024 13:45:35 -0500 Subject: [PATCH 80/86] wip --- synapse/lib/cell.py | 16 +++------------- synapse/tests/test_common.py | 12 ++++++++++++ synapse/tests/test_lib_aha.py | 12 +++++++++++- synapse/tests/test_lib_cell.py | 11 +++++++++++ 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index a814120b330..63a29764a71 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -437,16 +437,6 @@ async def ps(self): async def kill(self, iden): return await self.cell.kill(self.user, iden) - @adminapi() - async def callPeerApi(self, todo, timeout=None): - async for item in self.cell.callPeerApi(todo, timeout=timeout): - yield item - - @adminapi() - async def callPeerGenr(self, todo, timeout=None): - async for item in self.cell.callPeerGenr(todo, timeout=timeout): - yield item - @adminapi() async def getTasks(self, peers=True, timeout=None): async for task in self.cell.getTasks(peers=peers, timeout=timeout): @@ -4536,9 +4526,9 @@ async def killTask(self, iden, peers=True, timeout=None): if not peers: return False - todo = s_common.todo('killTask', peers=False, timeout=timeout) - async for ahasvc, retn in self.callPeerApi(todo, timeout=timeout): - if retn: + todo = s_common.todo('killTask', iden, peers=False, timeout=timeout) + async for ahasvc, (ok, retn) in self.callPeerApi(todo, timeout=timeout): + if ok and retn: return True return False diff --git a/synapse/tests/test_common.py b/synapse/tests/test_common.py index f1c258f71a6..7badf334980 100644 --- a/synapse/tests/test_common.py +++ b/synapse/tests/test_common.py @@ -12,6 +12,18 @@ logger = logging.getLogger(__name__) class CommonTest(s_t_utils.SynTest): + + async def test_waitgenr(self): + + async def genr(): + yield 10 + raise Exception('omg') + + rets = [retn async for retn in s_common.waitgenr(genr(), 10)] + + self.true(rets[0][0]) + self.false(rets[1][0]) + def test_tuplify(self): tv = ['node', [['test:str', 'test'], {'tags': { diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index 1a5cb3c004d..fa2d3740f34 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -1393,6 +1393,16 @@ async def sleep99(cell): self.eq(tasks[0], await cell00.getTask(tasks[0].get('iden'))) self.eq(tasks[1], await cell00.getTask(tasks[1].get('iden'))) + self.none(await cell00.getTask(tasks[1].get('iden'), peers=False)) self.true(await cell00.killTask(tasks[0].get('iden'))) - self.true(await cell00.killTask(tasks[1].get('iden'))) + + task01 = tasks[1].get('iden') + self.false(await cell00.killTask(task01, peers=False)) + + self.true(await cell00.killTask(task01)) + + self.none(await cell00.getTask(task01)) + self.false(await cell00.killTask(task01)) + + self.none(await cell00.getAhaProxy(feats=(('newp', 9),))) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index 7457c62b89c..392ffe3261f 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -3241,3 +3241,14 @@ async def test_lib_cell_promote_schism_prevent(self): with self.raises(s_exc.BadState) as cm: await cell00.promote(graceful=True) self.isin('02.cell is not the current leader', cm.exception.get('mesg')) + + async def test_lib_cell_sadaha(self): + + async with self.getTestCell() as cell: + + self.none(await cell.getAhaProxy()) + cell.ahaclient = await s_telepath.Client.anit('cell:///tmp/newp') + + # coverage for failure of aha client to connect + with self.raises(TimeoutError): + self.none(await cell.getAhaProxy(timeout=0.1)) From c6ff805c6389e944d968c2481b0a50604b651de1 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 6 Dec 2024 10:53:11 -0500 Subject: [PATCH 81/86] wip --- synapse/common.py | 6 ++---- synapse/lib/aha.py | 4 ---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/synapse/common.py b/synapse/common.py index ccc7d0ef047..602bed9445a 100644 --- a/synapse/common.py +++ b/synapse/common.py @@ -1247,8 +1247,8 @@ async def waitretn(futu, timeout): async def waitgenr(genr, timeout): - genr = genr.__aiter__() - try: + async with contextlib.aclosing(genr.__aiter__()) as genr: + while True: retn = await waitretn(genr.__anext__(), timeout) @@ -1259,8 +1259,6 @@ async def waitgenr(genr, timeout): if not retn[0]: return - finally: - await genr.aclose() def _release_waiter(waiter, *args): if not waiter.done(): diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index a275f1a9043..d486613d0aa 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -707,10 +707,6 @@ async def _callAhaSvcApi(self, svcdef, todo, timeout=None): # in case proxy construction fails return (False, s_common.excinfo(e)) - async def callAhaSvcGenr(self, name, todo, timeout=None): - name = self._getAhaName(name) - svcdef = await self._getAhaSvc(name) - async def _callAhaSvcGenr(self, svcdef, todo, timeout=None): try: proxy = await self.getAhaSvcProxy(svcdef, timeout=timeout) From 9dd25fec20d83346d537b2076522acb0e018224f Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 6 Dec 2024 11:00:20 -0500 Subject: [PATCH 82/86] wip --- synapse/lib/cell.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 63a29764a71..28d27e882e0 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -4494,7 +4494,12 @@ async def getTasks(self, peers=True, timeout=None): todo = s_common.todo('getTasks', peers=False) # we can ignore the yielded aha names because we embed it in the task - async for (svcname, (ok, retn)) in self.callPeerGenr(todo, timeout=timeout): + async for (ahasvc, (ok, retn)) in self.callPeerGenr(todo, timeout=timeout): + + if not ok: + logger.warning(f'getTasks() on {ahasvc} failed: {retn}') + continue + yield retn async def getTask(self, iden, peers=True, timeout=None): @@ -4511,10 +4516,12 @@ async def getTask(self, iden, peers=True, timeout=None): todo = s_common.todo('getTask', iden, peers=False, timeout=timeout) async for ahasvc, (ok, retn) in self.callPeerApi(todo, timeout=timeout): - if ok: - return retn + if not ok: + logger.warning(f'getTask() on {ahasvc} failed: {retn}') + continue - logger.warning(f'getTask() on {ahasvc} failed: {retn}') + if retn is not None: + return retn async def killTask(self, iden, peers=True, timeout=None): @@ -4528,7 +4535,12 @@ async def killTask(self, iden, peers=True, timeout=None): todo = s_common.todo('killTask', iden, peers=False, timeout=timeout) async for ahasvc, (ok, retn) in self.callPeerApi(todo, timeout=timeout): - if ok and retn: + + if not ok: + logger.warning(f'killTask() on {ahasvc} failed: {retn}') + continue + + if retn: return True return False From b580cbd0bdda88578d0f7c4d16572dcf2e2235e5 Mon Sep 17 00:00:00 2001 From: visi Date: Fri, 6 Dec 2024 11:23:45 -0500 Subject: [PATCH 83/86] wip --- synapse/lib/aha.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index d486613d0aa..f5d567b7e47 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -1037,7 +1037,6 @@ async def reqAhaSvcProxy(self, svcdef, timeout=None): async def getAhaSvcProxy(self, svcdef, timeout=None): - assert self.isactive client = await self.getAhaSvcClient(svcdef) if client is None: return None @@ -1046,8 +1045,6 @@ async def getAhaSvcProxy(self, svcdef, timeout=None): async def getAhaSvcClient(self, svcdef): - assert self.isactive - svcfull = svcdef.get('name') client = self.clients.get(svcfull) From 8c2bfa60a55345e4ca08ede25d2e5f3f7d908d35 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Fri, 10 Jan 2025 14:13:13 -0500 Subject: [PATCH 84/86] add mirror tool for cluster service status report (SYN-6920) (#4019) Co-authored-by: bender Co-authored-by: blackout Co-authored-by: vEpiphyte Co-authored-by: Cisphyx --- synapse/common.py | 45 +-- synapse/cortex.py | 1 + synapse/lib/aha.py | 6 +- synapse/lib/stormlib/aha.py | 352 ++++++++++++++++++++++- synapse/lib/stormlib/utils.py | 37 +++ synapse/tests/test_common.py | 6 + synapse/tests/test_lib_aha.py | 145 ++++++++++ synapse/tests/test_lib_cell.py | 34 +++ synapse/tests/test_lib_stormlib_aha.py | 188 ++++++++++++ synapse/tests/test_lib_stormlib_utils.py | 14 + synapse/tests/test_tools_aha.py | 192 +++++++++++++ synapse/tools/aha/mirror.py | 193 +++++++++++++ 12 files changed, 1186 insertions(+), 27 deletions(-) create mode 100644 synapse/lib/stormlib/utils.py create mode 100644 synapse/tests/test_lib_stormlib_utils.py create mode 100644 synapse/tools/aha/mirror.py diff --git a/synapse/common.py b/synapse/common.py index 75f2c46201c..95222fde7c7 100644 --- a/synapse/common.py +++ b/synapse/common.py @@ -1253,28 +1253,6 @@ async def wait_for(fut, timeout): async with _timeout(timeout): return await fut -async def waitretn(futu, timeout): - try: - valu = await wait_for(futu, timeout) - return (True, valu) - except Exception as e: - return (False, excinfo(e)) - -async def waitgenr(genr, timeout): - - async with contextlib.aclosing(genr.__aiter__()) as genr: - - while True: - retn = await waitretn(genr.__anext__(), timeout) - - if not retn[0] and retn[1]['err'] == 'StopAsyncIteration': - return - - yield retn - - if not retn[0]: - return - def _release_waiter(waiter, *args): if not waiter.done(): waiter.set_result(None) @@ -1405,6 +1383,29 @@ def _timeout(delay): """ loop = asyncio.get_running_loop() return _Timeout(loop.time() + delay if delay is not None else None) +# End - Vendored Code from Python 3.12+ + +async def waitretn(futu, timeout): + try: + valu = await wait_for(futu, timeout) + return (True, valu) + except Exception as e: + return (False, excinfo(e)) + +async def waitgenr(genr, timeout): + + async with contextlib.aclosing(genr.__aiter__()) as genr: + + while True: + retn = await waitretn(genr.__anext__(), timeout) + + if not retn[0] and retn[1]['err'] == 'StopAsyncIteration': + return + + yield retn + + if not retn[0]: + return def format(text, **kwargs): ''' diff --git a/synapse/cortex.py b/synapse/cortex.py index 603da3dee6d..08232c6d89a 100644 --- a/synapse/cortex.py +++ b/synapse/cortex.py @@ -83,6 +83,7 @@ import synapse.lib.stormlib.oauth as s_stormlib_oauth # NOQA import synapse.lib.stormlib.stats as s_stormlib_stats # NOQA import synapse.lib.stormlib.storm as s_stormlib_storm # NOQA +import synapse.lib.stormlib.utils as s_stormlib_utils # NOQA import synapse.lib.stormlib.vault as s_stormlib_vault # NOQA import synapse.lib.stormlib.backup as s_stormlib_backup # NOQA import synapse.lib.stormlib.cortex as s_stormlib_cortex # NOQA diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index f5d567b7e47..98c45d0dbd3 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -133,8 +133,6 @@ async def addAhaClone(self, host, port=27492, conf=None): async def getAhaUrls(self, user='root'): ahaurls = await self.cell.getAhaUrls(user=user) - if ahaurls is None: - return () return ahaurls @s_cell.adminapi() @@ -723,7 +721,7 @@ async def getAhaSvcsByIden(self, iden, online=True, skiprun=None): async for svcdef in self.getAhaSvcs(): await asyncio.sleep(0) - # TODO services by iden indexes! + # TODO services by iden indexes (SYN-8467) if svcdef['svcinfo'].get('iden') != iden: continue @@ -1032,7 +1030,7 @@ async def reqAhaSvcProxy(self, svcdef, timeout=None): if proxy is not None: return proxy - mesg = f'The service is not ready {svcfull}.' + mesg = f'The service is not ready {svcdef.get("name")}.' raise s_exc.NotReady(mesg=mesg) async def getAhaSvcProxy(self, svcdef, timeout=None): diff --git a/synapse/lib/stormlib/aha.py b/synapse/lib/stormlib/aha.py index d5f060ef20a..43f105e2647 100644 --- a/synapse/lib/stormlib/aha.py +++ b/synapse/lib/stormlib/aha.py @@ -1,4 +1,7 @@ +import textwrap + import synapse.exc as s_exc +import synapse.common as s_common import synapse.lib.stormtypes as s_stormtypes @s_stormtypes.registry.registerLib @@ -49,6 +52,80 @@ class AhaLib(s_stormtypes.Lib): 'type': {'type': 'function', '_funcname': '_methAhaList', 'args': (), 'returns': {'name': 'Yields', 'type': 'list', 'desc': 'The AHA service dictionaries.', }}}, + {'name': 'callPeerApi', 'desc': '''Call an API on all peers (leader and mirrors) of an AHA service and yield the responses from each. + + Examples: + Call getCellInfo on an AHA service:: + + $todo = $lib.utils.todo('getCellInfo') + for $info in $lib.aha.callPeerApi(cortex..., $todo) { + $lib.print($info) + } + + Call getCellInfo on an AHA service, skipping the invoking service:: + + $todo = $lib.utils.todo('getCellInfo') + for $info in $lib.aha.callPeerApi(cortex..., $todo, skiprun=$lib.cell.getCellInfo().cell.run) { + $lib.print($info) + } + + Call method with arguments:: + + $todo = $lib.utils.todo(('method', ([1, 2]), ({'foo': 'bar'}))) + for $info in $lib.aha.callPeerApi(cortex..., $todo) { + $lib.print($info) + } + + ''', + 'type': {'type': 'function', '_funcname': '_methCallPeerApi', + 'args': ( + {'name': 'svcname', 'type': 'str', + 'desc': 'The name of the AHA service to call. It is easiest to use the relative name of a service, ending with "...".', }, + {'name': 'todo', 'type': 'list', + 'desc': 'The todo tuple (name, args, kwargs).'}, + {'name': 'timeout', 'type': 'int', 'default': None, + 'desc': 'Optional timeout in seconds.'}, + {'name': 'skiprun', 'type': 'str', 'default': None, + 'desc': '''Optional run ID argument that allows skipping results from a specific service run ID. + This is most often used to omit the invoking service from the results, ensuring that only responses from other services are included. + '''}, + ), + 'returns': {'name': 'yields', 'type': 'list', + 'desc': 'Yields the results of the API calls as tuples of (svcname, (ok, info)).', }}}, + {'name': 'callPeerGenr', 'desc': '''Call a generator API on all peers (leader and mirrors) of an AHA service and yield the responses from each. + + Examples: + Call getNexusChanges on an AHA service:: + + $todo = $lib.utils.todo('getNexusChanges', (0), wait=$lib.false) + for $info in $lib.aha.callPeerGenr(cortex..., $todo) { + $lib.print($info) + } + + Call getNexusChanges on an AHA service, skipping the invoking service:: + + $todo = $lib.utils.todo('getNexusChanges', (0), wait=$lib.false) + for $info in $lib.aha.callPeerGenr(cortex..., $todo, skiprun=$lib.cell.getCellInfo().cell.run) { + $lib.print($info) + } + + ''', + 'type': {'type': 'function', '_funcname': '_methCallPeerGenr', + 'args': ( + {'name': 'svcname', 'type': 'str', + 'desc': 'The name of the AHA service to call. It is easiest to use the relative name of a service, ending with "...".', }, + {'name': 'todo', 'type': 'list', + 'desc': 'The todo tuple (name, args, kwargs).'}, + {'name': 'timeout', 'type': 'int', 'default': None, + 'desc': 'Optional timeout in seconds.'}, + {'name': 'skiprun', 'type': 'str', 'default': None, + 'desc': '''Optional run ID argument that allows skipping results from a specific service run ID. + This is most often used to omit the invoking service from the results, ensuring that only responses from other services are included. + '''}, + ), + 'returns': {'name': 'yields', 'type': 'list', + 'desc': 'Yields the results of the API calls as tuples containing (svcname, (ok, info)).', }}} + ) _storm_lib_path = ('aha',) def getObjLocals(self): @@ -56,6 +133,8 @@ def getObjLocals(self): 'del': self._methAhaDel, 'get': self._methAhaGet, 'list': self._methAhaList, + 'callPeerApi': self._methCallPeerApi, + 'callPeerGenr': self._methCallPeerGenr, } @s_stormtypes.stormfunc(readonly=True) @@ -85,6 +164,62 @@ async def _methAhaGet(self, svcname, filters=None): proxy = await self.runt.snap.core.reqAhaProxy() return await proxy.getAhaSvc(svcname, filters=filters) + async def _methCallPeerApi(self, svcname, todo, timeout=None, skiprun=None): + ''' + Call an API on an AHA service. + + Args: + svcname (str): The name of the AHA service to call. + todo (list): The todo tuple from $lib.utils.todo(). + timeout (int): Optional timeout in seconds. + skiprun (str): Optional run ID argument allows skipping self-enumeration. + ''' + svcname = await s_stormtypes.tostr(svcname) + todo = await s_stormtypes.toprim(todo) + timeout = await s_stormtypes.toint(timeout, noneok=True) + skiprun = await s_stormtypes.tostr(skiprun, noneok=True) + + proxy = await self.runt.snap.core.reqAhaProxy() + svc = await proxy.getAhaSvc(svcname) + if svc is None: + raise s_exc.NoSuchName(mesg=f'No AHA service found for {svcname}') + + svcinfo = svc.get('svcinfo') + svciden = svcinfo.get('iden') + if svciden is None: + raise s_exc.NoSuchName(mesg=f'Service {svcname} has no iden') + + async for svcname, (ok, info) in proxy.callAhaPeerApi(svciden, todo, timeout=timeout, skiprun=skiprun): + yield (svcname, (ok, info)) + + async def _methCallPeerGenr(self, svcname, todo, timeout=None, skiprun=None): + ''' + Call a generator API on an AHA service. + + Args: + svcname (str): The name of the AHA service to call. + todo (list): The todo tuple from $lib.utils.todo(). + timeout (int): Optional timeout in seconds. + skiprun (str): Optional run ID argument allows skipping self-enumeration. + ''' + svcname = await s_stormtypes.tostr(svcname) + todo = await s_stormtypes.toprim(todo) + timeout = await s_stormtypes.toint(timeout, noneok=True) + skiprun = await s_stormtypes.tostr(skiprun, noneok=True) + + proxy = await self.runt.snap.core.reqAhaProxy() + svc = await proxy.getAhaSvc(svcname) + if svc is None: + raise s_exc.NoSuchName(mesg=f'No AHA service found for {svcname}') + + svcinfo = svc.get('svcinfo') + svciden = svcinfo.get('iden') + if svciden is None: + raise s_exc.NoSuchName(mesg=f'Service {svcname} has no iden') + + async for svcname, (ok, info) in proxy.callAhaPeerGenr(svciden, todo, timeout=timeout, skiprun=skiprun): + yield (svcname, (ok, info)) + @s_stormtypes.registry.registerLib class AhaPoolLib(s_stormtypes.Lib): ''' @@ -531,5 +666,220 @@ async def _methPoolSvcDel(self, svcname): } } ''' - } + }, + { + 'name': 'aha.svc.mirror', + 'descr': textwrap.dedent('''\ + Query the AHA services and their mirror relationships. + + Note: non-mirror services are not displayed. + '''), + 'cmdargs': ( + ('--timeout', {'help': 'The timeout in seconds for individual service API calls.', + 'default': 10, 'type': 'int'}), + ('--wait', {'help': 'Whether to wait for the mirrors to sync.', + 'action': 'store_true'}), + ), + 'storm': ''' + init { + $conf = ({ + "columns": [ + {"name": "name", "width": 40}, + {"name": "role", "width": 9}, + {"name": "online", "width": 7}, + {"name": "ready", "width": 6}, + {"name": "host", "width": 16}, + {"name": "port", "width": 8}, + {"name": "version", "width": 12}, + {"name": "nexus idx", "width": 10}, + ], + "separators": { + "row:outline": false, + "column:outline": false, + "header:row": "#", + "data:row": "", + "column": "", + }, + }) + $printer = $lib.tabular.printer($conf) + $timeout = $cmdopts.timeout + $wait = $cmdopts.wait + } + + function get_cell_infos(vname, timeout) { + $cell_infos = ({}) + $todo = $lib.utils.todo('getCellInfo') + for $info in $lib.aha.callPeerApi($vname, $todo, timeout=$timeout) { + $svcname = $info.0 + ($ok, $info) = $info.1 + if $ok { + $cell_infos.$svcname = $info + } + } + return($cell_infos) + } + + function build_status_list(members, cell_infos) { + $group_status = () + for $svc in $members { + $svcinfo = $svc.svcinfo + $svcname = $svc.name + $status = ({ + 'name': $svcname, + 'role': '', + 'online': $lib.dict.has($svcinfo, 'online'), + 'ready': $svcinfo.ready, + 'host': $svcinfo.urlinfo.host, + 'port': $svcinfo.urlinfo.port, + 'version': '', + 'nexs_indx': (0) + }) + if ($cell_infos.$svcname) { + $info = $cell_infos.$svcname + $cell_info = $info.cell + $status.nexs_indx = $cell_info.nexsindx + if ($cell_info.uplink) { + $status.role = 'follower' + } else { + $status.role = 'leader' + } + $status.version = $info.synapse.verstring + } + $group_status.append($status) + } + return($group_status) + } + + function check_sync_status(group_status) { + $indices = $lib.set() + $known_count = (0) + for $status in $group_status { + $indices.add($status.nexs_indx) + $known_count = ($known_count + (1)) + } + if ($lib.len($indices) = 1) { + if ($known_count = $lib.len($group_status)) { + return(true) + } + } + } + + function output_status(vname, group_status, printer) { + $lib.print($printer.header()) + $lib.print($vname) + for $status in $group_status { + if ($status.nexs_indx = 0) { + $status.nexs_indx = '' + } + $row = ( + $status.name, + $status.role, + $status.online, + $status.ready, + $status.host, + $status.port, + $status.version, + $status.nexs_indx + ) + $lib.print($printer.row($row)) + } + } + + $virtual_services = ({}) + $member_servers = ({}) + + for $svc in $lib.aha.list() { + $name = $svc.name + $svcinfo = $svc.svcinfo + $urlinfo = $svcinfo.urlinfo + $hostname = $urlinfo.hostname + + if ($name != $hostname) { + $virtual_services.$name = $svc + } else { + $member_servers.$name = $svc + } + } + + $mirror_groups = ({}) + for ($vname, $vsvc) in $virtual_services { + $vsvc_info = $vsvc.svcinfo + $vsvc_iden = $vsvc_info.iden + $vsvc_leader = $vsvc_info.leader + $vsvc_hostname = $vsvc_info.urlinfo.hostname + + if (not $vsvc_iden or not $vsvc_hostname or not $vsvc_leader) { + continue + } + + $primary_member = $member_servers.$vsvc_hostname + if (not $primary_member) { + continue + } + + $members = ([$primary_member]) + for ($mname, $msvc) in $member_servers { + if ($mname != $vsvc_hostname) { + $msvc_info = $msvc.svcinfo + if ($msvc_info.iden = $vsvc_iden and $msvc_info.leader = $vsvc_leader) { + $members.append($msvc) + } + } + } + + if ($lib.len($members) > 1) { + $mirror_groups.$vname = $members + } + } + + for ($vname, $members) in $mirror_groups { + $cell_infos = $get_cell_infos($vname, $timeout) + $group_status = $build_status_list($members, $cell_infos) + $lib.print('Service Mirror Groups:') + $output_status($vname, $group_status, $printer) + + if $check_sync_status($group_status) { + $lib.print('Group Status: In Sync') + } else { + $lib.print(`Group Status: Out of Sync`) + if $wait { + $leader_nexs = (0) + for $status in $group_status { + if (($status.role = 'leader') and ($status.nexs_indx > 0)) { + $leader_nexs = $status.nexs_indx + } + } + if ($leader_nexs > 0) { + while (true) { + $responses = () + $todo = $lib.utils.todo(waitNexsOffs, ($leader_nexs - 1), timeout=$timeout) + for $info in $lib.aha.callPeerApi($vname, $todo, timeout=$timeout) { + $svcname = $info.0 + ($ok, $info) = $info.1 + if ($ok and $info) { + $responses.append(($svcname, $info)) + } + } + + if ($lib.len($responses) = $lib.len($members)) { + $cell_infos = $get_cell_infos($vname, $timeout) + $group_status = $build_status_list($members, $cell_infos) + + $lib.print('') + $lib.print('Updated status:') + $output_status($vname, $group_status, $printer) + + if $check_sync_status($group_status) { + $lib.print('Group Status: In Sync') + break + } + } + } + } + } + } + $lib.print('') + } + ''' + }, ) diff --git a/synapse/lib/stormlib/utils.py b/synapse/lib/stormlib/utils.py new file mode 100644 index 00000000000..b0ed1f280ad --- /dev/null +++ b/synapse/lib/stormlib/utils.py @@ -0,0 +1,37 @@ +import synapse.lib.stormtypes as s_stormtypes + +@s_stormtypes.registry.registerLib +class LibUtils(s_stormtypes.Lib): + ''' + A Storm library for working with utility functions. + ''' + _storm_locals = ( + {'name': 'todo', + 'desc': ''' + Create a todo tuple of (name, args, kwargs). + ''', + 'type': {'type': 'function', '_funcname': '_todo', + 'args': ( + {'name': '_todoname', 'type': 'str', + 'desc': 'The todo name.'}, + {'name': '*args', 'type': 'any', + 'desc': 'Positional arguments for the todo.'}, + {'name': '**kwargs', 'type': 'any', + 'desc': 'Keyword arguments for the todo.'}, + ), + 'returns': {'type': 'list', 'desc': 'A todo tuple of (name, args, kwargs).'}, + }}, + ) + _storm_lib_path = ('utils',) + + def getObjLocals(self): + return { + 'todo': self._todo, + } + + @s_stormtypes.stormfunc(readonly=True) + async def _todo(self, _todoname, *args, **kwargs): + _todoname = await s_stormtypes.tostr(_todoname) + args = await s_stormtypes.toprim(args) + kwargs = await s_stormtypes.toprim(kwargs) + return (_todoname, args, kwargs) diff --git a/synapse/tests/test_common.py b/synapse/tests/test_common.py index 7badf334980..a0a6307f86f 100644 --- a/synapse/tests/test_common.py +++ b/synapse/tests/test_common.py @@ -24,6 +24,12 @@ async def genr(): self.true(rets[0][0]) self.false(rets[1][0]) + async def one(): + yield 'item' + + rets = [retn async for retn in s_common.waitgenr(one(), timeout=1.0)] + self.eq(rets, [(True, 'item')]) + def test_tuplify(self): tv = ['node', [['test:str', 'test'], {'tags': { diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py index fa2d3740f34..1b0e0e8c6e9 100644 --- a/synapse/tests/test_lib_aha.py +++ b/synapse/tests/test_lib_aha.py @@ -828,6 +828,33 @@ async def test_lib_aha_provision(self): async with await s_telepath.openurl(url) as prox: self.fail(f'Connected to an expired clone URL {url}') # pragma: no cover + async def test_lib_aha_mirrors(self): + + async with self.getTestAha() as aha: + async with await s_base.Base.anit() as base: + with self.getTestDir() as dirn: + user = 'synuser' + dirn00 = s_common.genpath(dirn, 'cell00') + dirn01 = s_common.genpath(dirn, 'cell01') + + core00 = await base.enter_context(self.addSvcToAha(aha, '00.cortex', s_cortex.Cortex, dirn=dirn00, + provinfo={'conf': {'aha:user': user}})) + self.eq(core00.conf.get('aha:user'), user) + + core01 = await base.enter_context(self.addSvcToAha(aha, '01.cortex', s_cortex.Cortex, dirn=dirn01, + conf={'axon': 'aha://cortex...'}, + provinfo={'conf': {'aha:user': user}})) + self.eq(core01.conf.get('aha:user'), user) + + async with aha.getLocalProxy() as ahaproxy: + self.eq(None, await ahaproxy.getAhaSvcMirrors('99.bogus')) + self.len(1, await ahaproxy.getAhaSvcMirrors('00.cortex.synapse')) + self.nn(await ahaproxy.getAhaServer(host=aha._getDnsName(), port=aha.sockaddr[1])) + + todo = s_common.todo('getCellInfo') + res = await asyncio.wait_for(await aha.callAhaSvcApi('00.cortex.synapse', todo, timeout=3), 3) + self.nn(res) + async def test_aha_httpapi(self): async with self.getTestAha() as aha: @@ -905,6 +932,10 @@ async def test_aha_httpapi(self): info = await resp.json() self.eq(info.get('status'), 'err') self.eq(info.get('code'), 'SchemaViolation') + async with sess.post(url, json={'name': 'doom' * 16}) as resp: + info = await resp.json() + self.eq(info.get('status'), 'err') + self.eq(info.get('code'), 'BadArg') # Not an admin await aha.addUser('lowuser', passwd='lowuser') @@ -1093,6 +1124,35 @@ async def test_aha_service_pools(self): self.stormHasNoWarnErr(msgs) self.stormIsInPrint('Removed AHA service pool: pool00.synapse', msgs) + async def test_aha_svc_api_exception(self): + + async with self.getTestAha() as aha: + + async def mockGetAhaSvcProxy(*args, **kwargs): + raise s_exc.SynErr(mesg='proxy error') + + aha.getAhaSvcProxy = mockGetAhaSvcProxy + name = '00.cortex.synapse' + todo = ('bogus', (), {}) + retn = await asyncio.wait_for(await aha.callAhaSvcApi(name, todo), 3) + + self.false(retn[0]) + self.eq('SynErr', retn[1].get('err')) + self.eq('proxy error', retn[1].get('errinfo').get('mesg')) + + bad_info = { + 'urlinfo': { + 'host': 'nonexistent.host', + 'port': 12345, + 'scheme': 'ssl' + } + } + + await aha.addAhaSvc(name, bad_info) + async for ok, info in aha.callAhaPeerGenr(name, ('nonexistent.method', (), {})): + self.false(ok) + self.isin('err', info) + async def test_aha_reprovision(self): with self.withNexusReplay() as stack: @@ -1406,3 +1466,88 @@ async def sleep99(cell): self.false(await cell00.killTask(task01)) self.none(await cell00.getAhaProxy(feats=(('newp', 9),))) + + async def test_lib_aha_peer_api(self): + + async with self.getTestAha() as aha: + + purl00 = await aha.addAhaSvcProv('0.cell') + purl01 = await aha.addAhaSvcProv('1.cell', provinfo={'mirror': '0.cell'}) + purl02 = await aha.addAhaSvcProv('2.cell', provinfo={'mirror': '0.cell'}) + + cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00})) + cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01})) + cell02 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl02})) + + await cell01.sync() + await cell02.sync() + + todo = s_common.todo('getCellInfo') + items = [item async for item in cell00.callPeerApi(todo)] + self.len(2, items) + + async def test_lib_aha_peer_genr(self): + + async with self.getTestAha() as aha: + + purl00 = await aha.addAhaSvcProv('0.cell') + purl01 = await aha.addAhaSvcProv('1.cell', provinfo={'mirror': '0.cell'}) + + cell00 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl00})) + cell01 = await aha.enter_context(self.getTestCell(conf={'aha:provision': purl01})) + + await cell01.sync() + + todo = s_common.todo('getNexusChanges', 0, wait=False) + items = dict([item async for item in cell00.callPeerGenr(todo)]) + self.len(1, items) + + todo = s_common.todo('getNexusChanges', 0, wait=False) + items = dict([item async for item in cell00.callPeerGenr(todo, timeout=2)]) + self.len(1, items) + + async def test_lib_aha_call_aha_peer_api_isactive(self): + + async with self.getTestAha() as aha0: + + async with aha0.waiter(3, 'aha:svcadd', timeout=10): + + conf = {'aha:provision': await aha0.addAhaSvcProv('00.cell')} + cell00 = await aha0.enter_context(self.getTestCell(conf=conf)) + + conf = {'aha:provision': await aha0.addAhaSvcProv('01.cell', {'mirror': 'cell'})} + cell01 = await aha0.enter_context(self.getTestCell(conf=conf)) + + await cell01.sync() + + # test active AHA peer + todo = s_common.todo('getCellInfo') + items = dict([item async for item in aha0.callAhaPeerApi(cell00.iden, todo, timeout=3)]) + self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) + + todo = s_common.todo('getNexusChanges', 0, wait=False) + items = dict([item async for item in aha0.callAhaPeerGenr(cell00.iden, todo, timeout=3)]) + self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) + + async with aha0.getLocalProxy() as proxy0: + purl = await proxy0.addAhaClone('01.aha.loop.vertex.link') + + conf1 = {'clone': purl} + async with self.getTestAha(conf=conf1) as aha1: + + await aha1.sync() + + self.eq(aha0.iden, aha1.iden) + self.nn(aha1.conf.get('mirror')) + + self.true(aha0.isactive) + self.false(aha1.isactive) + + # test non-active AHA peer + todo = s_common.todo('getCellInfo') + items = dict([item async for item in aha1.callAhaPeerApi(cell00.iden, todo, timeout=3)]) + self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) + + todo = s_common.todo('getNexusChanges', 0, wait=False) + items = dict([item async for item in aha1.callAhaPeerGenr(cell00.iden, todo, timeout=3)]) + self.sorteq(items.keys(), ('00.cell.synapse', '01.cell.synapse')) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index a103b592d71..ee3efa8e9f3 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -3284,6 +3284,40 @@ async def test_lib_cell_promote_schism_prevent(self): await cell00.promote(graceful=True) self.isin('02.cell is not the current leader', cm.exception.get('mesg')) + async def test_cell_get_aha_proxy(self): + + async with self.getTestCell() as cell: + + self.none(await cell.getAhaProxy()) + + class MockAhaClient: + def __init__(self, proxy=None): + self._proxy = proxy + + async def proxy(self, timeout=None): + return self._proxy + + with self.getAsyncLoggerStream('synapse.lib.cell', 'AHA client connection failed.') as stream: + cell.ahaclient = MockAhaClient() + self.none(await cell.getAhaProxy()) + self.true(await stream.wait(timeout=1)) + + class MockProxyHasNot: + def _hasTeleFeat(self, name, vers): + return False + + cell.ahaclient = MockAhaClient(proxy=MockProxyHasNot()) + self.none(await cell.getAhaProxy(feats=(('test', 1),))) + + class MockProxyHas: + def _hasTeleFeat(self, name, vers): + return True + + mock_proxy = MockProxyHas() + cell.ahaclient = MockAhaClient(proxy=mock_proxy) + self.eq(await cell.getAhaProxy(), mock_proxy) + self.eq(await cell.getAhaProxy(feats=(('test', 1),)), mock_proxy) + async def test_lib_cell_sadaha(self): async with self.getTestCell() as cell: diff --git a/synapse/tests/test_lib_stormlib_aha.py b/synapse/tests/test_lib_stormlib_aha.py index 47a5edde125..7a8c4f9424f 100644 --- a/synapse/tests/test_lib_stormlib_aha.py +++ b/synapse/tests/test_lib_stormlib_aha.py @@ -6,6 +6,8 @@ import synapse.tests.utils as s_test +import unittest.mock as mock + class AhaLibTest(s_test.SynTest): async def test_stormlib_aha_basics(self): @@ -193,3 +195,189 @@ async def test_stormlib_aha_basics(self): with self.raises(s_exc.NoSuchName): await core00.callStorm('$lib.aha.del(axon...)') + + async def test_stormlib_aha_mirror(self): + + async with self.getTestAha() as aha: + + with self.getTestDir() as dirn: + + dirn00 = s_common.genpath(dirn, 'cell00') + dirn01 = s_common.genpath(dirn, 'cell01') + dirn02 = s_common.genpath(dirn, 'cell02') + + async with aha.waiter(3, 'aha:svcadd', timeout=10): + + cell00 = await aha.enter_context(self.addSvcToAha(aha, '00.cell', s_cell.Cell, dirn=dirn00)) + cell01 = await aha.enter_context(self.addSvcToAha(aha, '01.cell', s_cell.Cell, dirn=dirn01, + provinfo={'mirror': 'cell'})) + core00 = await aha.enter_context(self.addSvcToAha(aha, 'core', s_cortex.Cortex, dirn=dirn02)) + await cell01.sync() + + # PeerGenr + resp = await core00.callStorm(''' + $resps = $lib.list() + $todo = $lib.utils.todo('getTasks') + for ($name, $info) in $lib.aha.callPeerGenr(cell..., $todo) { + $resps.append(($name, $info)) + } + return($resps) + ''') + self.len(0, resp) + + resp = await core00.callStorm(''' + $resps = $lib.list() + $todo = $lib.utils.todo('getNexusChanges', (0), wait=(false)) + for ($name, $info) in $lib.aha.callPeerGenr(cell..., $todo) { + $resps.append(($name, $info)) + } + return($resps) + ''') + self.len(4, resp) + + cell00_rid = (await cell00.getCellInfo())['cell']['run'] + resp = await core00.callStorm(''' + $resps = $lib.list() + $todo = $lib.utils.todo('getNexusChanges', (0), wait=(false)) + for $info in $lib.aha.callPeerGenr(cell..., $todo, skiprun=$skiprun) { + $resps.append($info) + } + return($resps) + ''', opts={'vars': {'skiprun': cell00_rid}}) + self.len(2, resp) + + await self.asyncraises(s_exc.NoSuchName, core00.callStorm(''' + $todo = $lib.utils.todo('getTasks') + $lib.aha.callPeerGenr(null, $todo) + ''')) + + # PeerApi + resp = await core00.callStorm(''' + $resps = $lib.list() + $todo = $lib.utils.todo('getCellInfo') + for ($name, $info) in $lib.aha.callPeerApi(cell..., $todo) { + $resps.append(($name, $info)) + } + return($resps) + ''') + self.len(2, resp) + self.eq(resp[0][0], '00.cell.synapse') + self.eq(resp[0][1][0], True) + self.isinstance(resp[0][1][1], dict) + + resp = await core00.callStorm(''' + $resps = $lib.list() + $todo = $lib.utils.todo(getCellInfo) + for ($name, $info) in $lib.aha.callPeerApi(cell..., $todo) { + $resps.append(($name, $info)) + } + return($resps) + ''') + self.len(2, resp) + + resp = await core00.callStorm(''' + $resps = $lib.list() + $todo = $lib.utils.todo(getCellInfo) + for ($name, $info) in $lib.aha.callPeerApi(cell..., $todo, skiprun=$skiprun) { + $resps.append(($name, $info)) + } + return($resps) + ''', opts={'vars': {'skiprun': cell00_rid}}) + self.len(1, resp) + + resp = await core00.callStorm(''' + $resps = $lib.list() + $todo = $lib.utils.todo('getCellInfo') + for ($name, $info) in $lib.aha.callPeerApi(cell..., $todo, timeout=(10)) { + $resps.append(($name, $info)) + } + return($resps) + ''') + self.len(2, resp) + + await self.asyncraises(s_exc.NoSuchName, core00.callStorm(''' + $todo = $lib.utils.todo('getCellInfo') + $lib.aha.callPeerApi(newp..., $todo) + ''')) + + await self.asyncraises(s_exc.NoSuchName, core00.callStorm(''' + $todo = $lib.utils.todo('getCellInfo') + $lib.aha.callPeerApi(null, $todo) + ''')) + + await self.asyncraises(s_exc.NoSuchMeth, core00.callStorm(''' + $todo = $lib.utils.todo('bogusMethod') + for $info in $lib.aha.callPeerApi(cell..., $todo) { + ($ok, $info) = $info.1 + if (not $ok) { + $lib.raise($info.err, $info.errmsg) + } + } + ''')) + + await aha.addAhaSvc('noiden.cell', info={'urlinfo': {'scheme': 'tcp', + 'host': '0.0.0.0', + 'port': '3030'}}, + network='synapse') + await self.asyncraises(s_exc.NoSuchName, core00.callStorm(''' + $todo = $lib.utils.todo('getTasks') + $lib.aha.callPeerGenr(noiden.cell..., $todo) + ''')) + await self.asyncraises(s_exc.NoSuchName, core00.callStorm(''' + $todo = $lib.utils.todo('getCellInfo') + $lib.aha.callPeerApi(noiden.cell..., $todo) + ''')) + + msgs = await core00.stormlist('aha.svc.mirror') + self.stormIsInPrint('Service Mirror Groups:', msgs) + self.stormIsInPrint('cell.synapse', msgs) + self.stormIsInPrint('00.cell.synapse', msgs) + self.stormIsInPrint('01.cell.synapse', msgs) + self.stormIsInPrint('Group Status: In Sync', msgs) + + msgs = await core00.stormlist('aha.svc.mirror --timeout 30') + self.stormIsInPrint('Service Mirror Groups:', msgs) + self.stormIsInPrint('Group Status: In Sync', msgs) + + async def mockCellInfo(): + return { + 'cell': {'ready': True, 'nexsindx': 10, 'uplink': None}, + 'synapse': {'verstring': '2.190.0'}, + } + + async def mockOutOfSyncCellInfo(): + return { + 'cell': {'ready': True, 'nexsindx': 5, 'uplink': cell00.iden}, + 'synapse': {'verstring': '2.190.0'}, + } + + with mock.patch.object(cell00, 'getCellInfo', mockCellInfo): + with mock.patch.object(cell01, 'getCellInfo', mockOutOfSyncCellInfo): + async def mock_call_aha(*args, **kwargs): + todo = args[1] + if todo[0] == 'waitNexsOffs': + yield ('00.cell.synapse', (True, True)) + yield ('01.cell.synapse', (True, True)) + elif todo[0] == 'getCellInfo': + if not hasattr(mock_call_aha, 'called'): + mock_call_aha.called = True + yield ('00.cell.synapse', (True, await mockCellInfo())) + yield ('01.cell.synapse', (True, await mockOutOfSyncCellInfo())) + else: + yield ('00.cell.synapse', (True, await mockCellInfo())) + yield ('01.cell.synapse', (True, await mockCellInfo())) + + with mock.patch.object(aha, 'callAhaPeerApi', mock_call_aha): + msgs = await core00.stormlist('aha.svc.mirror --wait') + self.stormIsInPrint('Group Status: Out of Sync', msgs) + self.stormIsInPrint('Updated status:', msgs) + self.stormIsInPrint('Group Status: In Sync', msgs) + + with mock.patch.object(cell00, 'getCellInfo', mockCellInfo): + with mock.patch.object(cell01, 'getCellInfo', mockOutOfSyncCellInfo): + msgs = await core00.stormlist('aha.svc.mirror --timeout 1') + self.stormIsInPrint('Group Status: Out of Sync', msgs) + + await aha.delAhaSvc('00.cell', network='synapse') + msgs = await core00.stormlist('aha.svc.mirror') + self.stormNotInPrint('Service Mirror Groups:', msgs) diff --git a/synapse/tests/test_lib_stormlib_utils.py b/synapse/tests/test_lib_stormlib_utils.py new file mode 100644 index 00000000000..68d3a316961 --- /dev/null +++ b/synapse/tests/test_lib_stormlib_utils.py @@ -0,0 +1,14 @@ +import synapse.exc as s_exc +import synapse.tests.utils as s_test + +class UtilsTest(s_test.SynTest): + + async def test_lib_stormlib_utils_todo(self): + + async with self.getTestCore() as core: + + valu = await core.callStorm('return($lib.utils.todo(foo))') + self.eq(valu, ('foo', (), {})) + + valu = await core.callStorm('return($lib.utils.todo(fooName, arg1, arg2, keyword=bar, anotherkeyword=hehe))') + self.eq(valu, ('fooName', ('arg1', 'arg2'), {'keyword': 'bar', 'anotherkeyword': 'hehe'})) diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py index 8938a276a38..b7b866a581e 100644 --- a/synapse/tests/test_tools_aha.py +++ b/synapse/tests/test_tools_aha.py @@ -3,14 +3,17 @@ from unittest import mock +import synapse.exc as s_exc import synapse.common as s_common import synapse.lib.cell as s_cell +import synapse.lib.version as s_version import synapse.tests.utils as s_t_utils import synapse.tools.aha.list as s_a_list import synapse.tools.aha.clone as s_a_clone import synapse.tools.aha.enroll as s_a_enroll +import synapse.tools.aha.mirror as s_a_mirror import synapse.tools.aha.easycert as s_a_easycert import synapse.tools.aha.provision.user as s_a_provision_user @@ -170,3 +173,192 @@ async def test_aha_enroll(self): teleyaml = s_common.yamlload(syndir, 'telepath.yaml') self.eq(teleyaml.get('version'), 1) + + async def test_aha_mirror(self): + + async with self.getTestAha() as aha: + + base_svcinfo = { + 'iden': 'test_iden', + 'leader': 'leader', + 'urlinfo': { + 'scheme': 'tcp', + 'host': '127.0.0.1', + 'port': 0, + 'hostname': 'test.host' + } + } + + conf_no_iden = {'aha:provision': await aha.addAhaSvcProv('no.iden')} + async with self.getTestCell(s_cell.Cell, conf=conf_no_iden) as cell_no_iden: + svcinfo = {k: v for k, v in base_svcinfo.items() if k != 'iden'} + await aha.addAhaSvc('no.iden', svcinfo) + + argv = ['--url', aha.getLocalUrl()] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 0) + outp.expect('Service Mirror Groups:') + self.notin('no.iden', str(outp)) + + conf_no_host = {'aha:provision': await aha.addAhaSvcProv('no.host')} + async with self.getTestCell(s_cell.Cell, conf=conf_no_host) as cell_no_host: + svcinfo = dict(base_svcinfo) + svcinfo['urlinfo'] = {k: v for k, v in base_svcinfo['urlinfo'].items() if k != 'hostname'} + await aha.addAhaSvc('no.host', svcinfo) + + argv = ['--url', aha.getLocalUrl()] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 0) + outp.expect('Service Mirror Groups:') + self.notin('no.host', str(outp)) + + conf_no_leader = {'aha:provision': await aha.addAhaSvcProv('no.leader')} + async with self.getTestCell(s_cell.Cell, conf=conf_no_leader) as cell_no_leader: + svcinfo = {k: v for k, v in base_svcinfo.items() if k != 'leader'} + await aha.addAhaSvc('no.leader', svcinfo) + + argv = ['--url', aha.getLocalUrl()] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 0) + outp.expect('Service Mirror Groups:') + self.notin('no.leader', str(outp)) + + conf_no_primary = {'aha:provision': await aha.addAhaSvcProv('no.primary')} + async with self.getTestCell(s_cell.Cell, conf=conf_no_primary) as cell_no_primary: + svcinfo = dict(base_svcinfo) + svcinfo['urlinfo']['hostname'] = 'nonexistent.host' + await aha.addAhaSvc('no.primary', svcinfo) + + async with aha.waiter(3, 'aha:svcadd', timeout=10): + + conf = {'aha:provision': await aha.addAhaSvcProv('00.cell')} + cell00 = await aha.enter_context(self.getTestCell(conf=conf)) + + conf = {'aha:provision': await aha.addAhaSvcProv('01.cell', {'mirror': 'cell'})} + cell01 = await aha.enter_context(self.getTestCell(conf=conf)) + + await cell01.sync() + + ahaurl = aha.getLocalUrl() + + argv = ['--url', ahaurl] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 0) + outp.expect('Service Mirror Groups:') + outp.expect('00.cell.synapse') + outp.expect('01.cell.synapse') + outp.expect('Group Status: In Sync') + + argv = ['--url', ahaurl, '--timeout', '30'] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 0) + + with mock.patch('synapse.telepath.Proxy._hasTeleFeat', + return_value=False): + argv = ['--url', ahaurl] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 1) + outp.expect(f'Service at {ahaurl} does not support the required callpeers feature.') + + with mock.patch('synapse.telepath.Proxy._hasTeleFeat', + side_effect=s_exc.NoSuchMeth(name='_hasTeleFeat')): + argv = ['--url', ahaurl] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 1) + outp.expect(f'Service at {ahaurl} does not support the required callpeers feature.') + + argv = ['--url', 'tcp://newp:1234/'] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 1) + outp.expect('ERROR:') + + async def mockCellInfo(): + return { + 'cell': {'ready': True, 'nexsindx': 10, 'uplink': None}, + 'synapse': {'verstring': s_version.verstring}, + } + + async def mockOutOfSyncCellInfo(): + return { + 'cell': {'ready': True, 'nexsindx': 5, 'uplink': cell00.iden}, + 'synapse': {'verstring': s_version.verstring}, + } + + with mock.patch.object(cell00, 'getCellInfo', mockCellInfo): + with mock.patch.object(cell01, 'getCellInfo', mockOutOfSyncCellInfo): + async def mock_call_aha(*args, **kwargs): + todo = args[1] + if todo[0] == 'waitNexsOffs': + yield ('00.cell.synapse', (True, True)) + yield ('01.cell.synapse', (True, True)) + elif todo[0] == 'getCellInfo': + if not hasattr(mock_call_aha, 'called'): + mock_call_aha.called = True + yield ('00.cell.synapse', (True, await mockCellInfo())) + yield ('01.cell.synapse', (True, await mockOutOfSyncCellInfo())) + else: + yield ('00.cell.synapse', (True, await mockCellInfo())) + yield ('01.cell.synapse', (True, await mockCellInfo())) + + with mock.patch.object(aha, 'callAhaPeerApi', mock_call_aha): + argv = ['--url', ahaurl, '--wait'] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 0) + outp.expect('Group Status: Out of Sync') + outp.expect('Updated status:') + outp.expect('Group Status: In Sync') + + with mock.patch.object(cell00, 'getCellInfo', mockCellInfo): + with mock.patch.object(cell01, 'getCellInfo', mockOutOfSyncCellInfo): + argv = ['--url', ahaurl, '--timeout', '1'] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 0) + outp.expect('Group Status: Out of Sync') + + async with self.getTestCore() as core: + curl = core.getLocalUrl() + argv = ['--url', curl] + with mock.patch('synapse.telepath.Proxy._hasTeleFeat', + return_value=True): + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(1, retn) + outp.expect(f'Service at {curl} is not an Aha server') + + async with aha.waiter(1, 'aha:svcadd', timeout=10): + + conf = {'aha:provision': await aha.addAhaSvcProv('02.cell', {'mirror': 'cell'})} + cell02 = await aha.enter_context(self.getTestCell(conf=conf)) + await cell02.sync() + + async def mock_failed_api(*args, **kwargs): + yield ('00.cell.synapse', (True, {'cell': {'ready': True, 'nexsindx': 10}})) + yield ('01.cell.synapse', (False, 'error')) + yield ('02.cell.synapse', (True, {'cell': {'ready': True, 'nexsindx': 12}})) + + with mock.patch.object(aha, 'callAhaPeerApi', mock_failed_api): + argv = ['--url', ahaurl, '--timeout', '1'] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + outp.expect('00.cell.synapse leader True True 127.0.0.1', whitespace=False) + outp.expect('nexsindx 10', whitespace=False) + outp.expect('02.cell.synapse leader True True 127.0.0.1', whitespace=False) + outp.expect('nexsindx 12', whitespace=False) + outp.expect('01.cell.synapse True True', whitespace=False) + outp.expect(' ', whitespace=False) + + self.eq(s_a_mirror.timeout_type('30'), 30) + self.eq(s_a_mirror.timeout_type('0'), 0) + + with self.raises(s_exc.BadArg) as cm: + s_a_mirror.timeout_type('-1') + self.isin('is not a valid non-negative integer', cm.exception.get('mesg')) + + with self.raises(s_exc.BadArg) as cm: + s_a_mirror.timeout_type('foo') + self.isin('is not a valid non-negative integer', cm.exception.get('mesg')) + + synerr = s_exc.SynErr(mesg='Oof') + with mock.patch('synapse.telepath.openurl', side_effect=synerr): + argv = ['--url', 'tcp://test:1234/'] + retn, outp = await self.execToolMain(s_a_mirror.main, argv) + self.eq(retn, 1) + outp.expect('ERROR: Oof') diff --git a/synapse/tools/aha/mirror.py b/synapse/tools/aha/mirror.py new file mode 100644 index 00000000000..77f5fa3c5b1 --- /dev/null +++ b/synapse/tools/aha/mirror.py @@ -0,0 +1,193 @@ +import sys +import asyncio +import argparse + +import synapse.exc as s_exc +import synapse.common as s_common +import synapse.telepath as s_telepath + +import synapse.lib.output as s_output +import synapse.lib.version as s_version + +descr = ''' +Query the Aha server for the service cluster status of mirrors. + +Examples: + + python -m synapse.tools.aha.mirror --timeout 30 + +''' + +async def get_cell_infos(prox, iden, members, timeout): + cell_infos = {} + if iden is not None: + todo = s_common.todo('getCellInfo') + async for svcname, (ok, info) in prox.callAhaPeerApi(iden, todo, timeout=timeout): + if not ok: + continue + cell_infos[svcname] = info + return cell_infos + +def build_status_list(members, cell_infos): + group_status = [] + for svc in members: + svcname = svc.get('name') + svcinfo = svc.get('svcinfo', {}) + status = { + 'name': svcname, + 'role': '', + 'online': str('online' in svcinfo), + 'ready': 'True', + 'host': svcinfo.get('urlinfo', {}).get('host', ''), + 'port': str(svcinfo.get('urlinfo', {}).get('port', '')), + 'version': '', + 'nexs_indx': 0 + } + if svcname in cell_infos: + info = cell_infos[svcname] + cell_info = info.get('cell', {}) + status.update({ + 'nexs_indx': cell_info.get('nexsindx', 0), + 'role': 'follower' if cell_info.get('uplink') else 'leader', + 'version': str(info.get('synapse', {}).get('verstring', '')), + 'online': 'True', + 'ready': str(cell_info.get('ready', False)) + }) + group_status.append(status) + return group_status + +def output_status(outp, vname, group_status): + header = ' {:<40} {:<10} {:<8} {:<7} {:<16} {:<9} {:<12} {:<10}'.format( + 'name', 'role', 'online', 'ready', 'host', 'port', 'version', 'nexus idx') + outp.printf(header) + outp.printf('#' * 120) + outp.printf(vname) + for status in group_status: + if status['nexs_indx'] == 0: + status['nexs_indx'] = '' + line = ' {name:<40} {role:<10} {online:<8} {ready:<7} {host:<16} {port:<9} {version:<12} {nexs_indx:<10}'.format(**status) + outp.printf(line) + +def check_sync_status(group_status): + indices = {status['nexs_indx'] for status in group_status} + known_count = sum(1 for status in group_status) + return len(indices) == 1 and known_count == len(group_status) + +def timeout_type(valu): + try: + ivalu = int(valu) + if ivalu < 0: + raise ValueError + except ValueError: + raise s_exc.BadArg(mesg=f"{valu} is not a valid non-negative integer") + return ivalu + +async def main(argv, outp=s_output.stdout): + + pars = argparse.ArgumentParser(prog='synapse.tools.aha.mirror', description=descr, + formatter_class=argparse.RawDescriptionHelpFormatter) + + pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL to connect to the AHA service.') + pars.add_argument('--timeout', type=timeout_type, default=10, help='The timeout in seconds for individual service API calls') + pars.add_argument('--wait', action='store_true', help='Whether to wait for the mirrors to sync.') + opts = pars.parse_args(argv) + + async with s_telepath.withTeleEnv(): + try: + async with await s_telepath.openurl(opts.url) as prox: + try: + if not prox._hasTeleFeat('callpeers', vers=1): + outp.printf(f'Service at {opts.url} does not support the required callpeers feature.') + return 1 + except s_exc.NoSuchMeth: + outp.printf(f'Service at {opts.url} does not support the required callpeers feature.') + return 1 + classes = prox._getClasses() + if 'synapse.lib.aha.AhaApi' not in classes: + outp.printf(f'Service at {opts.url} is not an Aha server') + return 1 + + virtual_services, member_servers = {}, {} + async for svc in prox.getAhaSvcs(): + name = svc.get('name', '') + svcinfo = svc.get('svcinfo', {}) + urlinfo = svcinfo.get('urlinfo', {}) + hostname = urlinfo.get('hostname', '') + + if name != hostname: + virtual_services[name] = svc + else: + member_servers[name] = svc + + mirror_groups = {} + for vname, vsvc in virtual_services.items(): + vsvc_info = vsvc.get('svcinfo', {}) + vsvc_iden = vsvc_info.get('iden') + vsvc_leader = vsvc_info.get('leader') + vsvc_hostname = vsvc_info.get('urlinfo', {}).get('hostname', '') + + if not vsvc_iden or not vsvc_hostname or not vsvc_leader: + continue + + primary_member = member_servers.get(vsvc_hostname) + if not primary_member: + continue + + members = [primary_member] + [ + msvc for mname, msvc in member_servers.items() + if mname != vsvc_hostname and + msvc.get('svcinfo', {}).get('iden') == vsvc_iden and + msvc.get('svcinfo', {}).get('leader') == vsvc_leader + ] + + if len(members) > 1: + mirror_groups[vname] = members + + outp.printf('Service Mirror Groups:') + for vname, members in mirror_groups.items(): + iden = members[0].get('svcinfo', {}).get('iden') + + cell_infos = await get_cell_infos(prox, iden, members, opts.timeout) + group_status = build_status_list(members, cell_infos) + output_status(outp, vname, group_status) + + if check_sync_status(group_status): + outp.printf('Group Status: In Sync') + else: + outp.printf(f'Group Status: Out of Sync') + if opts.wait: + leader_nexs = None + for status in group_status: + if status['role'] == 'leader' and isinstance(status['nexs_indx'], int): + leader_nexs = status['nexs_indx'] + + if leader_nexs is not None: + while True: + responses = [] + todo = s_common.todo('waitNexsOffs', leader_nexs - 1, timeout=opts.timeout) + async for svcname, (ok, info) in prox.callAhaPeerApi(iden, todo, timeout=opts.timeout): + if ok and info: + responses.append((svcname, info)) + + if len(responses) == len(members): + cell_infos = await get_cell_infos(prox, iden, members, opts.timeout) + group_status = build_status_list(members, cell_infos) + + outp.printf('\nUpdated status:') + output_status(outp, vname, group_status) + + if check_sync_status(group_status): + outp.printf('Group Status: In Sync') + break + + return 0 + + except Exception as e: + mesg = repr(e) + if isinstance(e, s_exc.SynErr): + mesg = e.errinfo.get('mesg', repr(e)) + outp.printf(f'ERROR: {mesg}') + return 1 + +if __name__ == '__main__': # pragma: no cover + sys.exit(asyncio.run(main(sys.argv[1:]))) From e2f40f7fddd4d546f0977a5e0a41c502b4ca38c2 Mon Sep 17 00:00:00 2001 From: visi Date: Thu, 23 Jan 2025 09:45:54 -0500 Subject: [PATCH 85/86] wip --- synapse/lib/aha.py | 9 --------- synapse/lib/cell.py | 10 +++++----- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 98c45d0dbd3..511e6fcf909 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -1024,15 +1024,6 @@ async def addAhaSvc(self, name, info, network=None): await self.fire('aha:svcadd', svcinfo=svcinfo) await self.fire(f'aha:svcadd:{svcfull}', svcinfo=svcinfo) - async def reqAhaSvcProxy(self, svcdef, timeout=None): - - proxy = await self.getAhaSvcProxy(svcdef, timeout=timeout) - if proxy is not None: - return proxy - - mesg = f'The service is not ready {svcdef.get("name")}.' - raise s_exc.NotReady(mesg=mesg) - async def getAhaSvcProxy(self, svcdef, timeout=None): client = await self.getAhaSvcClient(svcdef) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 78c6b0b6dca..1012342ed33 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -4457,7 +4457,7 @@ async def callPeerApi(self, todo, timeout=None): Yield responses from our peers via the AHA gather call API. ''' proxy = await self.getAhaProxy(timeout=timeout, feats=(feat_aha_callpeers_v1,)) - if proxy is None: + if proxy is None: # pragma: no cover return async for item in proxy.callAhaPeerApi(self.iden, todo, timeout=timeout, skiprun=self.runid): @@ -4468,7 +4468,7 @@ async def callPeerGenr(self, todo, timeout=None): Yield responses from invoking a generator via the AHA gather API. ''' proxy = await self.getAhaProxy(timeout=timeout, feats=(feat_aha_callpeers_v1,)) - if proxy is None: + if proxy is None: # pragma: no cover return async for item in proxy.callAhaPeerGenr(self.iden, todo, timeout=timeout, skiprun=self.runid): @@ -4490,7 +4490,7 @@ async def getTasks(self, peers=True, timeout=None): # we can ignore the yielded aha names because we embed it in the task async for (ahasvc, (ok, retn)) in self.callPeerGenr(todo, timeout=timeout): - if not ok: + if not ok: # pragma: no cover logger.warning(f'getTasks() on {ahasvc} failed: {retn}') continue @@ -4510,7 +4510,7 @@ async def getTask(self, iden, peers=True, timeout=None): todo = s_common.todo('getTask', iden, peers=False, timeout=timeout) async for ahasvc, (ok, retn) in self.callPeerApi(todo, timeout=timeout): - if not ok: + if not ok: # pragma: no cover logger.warning(f'getTask() on {ahasvc} failed: {retn}') continue @@ -4530,7 +4530,7 @@ async def killTask(self, iden, peers=True, timeout=None): todo = s_common.todo('killTask', iden, peers=False, timeout=timeout) async for ahasvc, (ok, retn) in self.callPeerApi(todo, timeout=timeout): - if not ok: + if not ok: # pragma: no cover logger.warning(f'killTask() on {ahasvc} failed: {retn}') continue From 098c611d97030923ec68cde59ce6fb3d0b55761d Mon Sep 17 00:00:00 2001 From: visi Date: Thu, 23 Jan 2025 10:45:22 -0500 Subject: [PATCH 86/86] feedback --- synapse/lib/aha.py | 1 - 1 file changed, 1 deletion(-) diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py index 511e6fcf909..95f4c8a11d4 100644 --- a/synapse/lib/aha.py +++ b/synapse/lib/aha.py @@ -3,7 +3,6 @@ import random import asyncio import logging -import contextlib import collections import cryptography.x509 as c_x509