Skip to content

Commit

Permalink
Remove unknown roles/users during migration, additional logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@Cisphyx
Cisphyx committed Jun 10, 2024
1 parent 85c8425 commit d186422
Show file tree
Hide file tree
Showing 4 changed files with 111 additions and 66 deletions.
4 changes: 4 additions & 0 deletions synapse/axon.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -862,11 +862,15 @@ async def _axonHealth(self, health):
health.update('axon', 'nominal', '', data=await self.metrics())

async def _migrateAxonMetrics(self):
logger.warning('migrating Axon metrics data out of hive')

async with await self.hive.open(('axon', 'metrics')) as hivenode:
axonmetrics = await hivenode.dict()
self.axonmetrics.set('size:bytes', axonmetrics.get('size:bytes', 0))
self.axonmetrics.set('file:count', axonmetrics.get('file:count', 0))

logger.warning('...Axon metrics migration complete!')

async def _initBlobStor(self):

self.byterange = True
Expand Down
4 changes: 4 additions & 0 deletions synapse/cortex.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1016,6 +1016,8 @@ async def initServiceStorage(self):

async def _storCortexHiveMigration(self):

logger.warning('migrating Cortex data out of hive')

viewdefs = self.cortexdata.getSubKeyVal('view:info:')
async with await self.hive.open(('cortex', 'views')) as viewnodes:
for iden, node in viewnodes:
Expand Down Expand Up @@ -1070,6 +1072,8 @@ async def _storCortexHiveMigration(self):
for name, node in hivenode:
subkv.set(name, node.valu)

logger.warning('...Cortex data migration complete!')

async def _viewNomergeToProtected(self):
for view in self.views.values():
nomerge = view.info.get('nomerge', False)
Expand Down
99 changes: 62 additions & 37 deletions synapse/lib/cell.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1289,6 +1289,8 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
await self.initServiceNetwork()

async def _storCellHiveMigration(self):
logger.warning(f'migrating Cell ({self.getCellType()}) info out of hive')

async with await self.hive.open(('cellvers',)) as versnode:
versdict = await versnode.dict()
for key, valu in versdict.items():
Expand All @@ -1299,48 +1301,18 @@ async def _storCellHiveMigration(self):
for key, valu in infodict.items():
self.cellinfo.set(key, valu)

logger.warning(f'...Cell ({self.getCellType()}) info migration complete!')

async def _storCellAuthMigration(self):
if self.conf.get('auth:ctor') is not None:
return

logger.warning(f'migrating Cell ({self.getCellType()}) auth out of hive')

authkv = self.slab.getSafeKeyVal('auth')

async with await self.hive.open(('auth',)) as rootnode:

gateroles = {}
gateusers = {}

gatekv = authkv.getSubKeyVal('gate:info:')
async with await rootnode.open(('authgates',)) as authgates:
for gateiden, node in authgates:
gateinfo = {
'iden': gateiden,
'type': node.valu
}
gatekv.set(gateiden, gateinfo)

async with await node.open(('users',)) as usernodes:
for useriden, usernode in usernodes:
userinfo = await usernode.dict()
userdict = userinfo.pack()
userdict['iden'] = useriden
userdict.setdefault('admin', False)
authkv.set(f'gate:{gateiden}:user:{useriden}', userdict)

gateusers.setdefault(useriden, {})
gateusers[useriden][gateiden] = userdict

async with await node.open(('roles',)) as rolenodes:
for roleiden, rolenode in rolenodes:
roleinfo = await rolenode.dict()
roledict = roleinfo.pack()
roledict['iden'] = roleiden
roledict.setdefault('admin', False)
authkv.set(f'gate:{gateiden}:role:{roleiden}', roledict)

gateroles.setdefault(roleiden, {})
gateroles[roleiden][gateiden] = roledict

rolekv = authkv.getSubKeyVal('role:info:')
rolenamekv = authkv.getSubKeyVal('role:name:')

Expand All @@ -1351,7 +1323,7 @@ async def _storCellAuthMigration(self):

roleinfo['iden'] = iden
roleinfo['name'] = node.valu
roleinfo['authgates'] = gateroles.get(iden, {})
roleinfo['authgates'] = {}
roleinfo.setdefault('admin', False)
roleinfo.setdefault('rules', ())

Expand All @@ -1368,14 +1340,24 @@ async def _storCellAuthMigration(self):

userinfo['iden'] = iden
userinfo['name'] = node.valu
userinfo['authgates'] = gateusers.get(iden, {})
userinfo['authgates'] = {}
userinfo.setdefault('admin', False)
userinfo.setdefault('roles', ())
userinfo.setdefault('ruler', ())
userinfo.setdefault('locked', False)
userinfo.setdefault('passwd', None)
userinfo.setdefault('archived', False)

realroles = []
for userrole in userinfo.get('roles', ()):
if rolekv.get(userrole) is None:
mesg = f'Unknown role {userrole} on user {iden} during migration, ignoring.'
logger.warning(mesg)
continue

realroles.append(userrole)

userinfo['roles'] = tuple(realroles)

userkv.set(iden, userinfo)
usernamekv.set(node.valu, iden)

Expand All @@ -1389,6 +1371,49 @@ async def _storCellAuthMigration(self):
for name, profnode in profnodes:
profkv.set(name, profnode.valu)

gatekv = authkv.getSubKeyVal('gate:info:')
async with await rootnode.open(('authgates',)) as authgates:
for gateiden, node in authgates:
gateinfo = {
'iden': gateiden,
'type': node.valu
}
gatekv.set(gateiden, gateinfo)

async with await node.open(('users',)) as usernodes:
for useriden, usernode in usernodes:
if (user := userkv.get(useriden)) is None:
mesg = f'Unknown user {useriden} on gate {gateiden} during migration, ignoring.'
logger.warning(mesg)
continue

userinfo = await usernode.dict()
userdict = userinfo.pack()
userdict['iden'] = useriden
userdict.setdefault('admin', False)
authkv.set(f'gate:{gateiden}:user:{useriden}', userdict)

user['authgates'][gateiden] = userdict
userkv.set(useriden, user)

async with await node.open(('roles',)) as rolenodes:
for roleiden, rolenode in rolenodes:
if (role := rolekv.get(roleiden)) is None:
mesg = f'Unknown role {roleiden} on gate {gateiden} during migration, ignoring.'
logger.warning(mesg)
continue

roleinfo = await rolenode.dict()
roledict = roleinfo.pack()
roledict['iden'] = roleiden
roledict.setdefault('admin', False)
authkv.set(f'gate:{gateiden}:role:{roleiden}', roledict)

role['authgates'][gateiden] = roledict
rolekv.set(roleiden, role)

logger.warning(f'...Cell ({self.getCellType()}) auth migration complete!')

def getPermDef(self, perm):
perm = tuple(perm)
if self.permlook is None:
Expand Down
70 changes: 41 additions & 29 deletions synapse/tests/test_lib_cell.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2680,45 +2680,57 @@ async def test_cell_nexus_compat(self):
self.none(await core0.callStorm('return($lib.user.profile.get(bar))'))

async def test_cell_hive_migration(self):
async with self.getRegrCore('hive-migration') as core:
visi = await core.auth.getUserByName('visi')
asvisi = {'user': visi.iden}

valu = await core.callStorm('return($lib.user.vars.get(foovar))', opts=asvisi)
self.eq('barvalu', valu)
with self.getAsyncLoggerStream('synapse.lib.cell') as stream:

valu = await core.callStorm('return($lib.user.profile.get(fooprof))', opts=asvisi)
self.eq('barprof', valu)
async with self.getRegrCore('hive-migration') as core:
visi = await core.auth.getUserByName('visi')
asvisi = {'user': visi.iden}

msgs = await core.stormlist('cron.list')
self.stormIsInPrint('visi 4f179a4e', msgs)
self.stormIsInPrint('[tel:mob:telem=*]', msgs)
valu = await core.callStorm('return($lib.user.vars.get(foovar))', opts=asvisi)
self.eq('barvalu', valu)

msgs = await core.stormlist('dmon.list')
self.stormIsInPrint('5baea074cdab6631a86787faa2ca6586: (foodmon ): running', msgs)
valu = await core.callStorm('return($lib.user.profile.get(fooprof))', opts=asvisi)
self.eq('barprof', valu)

msgs = await core.stormlist('trigger.list')
self.stormIsInPrint('visi 64e59ed3aa2eff46325972f636cc26a1', msgs)
self.stormIsInPrint('[ +#count test:str=$tag ]', msgs)
msgs = await core.stormlist('cron.list')
self.stormIsInPrint('visi 8437c35a', msgs)
self.stormIsInPrint('[tel:mob:telem=*]', msgs)

msgs = await core.stormlist('testcmd0 foo')
self.stormIsInPrint('foo haha', msgs)
msgs = await core.stormlist('dmon.list')
self.stormIsInPrint('0973342044469bc40b577969028c5079: (foodmon ): running', msgs)

msgs = await core.stormlist('testcmd1')
self.stormIsInPrint('hello', msgs)
msgs = await core.stormlist('trigger.list')
self.stormIsInPrint('visi 27f5dc524e7c3ee8685816ddf6ca1326', msgs)
self.stormIsInPrint('[ +#count test:str=$tag ]', msgs)

msgs = await core.stormlist('model.deprecated.locks')
self.stormIsInPrint('ou:hasalias', msgs)
msgs = await core.stormlist('testcmd0 foo')
self.stormIsInPrint('foo haha', msgs)

nodes = await core.nodes('_visi:int')
self.len(1, nodes)
node = nodes[0]
self.eq(node.get('tick'), 1577836800000,)
self.eq(node.get('._woot'), 5)
self.nn(node.getTagProp('test', 'score'), 6)
msgs = await core.stormlist('testcmd1')
self.stormIsInPrint('hello', msgs)

with self.raises(s_exc.BadTag):
await core.nodes('[ it:dev:str=foo +#test.newp ]')
msgs = await core.stormlist('model.deprecated.locks')
self.stormIsInPrint('ou:hasalias', msgs)

nodes = await core.nodes('_visi:int')
self.len(1, nodes)
node = nodes[0]
self.eq(node.get('tick'), 1577836800000,)
self.eq(node.get('._woot'), 5)
self.nn(node.getTagProp('test', 'score'), 6)

with self.raises(s_exc.BadTag):
await core.nodes('[ it:dev:str=foo +#test.newp ]')

stream.seek(0)
data = stream.getvalue()
newprole = s_common.guid('newprole')
newpuser = s_common.guid('newpuser')

self.isin(f'Unknown user {newpuser} on gate', data)
self.isin(f'Unknown role {newprole} on gate', data)
self.isin(f'Unknown role {newprole} on user', data)

async def test_cell_check_sysctl(self):
sysctls = s_linux.getSysctls()
Expand Down

0 comments on commit d186422

Please sign in to comment.