From 4617a12cea1cbdf6e66093da547c3d4967a99976 Mon Sep 17 00:00:00 2001 From: Cisphyx Date: Fri, 10 Jan 2025 10:55:57 -0500 Subject: [PATCH 1/2] Add conditional edit operator syntax (SYN-7486) (#4046) --- changes/e1b7c7693e7454c32ca657a2bed734d5.yaml | 5 + .../userguides/storm_ref_data_mod.rstorm | 30 ++++ synapse/datamodel.py | 9 +- synapse/lib/ast.py | 156 +++++++++++++----- synapse/lib/parser.py | 4 + synapse/lib/storm.lark | 17 +- synapse/lib/storm_format.py | 1 + synapse/lib/stormtypes.py | 2 +- synapse/tests/test_lib_ast.py | 89 +++++++++- synapse/tests/test_lib_grammar.py | 14 ++ synapse/tests/test_lib_stormtypes.py | 6 +- 11 files changed, 281 insertions(+), 52 deletions(-) create mode 100644 changes/e1b7c7693e7454c32ca657a2bed734d5.yaml diff --git a/changes/e1b7c7693e7454c32ca657a2bed734d5.yaml b/changes/e1b7c7693e7454c32ca657a2bed734d5.yaml new file mode 100644 index 00000000000..9cb004fb88b --- /dev/null +++ b/changes/e1b7c7693e7454c32ca657a2bed734d5.yaml @@ -0,0 +1,5 @@ +--- +desc: Added syntax for conditional node property edit operators in Storm. +prs: [] +type: feat +... diff --git a/docs/synapse/userguides/storm_ref_data_mod.rstorm b/docs/synapse/userguides/storm_ref_data_mod.rstorm index aa02c8c49ea..c0e01978eeb 100644 --- a/docs/synapse/userguides/storm_ref_data_mod.rstorm +++ b/docs/synapse/userguides/storm_ref_data_mod.rstorm @@ -60,6 +60,7 @@ what changes should be made and to what data: - `Edit Brackets`_ - `Edit Parentheses`_ - `"Try" Operator`_ +- `Conditional Edit Operators`_ - `Autoadds and Depadds`_ .. _edit-brackets: @@ -180,6 +181,31 @@ where the variable may contain unexpected values. For example: See the :ref:`type-syn-tag` section of the :ref:`storm-ref-type-specific` for additional detail on tags / ``syn:tag`` forms. +Conditional Edit Operators +++++++++++++++++++++++++++ + +The conditional edit operators ( ``*unset=`` and ``*$=`` ) can be used to only set properties when certain +conditions are met. + +The ``*unset=`` operator will only set a property when it does not already have a value to prevent overwriting +existing data. For example: + +``inet:ipv4 = 1.2.3.4 [ :asn *unset= 12345 ]`` + +will only set the ``:asn`` property on the ``inet:ipv4`` node if it is not already set. The conditional edit operators +can also be combined with the "try" operator ( ``*unset?=`` ) to prevent failures due to bad data: + +``inet:ipv4 = 1.2.3.4 [ :asn *unset?= invalid ]`` + +Variable values may also be used to control the conditional edit behavior, and allow two more values in addition to +``unset``; ``always`` and ``never``. For example: + +``$asn = 'always' $loc = 'never' inet:ipv4 = 1.2.4.5 [ :loc *$loc= us :asn *$asn?= 12345 ]`` + +will never set the ``:loc`` property and will always attempt to set the ``:asn`` property. This behavior is useful +when creating Storm ingest functions where fine tuned control over specific property edit behavior is needed. Rather +than creating variations of the same ingest function with different combinations of property set behavior, one function +can use a dictionary of configuration options to control the edit behavior used during each execution. .. _autoadds-depadds: @@ -312,6 +338,10 @@ The same syntax is used to apply a new property or modify an existing property. ** **[ :** ** **=** | **?=** ** ... **]** +** **[ :** ** ***unset=** | ***unset?** ** ... **]** + +** **[ :** ** ***$=** | ***$?=** ** ... **]** + .. TIP:: You can optionally use the :ref:`edit-try` ( ``?=`` ) when setting or modifying properties. diff --git a/synapse/datamodel.py b/synapse/datamodel.py index 0c68aa36fec..72b6db6e480 100644 --- a/synapse/datamodel.py +++ b/synapse/datamodel.py @@ -445,14 +445,17 @@ def prop(self, name: str): ''' return self.props.get(name) - def reqProp(self, name): + def reqProp(self, name, extra=None): prop = self.props.get(name) if prop is not None: return prop full = f'{self.name}:{name}' - mesg = f'No property named {full}.' - raise s_exc.NoSuchProp(mesg=mesg, name=full) + exc = s_exc.NoSuchProp.init(full) + if extra is not None: + exc = extra(exc) + + raise exc def pack(self): props = {p.name: p.pack() for p in self.props.values()} diff --git a/synapse/lib/ast.py b/synapse/lib/ast.py index dce4d2eed94..816797dface 100644 --- a/synapse/lib/ast.py +++ b/synapse/lib/ast.py @@ -28,6 +28,16 @@ from synapse.lib.stormtypes import tobool, toint, toprim, tostr, tonumber, tocmprvalu, undef +SET_ALWAYS = 0 +SET_UNSET = 1 +SET_NEVER = 2 + +COND_EDIT_SET = { + 'always': SET_ALWAYS, + 'unset': SET_UNSET, + 'never': SET_NEVER, +} + logger = logging.getLogger(__name__) def parseNumber(x): @@ -136,7 +146,7 @@ def hasAstClass(self, clss): retn = True break - if isinstance(kid, (EditPropSet, Function, CmdOper)): + if isinstance(kid, (EditPropSet, EditCondPropSet, Function, CmdOper)): continue if kid.hasAstClass(clss): @@ -179,6 +189,12 @@ def reqRuntSafe(self, runt, mesg): todo.extend(nkid.kids) + def reqNotReadOnly(self, runt, mesg=None): + if runt.readonly: + if mesg is None: + mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' + raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + def hasVarName(self, name): return any(k.hasVarName(name) for k in self.kids) @@ -238,9 +254,8 @@ def __init__(self, astinfo, kids, autoadd=False): async def run(self, runt, genr): - if runt.readonly and self.autoadd: - mesg = 'Autoadd may not be executed in readonly Storm runtime.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + if self.autoadd: + self.reqNotReadOnly(runt) async def getnode(form, valu): try: @@ -1269,15 +1284,17 @@ async def run(self, runt, genr): item = s_stormtypes.fromprim(await self.kids[0].compute(runt, path), basetypes=False) if runt.readonly and not getattr(item.setitem, '_storm_readonly', False): - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.kids[0].reqNotReadOnly(runt) name = await self.kids[1].compute(runt, path) valu = await self.kids[2].compute(runt, path) # TODO: ditch this when storm goes full heavy object with s_scope.enter({'runt': runt}): - await item.setitem(name, valu) + try: + await item.setitem(name, valu) + except s_exc.SynErr as e: + raise self.kids[0].addExcInfo(e) yield node, path @@ -1289,12 +1306,14 @@ async def run(self, runt, genr): valu = await self.kids[2].compute(runt, None) if runt.readonly and not getattr(item.setitem, '_storm_readonly', False): - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.kids[0].reqNotReadOnly(runt) # TODO: ditch this when storm goes full heavy object with s_scope.enter({'runt': runt}): - await item.setitem(name, valu) + try: + await item.setitem(name, valu) + except s_exc.SynErr as e: + raise self.kids[0].addExcInfo(e) class VarListSetOper(Oper): @@ -3572,7 +3591,8 @@ async def compute(self, runt, path): raise self.addExcInfo(s_exc.StormRuntimeError(mesg=mesg)) if runt.readonly and not getattr(func, '_storm_readonly', False): - mesg = f'Function ({func.__name__}) is not marked readonly safe.' + funcname = getattr(func, '_storm_funcpath', func.__name__) + mesg = f'{funcname}() is not marked readonly safe.' raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg)) argv = await self.kids[1].compute(runt, path) @@ -3998,9 +4018,7 @@ class EditParens(Edit): async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) nodeadd = self.kids[0] assert isinstance(nodeadd, EditNodeAdd) @@ -4093,9 +4111,7 @@ async def run(self, runt, genr): # case 2: [ foo:bar=($node, 20) ] # case 2: $blah=:baz [ foo:bar=($blah, 20) ] - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) runtsafe = self.isRuntSafe(runt) @@ -4161,13 +4177,79 @@ async def feedfunc(): async for item in agen: yield item +class CondSetOper(Oper): + def __init__(self, astinfo, kids, errok=False): + Value.__init__(self, astinfo, kids=kids) + self.errok = errok + + def prepare(self): + self.isconst = False + if isinstance(self.kids[0], Const): + self.isconst = True + self.valu = COND_EDIT_SET.get(self.kids[0].value()) + + async def compute(self, runt, path): + if self.isconst: + return self.valu + + valu = await self.kids[0].compute(runt, path) + if (retn := COND_EDIT_SET.get(valu)) is not None: + return retn + + mesg = f'Invalid conditional set operator ({valu}).' + exc = s_exc.StormRuntimeError(mesg=mesg) + raise self.addExcInfo(exc) + +class EditCondPropSet(Edit): + + async def run(self, runt, genr): + + self.reqNotReadOnly(runt) + + excignore = (s_exc.BadTypeValu,) if self.kids[1].errok else () + rval = self.kids[2] + + async for node, path in genr: + + propname = await self.kids[0].compute(runt, path) + name = await tostr(propname) + + prop = node.form.reqProp(name, extra=self.kids[0].addExcInfo) + + oper = await self.kids[1].compute(runt, path) + if oper == SET_NEVER or (oper == SET_UNSET and (oldv := node.get(name)) is not None): + yield node, path + await asyncio.sleep(0) + continue + + if not node.form.isrunt: + # runt node property permissions are enforced by the callback + runt.confirmPropSet(prop) + + isndef = isinstance(prop.type, s_types.Ndef) + + try: + valu = await rval.compute(runt, path) + valu = await s_stormtypes.tostor(valu, isndef=isndef) + + if isinstance(prop.type, s_types.Ival) and oldv is not None: + valu, _ = prop.type.norm(valu) + valu = prop.type.merge(oldv, valu) + + await node.set(name, valu) + + except excignore: + pass + + yield node, path + + await asyncio.sleep(0) + class EditPropSet(Edit): async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) oper = await self.kids[1].compute(runt, None) excignore = (s_exc.BadTypeValu,) if oper in ('?=', '?+=', '?-=') else () @@ -4212,7 +4294,7 @@ async def run(self, runt, genr): if not isarray: mesg = f'Property set using ({oper}) is only valid on arrays.' - exc = s_exc.StormRuntimeError(mesg) + exc = s_exc.StormRuntimeError(mesg=mesg) raise self.kids[0].addExcInfo(exc) arry = node.get(name) @@ -4260,9 +4342,7 @@ class EditPropDel(Edit): async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) async for node, path in genr: propname = await self.kids[0].compute(runt, path) @@ -4288,9 +4368,7 @@ class EditUnivDel(Edit): async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) univprop = self.kids[0] assert isinstance(univprop, UnivProp) @@ -4466,9 +4544,7 @@ def __init__(self, astinfo, kids=(), n2=False): async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) # SubQuery -> Query query = self.kids[1].kids[0] @@ -4531,9 +4607,7 @@ def __init__(self, astinfo, kids=(), n2=False): async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) query = self.kids[1].kids[0] @@ -4589,9 +4663,7 @@ class EditTagAdd(Edit): async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) if len(self.kids) > 1 and isinstance(self.kids[0], Const) and (await self.kids[0].compute(runt, None)) == '?': oper_offset = 1 @@ -4635,9 +4707,7 @@ class EditTagDel(Edit): async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) async for node, path in genr: @@ -4661,9 +4731,7 @@ class EditTagPropSet(Edit): ''' async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) oper = await self.kids[1].compute(runt, None) excignore = s_exc.BadTypeValu if oper == '?=' else () @@ -4697,9 +4765,7 @@ class EditTagPropDel(Edit): ''' async def run(self, runt, genr): - if runt.readonly: - mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' - raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg)) + self.reqNotReadOnly(runt) async for node, path in genr: diff --git a/synapse/lib/parser.py b/synapse/lib/parser.py index 614f0ab7c3c..56343d289f3 100644 --- a/synapse/lib/parser.py +++ b/synapse/lib/parser.py @@ -95,6 +95,7 @@ 'TRYSETPLUS': '?+=', 'TRYSETMINUS': '?-=', 'UNIVNAME': 'universal property', + 'UNSET': 'unset', 'EXPRUNIVNAME': 'universal property', 'VARTOKN': 'variable', 'EXPRVARTOKN': 'variable', @@ -642,6 +643,8 @@ def massage_vartokn(astinfo, x): 'andexpr': s_ast.AndCond, 'baresubquery': s_ast.SubQuery, 'catchblock': s_ast.CatchBlock, + 'condsetoper': s_ast.CondSetOper, + 'condtrysetoper': lambda astinfo, kids: s_ast.CondSetOper(astinfo, kids, errok=True), 'condsubq': s_ast.SubqCond, 'dollarexpr': s_ast.DollarExpr, 'edgeaddn1': s_ast.EditEdgeAdd, @@ -657,6 +660,7 @@ def massage_vartokn(astinfo, x): 'formname': s_ast.FormName, 'editpropdel': lambda astinfo, kids: s_ast.EditPropDel(astinfo, kids[1:]), 'editpropset': s_ast.EditPropSet, + 'editcondpropset': s_ast.EditCondPropSet, 'edittagadd': s_ast.EditTagAdd, 'edittagdel': lambda astinfo, kids: s_ast.EditTagDel(astinfo, kids[1:]), 'edittagpropset': s_ast.EditTagPropSet, diff --git a/synapse/lib/storm.lark b/synapse/lib/storm.lark index c7434aea4ef..1ca2375afa1 100644 --- a/synapse/lib/storm.lark +++ b/synapse/lib/storm.lark @@ -39,7 +39,7 @@ _editblock: "[" _editoper* "]" // A single edit operation _editoper: editnodeadd - | editpropset | editunivset | edittagpropset | edittagadd + | editpropset | editunivset | edittagpropset | edittagadd | editcondpropset | editpropdel | editunivdel | edittagpropdel | edittagdel | editparens | edgeaddn1 | edgedeln1 | edgeaddn2 | edgedeln2 @@ -49,11 +49,13 @@ edittagadd: "+" [SETTAGOPER] tagname [(EQSPACE | EQNOSPACE) _valu] editunivdel: EXPRMINUS univprop edittagdel: EXPRMINUS tagname editpropset: relprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu +editcondpropset: relprop condsetoper _valu editpropdel: EXPRMINUS relprop editunivset: univprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu editnodeadd: formname (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu edittagpropset: "+" tagprop (EQSPACE | EQNOSPACE | MODSET | TRYSET | TRYSETPLUS | TRYSETMINUS) _valu edittagpropdel: EXPRMINUS tagprop + EQSPACE: /((?<=\s)=|=(?=\s))/ MODSET.4: "+=" | "-=" TRYSETPLUS.1: "?+=" @@ -61,6 +63,19 @@ TRYSETMINUS.1: "?-=" TRYSET.1: "?=" SETTAGOPER: "?" +condsetoper: ("*" UNSET | _DEREF "$" _condvarvaluatom) "=" + | ("*" UNSET | _DEREF "$" _condvarvaluatom) "?=" -> condtrysetoper +UNSET: "unset" +_condvarvaluatom: condvarvalue | condvarderef | condfunccall +condvarvalue: VARTOKN -> varvalue + +!condvarderef: _condvarvaluatom "." (VARTOKN | "$" VARTOKN | _condderefexpr) -> varderef +_condderefexpr: "$"? conddollarexpr +conddollarexpr: "(" expror ")" -> dollarexpr + +condfunccall: _condvarvaluatom _condcallargs -> funccall +_condcallargs: _LPARNOSPACE [(_valu | VARTOKN | (VARTOKN | NONQUOTEWORD) (EQSPACE | EQNOSPACE) _valu) ("," (_valu | VARTOKN | (VARTOKN | NONQUOTEWORD) (EQSPACE | EQNOSPACE) _valu))*] ","? ")" + // The set of non-edit non-commands in storm _oper: stormfunc | initblock | emptyblock | finiblock | trycatch | subquery | _formpivot | formjoin diff --git a/synapse/lib/storm_format.py b/synapse/lib/storm_format.py index e2ec4a6865c..1a7c7ce3ac7 100644 --- a/synapse/lib/storm_format.py +++ b/synapse/lib/storm_format.py @@ -77,6 +77,7 @@ 'TRYSETMINUS': p_t.Operator, 'TRYSETPLUS': p_t.Operator, 'UNIVNAME': p_t.Name, + 'UNSET': p_t.Operator, 'EXPRUNIVNAME': p_t.Name, 'VARTOKN': p_t.Name.Variable, 'EXPRVARTOKN': p_t.Name.Variable, diff --git a/synapse/lib/stormtypes.py b/synapse/lib/stormtypes.py index d8987e05650..068104029db 100644 --- a/synapse/lib/stormtypes.py +++ b/synapse/lib/stormtypes.py @@ -575,7 +575,7 @@ async def setitem(self, name, valu): raise s_exc.NoSuchName(name=name, mesg=mesg) if s_scope.get('runt').readonly and not getattr(stor, '_storm_readonly', False): - mesg = f'Function ({stor.__name__}) is not marked readonly safe.' + mesg = f'Setting {name} on {self._storm_typename} is not marked readonly safe.' raise s_exc.IsReadOnly(mesg=mesg, name=name, valu=valu) await s_coro.ornot(stor, valu) diff --git a/synapse/tests/test_lib_ast.py b/synapse/tests/test_lib_ast.py index 50673db478a..d8092afdbf4 100644 --- a/synapse/tests/test_lib_ast.py +++ b/synapse/tests/test_lib_ast.py @@ -405,6 +405,93 @@ async def test_ast_variable_props(self): q = 'test:str=foo $newp=($node.repr(), bar) [*$newp=foo]' await self.asyncraises(s_exc.StormRuntimeError, core.nodes(q)) + async def test_ast_condsetoper(self): + async with self.getTestCore() as core: + + q = '$var=hehe $foo=unset [test:str=foo :$var*unset=heval]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.eq('heval', nodes[0].get('hehe')) + + q = '$var=hehe $foo=unset [test:str=foo :$var*unset=newp]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.eq('heval', nodes[0].get('hehe')) + + q = '$var=hehe $foo=unset [test:str=foo :$var*$foo=newp]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.eq('heval', nodes[0].get('hehe')) + + q = '$var=hehe $foo=always [test:str=foo :$var*$foo=yep]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.eq('yep', nodes[0].get('hehe')) + + q = '[test:str=foo -:hehe]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.none(nodes[0].get('hehe')) + + q = '$var=hehe $foo=never [test:str=foo :$var*$foo=yep]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.none(nodes[0].get('hehe')) + + q = '$var=hehe $foo=unset [test:str=foo :$var*$foo=heval]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.eq('heval', nodes[0].get('hehe')) + + with self.raises(s_exc.BadTypeValu): + q = '$var=tick $foo=always [test:str=foo :$var*$foo=heval]' + nodes = await core.nodes(q) + + q = '$var=tick $foo=always [test:str=foo :$var*$foo?=heval]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.none(nodes[0].get('tick')) + + q = ''' + $opts=({"tick": "unset", "hehe": "always"}) + [ test:str=foo + :hehe*$opts.hehe=newv + :tick*$opts.tick?=2020] + ''' + nodes = await core.nodes(q) + self.len(1, nodes) + self.eq('newv', nodes[0].get('hehe')) + tick = nodes[0].get('tick') + self.nn(tick) + + q = ''' + $opts=({"tick": "never", "hehe": "unset"}) + [ test:str=foo + :hehe*$opts.hehe=newp + :tick*$opts.tick?=2020] + ''' + nodes = await core.nodes(q) + self.len(1, nodes) + self.eq('newv', nodes[0].get('hehe')) + self.eq(tick, nodes[0].get('tick')) + + q = '$foo=always [test:str=foo :tick*$foo?=2021]' + nodes = await core.nodes(q) + self.len(1, nodes) + self.ne(tick, nodes[0].get('tick')) + + with self.raises(s_exc.IsReadOnly): + q = '[test:str=foo :hehe*unset=heval]' + nodes = await core.nodes(q, opts={'readonly': True}) + + with self.raises(s_exc.NoSuchProp): + q = '[test:str=foo :newp*unset=heval]' + nodes = await core.nodes(q) + + with self.raises(s_exc.StormRuntimeError): + q = '$foo=newp [test:str=foo :hehe*$foo=heval]' + nodes = await core.nodes(q) + async def test_ast_editparens(self): async with self.getTestCore() as core: @@ -2648,7 +2735,7 @@ async def test_ast_storm_readonly(self): q = 'function func(arg) { auth.user.addrule root $arg | return () } $func(hehe.haha)' msgs = await core.stormlist(q, opts={'readonly': True}) - self.stormIsInErr('Function (_methUserAddRule) is not marked readonly safe.', msgs) + self.stormIsInErr('auth:user.addRule() is not marked readonly safe.', msgs) async def test_ast_yield(self): diff --git a/synapse/tests/test_lib_grammar.py b/synapse/tests/test_lib_grammar.py index 2b6fd8e3f92..81812eca707 100644 --- a/synapse/tests/test_lib_grammar.py +++ b/synapse/tests/test_lib_grammar.py @@ -728,6 +728,13 @@ '$pvar=stuff test:arrayprop +:$pvar*[=neato]', '$pvar=ints test:arrayprop +:$pvar*[=$othervar]', '$foo = ({"foo": ${ inet:fqdn }})', + '[test:str=foo :hehe*unset=heval]', + '[test:str=foo :hehe*$foo=heval]', + '[test:str=foo :$foo*unset=heval]', + '[test:str=foo :$foo*$bar=heval]', + '[test:str=foo :$foo*$bar.baz=heval]', + '[test:str=foo :$foo*$bar.("baz")=heval]', + '[test:str=foo :$foo*$bar.baz()=heval]', ] # Generated with print_parse_list below @@ -1358,6 +1365,13 @@ 'Query: [SetVarOper: [Const: pvar, Const: stuff], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, Const: neato]]]', 'Query: [SetVarOper: [Const: pvar, Const: ints], LiftProp: [Const: test:arrayprop], FiltOper: [Const: +, ArrayCond: [RelProp: [VarValue: [Const: pvar]], Const: =, VarValue: [Const: othervar]]]]', 'Query: [SetVarOper: [Const: foo, DollarExpr: [ExprDict: [Const: foo, EmbedQuery: inet:fqdn]]]]', + 'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [Const: hehe], CondSetOper: [Const: unset], Const: heval]]', + 'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [Const: hehe], CondSetOper: [VarValue: [Const: foo]], Const: heval]]', + 'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [Const: unset], Const: heval]]', + 'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarValue: [Const: bar]], Const: heval]]', + 'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarDeref: [VarValue: [Const: bar], Const: baz]], Const: heval]]', + 'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [VarDeref: [VarValue: [Const: bar], DollarExpr: [Const: baz]]], Const: heval]]', + 'Query: [EditNodeAdd: [FormName: [Const: test:str], Const: =, Const: foo], EditCondPropSet: [RelProp: [VarValue: [Const: foo]], CondSetOper: [FuncCall: [VarDeref: [VarValue: [Const: bar], Const: baz], CallArgs: [], CallKwargs: []]], Const: heval]]', ] class GrammarTest(s_t_utils.SynTest): diff --git a/synapse/tests/test_lib_stormtypes.py b/synapse/tests/test_lib_stormtypes.py index f74642ddac9..e0eb964b36f 100644 --- a/synapse/tests/test_lib_stormtypes.py +++ b/synapse/tests/test_lib_stormtypes.py @@ -6639,7 +6639,7 @@ async def test_storm_stor_readonly(self): 'newname': 'oops' }}) - self.stormIsInErr('Function (_storUserName) is not marked readonly safe.', msgs) + self.stormIsInErr('Setting name on auth:user is not marked readonly safe.', msgs) mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.' @@ -6670,6 +6670,10 @@ async def test_storm_stor_readonly(self): msgs = await core.stormlist(q, opts={'readonly': True, 'vars': {'iden': user}}) self.stormIsInErr(mesg, msgs) + q = '$lib.pkg.add(({}))' + msgs = await core.stormlist(q, opts={'readonly': True, 'vars': {'iden': user}}) + self.stormIsInErr('$lib.pkg.add() is not marked readonly safe.', msgs) + async def test_storm_view_counts(self): async with self.getTestCore() as core: From 41ed219d04a1208c2be4d525134bb840070994ed Mon Sep 17 00:00:00 2001 From: blackout Date: Fri, 10 Jan 2025 11:59:44 -0500 Subject: [PATCH 2/2] SYN-8272: Cell password complexity allows None input in some cases and not others (#4059) --- changes/fd2d79b0daf0705278a48e86b15524c7.yaml | 6 ++++++ synapse/lib/auth.py | 1 + synapse/tests/test_lib_auth.py | 13 ++++++------- 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 changes/fd2d79b0daf0705278a48e86b15524c7.yaml diff --git a/changes/fd2d79b0daf0705278a48e86b15524c7.yaml b/changes/fd2d79b0daf0705278a48e86b15524c7.yaml new file mode 100644 index 00000000000..2d967615017 --- /dev/null +++ b/changes/fd2d79b0daf0705278a48e86b15524c7.yaml @@ -0,0 +1,6 @@ +--- +desc: Fixed bug in password complexity rules where setting a password to (null) or + None would fail. +prs: [] +type: bug +... diff --git a/synapse/lib/auth.py b/synapse/lib/auth.py index 60f149ab2e9..aab6e010e81 100644 --- a/synapse/lib/auth.py +++ b/synapse/lib/auth.py @@ -1545,6 +1545,7 @@ async def setPasswd(self, passwd, nexs=True, enforce_policy=True): # Prevent empty string or non-string values if passwd is None: shadow = None + enforce_policy = False elif passwd and isinstance(passwd, str): shadow = await s_passwd.getShadowV2(passwd=passwd) else: diff --git a/synapse/tests/test_lib_auth.py b/synapse/tests/test_lib_auth.py index 0d8f5380934..58940ccf766 100644 --- a/synapse/tests/test_lib_auth.py +++ b/synapse/tests/test_lib_auth.py @@ -495,7 +495,7 @@ async def test_auth_password_policy(self): ]) # Check sequences - seqmsg = f'Password must not contain forward/reverse sequences longer than 3 characters.' + seqmsg = 'Password must not contain forward/reverse sequences longer than 3 characters.' passwords = [ # letters 'abcA', 'dcbA', 'Abcd', 'Acba', @@ -531,12 +531,8 @@ async def test_auth_password_policy(self): 'Password must contain at least 2 digit characters, 0 found.' ]) - with self.raises(s_exc.BadArg) as exc: - await core.setUserPasswd(user.iden, None) - self.isin( - 'Password must be at least 12 characters.', - exc.exception.get('failures') - ) + # Setting password to None should work also + await core.setUserPasswd(user.iden, None) # Attempting to add a user with a bad passwd will add the user and fail to set the password with self.raises(s_exc.BadArg): @@ -582,6 +578,9 @@ async def test_auth_password_policy(self): await core.setUserPasswd(user.iden, pass2) await core.setUserPasswd(user.iden, pass3) + # Setting password to None should work also + await core.setUserPasswd(user.iden, None) + with self.raises(s_exc.BadArg) as exc: await core.setUserPasswd(user.iden, pass1) self.eq(exc.exception.get('failures'), [