Merge branch 'master' into cron-creator-to-user

changes/e1b7c7693e7454c32ca657a2bed734d5.yaml

@@ -0,0 +1,5 @@
+---
+desc: Added syntax for conditional node property edit operators in Storm.
+prs: []
+type: feat
+...

changes/fd2d79b0daf0705278a48e86b15524c7.yaml

@@ -0,0 +1,6 @@
+---
+desc: Fixed bug in password complexity rules where setting a password to (null) or
+  None would fail.
+prs: []
+type: bug
+...

docs/synapse/userguides/storm_ref_data_mod.rstorm

@@ -60,6 +60,7 @@ what changes should be made and to what data:
 - `Edit Brackets`_
 - `Edit Parentheses`_
 - `"Try" Operator`_
+- `Conditional Edit Operators`_
 - `Autoadds and Depadds`_
 
 .. _edit-brackets:
@@ -180,6 +181,31 @@ where the variable may contain unexpected values. For example:
 
 See the :ref:`type-syn-tag` section of the :ref:`storm-ref-type-specific` for additional detail on tags / ``syn:tag`` forms.
 
+Conditional Edit Operators
+++++++++++++++++++++++++++
+
+The conditional edit operators ( ``*unset=`` and ``*$<varname>=`` ) can be used to only set properties when certain
+conditions are met.
+
+The ``*unset=`` operator will only set a property when it does not already have a value to prevent overwriting
+existing data. For example:
+
+``inet:ipv4 = 1.2.3.4 [ :asn *unset= 12345 ]``
+
+will only set the ``:asn`` property on the ``inet:ipv4`` node if it is not already set. The conditional edit operators
+can also be combined with the "try" operator ( ``*unset?=`` ) to prevent failures due to bad data:
+
+``inet:ipv4 = 1.2.3.4 [ :asn *unset?= invalid ]``
+
+Variable values may also be used to control the conditional edit behavior, and allow two more values in addition to
+``unset``; ``always`` and ``never``. For example:
+
+``$asn = 'always' $loc = 'never' inet:ipv4 = 1.2.4.5 [ :loc *$loc= us :asn *$asn?= 12345 ]``
+
+will never set the ``:loc`` property and will always attempt to set the ``:asn`` property. This behavior is useful
+when creating Storm ingest functions where fine tuned control over specific property edit behavior is needed. Rather
+than creating variations of the same ingest function with different combinations of property set behavior, one function
+can use a dictionary of configuration options to control the edit behavior used during each execution.
 
 .. _autoadds-depadds:
 
@@ -312,6 +338,10 @@ The same syntax is used to apply a new property or modify an existing property.
 
 *<query>* **[ :** *<prop>* **=** | **?=** *<pval>* ... **]**
 
+*<query>* **[ :** *<prop>* ***unset=** | ***unset?** *<pval>* ... **]**
+
+*<query>* **[ :** *<prop>* ***$<varname>=** | ***$<varname>?=** *<pval>* ... **]**
+
 .. TIP::
 
   You can optionally use the :ref:`edit-try` ( ``?=`` ) when setting or modifying properties.

synapse/datamodel.py

@@ -445,14 +445,17 @@ def prop(self, name: str):
         '''
         return self.props.get(name)
 
-    def reqProp(self, name):
+    def reqProp(self, name, extra=None):
         prop = self.props.get(name)
         if prop is not None:
             return prop
 
         full = f'{self.name}:{name}'
-        mesg = f'No property named {full}.'
-        raise s_exc.NoSuchProp(mesg=mesg, name=full)
+        exc = s_exc.NoSuchProp.init(full)
+        if extra is not None:
+            exc = extra(exc)
+
+        raise exc
 
     def pack(self):
         props = {p.name: p.pack() for p in self.props.values()}

synapse/lib/ast.py

@@ -28,6 +28,16 @@
 
 from synapse.lib.stormtypes import tobool, toint, toprim, tostr, tonumber, tocmprvalu, undef
 
+SET_ALWAYS = 0
+SET_UNSET = 1
+SET_NEVER = 2
+
+COND_EDIT_SET = {
+    'always': SET_ALWAYS,
+    'unset': SET_UNSET,
+    'never': SET_NEVER,
+}
+
 logger = logging.getLogger(__name__)
 
 def parseNumber(x):
@@ -136,7 +146,7 @@ def hasAstClass(self, clss):
                 retn = True
                 break
 
-            if isinstance(kid, (EditPropSet, Function, CmdOper)):
+            if isinstance(kid, (EditPropSet, EditCondPropSet, Function, CmdOper)):
                 continue
 
             if kid.hasAstClass(clss):
@@ -179,6 +189,12 @@ def reqRuntSafe(self, runt, mesg):
 
             todo.extend(nkid.kids)
 
+    def reqNotReadOnly(self, runt, mesg=None):
+        if runt.readonly:
+            if mesg is None:
+                mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
+            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+
     def hasVarName(self, name):
         return any(k.hasVarName(name) for k in self.kids)
 
@@ -238,9 +254,8 @@ def __init__(self, astinfo, kids, autoadd=False):
 
     async def run(self, runt, genr):
 
-        if runt.readonly and self.autoadd:
-            mesg = 'Autoadd may not be executed in readonly Storm runtime.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        if self.autoadd:
+            self.reqNotReadOnly(runt)
 
         async def getnode(form, valu):
             try:
@@ -1269,15 +1284,17 @@ async def run(self, runt, genr):
             item = s_stormtypes.fromprim(await self.kids[0].compute(runt, path), basetypes=False)
 
             if runt.readonly and not getattr(item.setitem, '_storm_readonly', False):
-                mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-                raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+                self.kids[0].reqNotReadOnly(runt)
 
             name = await self.kids[1].compute(runt, path)
             valu = await self.kids[2].compute(runt, path)
 
             # TODO: ditch this when storm goes full heavy object
             with s_scope.enter({'runt': runt}):
-                await item.setitem(name, valu)
+                try:
+                    await item.setitem(name, valu)
+                except s_exc.SynErr as e:
+                    raise self.kids[0].addExcInfo(e)
 
             yield node, path
 
@@ -1289,12 +1306,14 @@ async def run(self, runt, genr):
             valu = await self.kids[2].compute(runt, None)
 
             if runt.readonly and not getattr(item.setitem, '_storm_readonly', False):
-                mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-                raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+                self.kids[0].reqNotReadOnly(runt)
 
             # TODO: ditch this when storm goes full heavy object
             with s_scope.enter({'runt': runt}):
-                await item.setitem(name, valu)
+                try:
+                    await item.setitem(name, valu)
+                except s_exc.SynErr as e:
+                    raise self.kids[0].addExcInfo(e)
 
 class VarListSetOper(Oper):
 
@@ -3572,7 +3591,8 @@ async def compute(self, runt, path):
             raise self.addExcInfo(s_exc.StormRuntimeError(mesg=mesg))
 
         if runt.readonly and not getattr(func, '_storm_readonly', False):
-            mesg = f'Function ({func.__name__}) is not marked readonly safe.'
+            funcname = getattr(func, '_storm_funcpath', func.__name__)
+            mesg = f'{funcname}() is not marked readonly safe.'
             raise self.kids[0].addExcInfo(s_exc.IsReadOnly(mesg=mesg))
 
         argv = await self.kids[1].compute(runt, path)
@@ -3998,9 +4018,7 @@ class EditParens(Edit):
 
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         nodeadd = self.kids[0]
         assert isinstance(nodeadd, EditNodeAdd)
@@ -4093,9 +4111,7 @@ async def run(self, runt, genr):
         # case 2: <query> [ foo:bar=($node, 20) ]
         # case 2: <query> $blah=:baz [ foo:bar=($blah, 20) ]
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         runtsafe = self.isRuntSafe(runt)
 
@@ -4161,13 +4177,79 @@ async def feedfunc():
             async for item in agen:
                 yield item
 
+class CondSetOper(Oper):
+    def __init__(self, astinfo, kids, errok=False):
+        Value.__init__(self, astinfo, kids=kids)
+        self.errok = errok
+
+    def prepare(self):
+        self.isconst = False
+        if isinstance(self.kids[0], Const):
+            self.isconst = True
+            self.valu = COND_EDIT_SET.get(self.kids[0].value())
+
+    async def compute(self, runt, path):
+        if self.isconst:
+            return self.valu
+
+        valu = await self.kids[0].compute(runt, path)
+        if (retn := COND_EDIT_SET.get(valu)) is not None:
+            return retn
+
+        mesg = f'Invalid conditional set operator ({valu}).'
+        exc = s_exc.StormRuntimeError(mesg=mesg)
+        raise self.addExcInfo(exc)
+
+class EditCondPropSet(Edit):
+
+    async def run(self, runt, genr):
+
+        self.reqNotReadOnly(runt)
+
+        excignore = (s_exc.BadTypeValu,) if self.kids[1].errok else ()
+        rval = self.kids[2]
+
+        async for node, path in genr:
+
+            propname = await self.kids[0].compute(runt, path)
+            name = await tostr(propname)
+
+            prop = node.form.reqProp(name, extra=self.kids[0].addExcInfo)
+
+            oper = await self.kids[1].compute(runt, path)
+            if oper == SET_NEVER or (oper == SET_UNSET and (oldv := node.get(name)) is not None):
+                yield node, path
+                await asyncio.sleep(0)
+                continue
+
+            if not node.form.isrunt:
+                # runt node property permissions are enforced by the callback
+                runt.confirmPropSet(prop)
+
+            isndef = isinstance(prop.type, s_types.Ndef)
+
+            try:
+                valu = await rval.compute(runt, path)
+                valu = await s_stormtypes.tostor(valu, isndef=isndef)
+
+                if isinstance(prop.type, s_types.Ival) and oldv is not None:
+                    valu, _ = prop.type.norm(valu)
+                    valu = prop.type.merge(oldv, valu)
+
+                await node.set(name, valu)
+
+            except excignore:
+                pass
+
+            yield node, path
+
+            await asyncio.sleep(0)
+
 class EditPropSet(Edit):
 
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         oper = await self.kids[1].compute(runt, None)
         excignore = (s_exc.BadTypeValu,) if oper in ('?=', '?+=', '?-=') else ()
@@ -4212,7 +4294,7 @@ async def run(self, runt, genr):
 
                     if not isarray:
                         mesg = f'Property set using ({oper}) is only valid on arrays.'
-                        exc = s_exc.StormRuntimeError(mesg)
+                        exc = s_exc.StormRuntimeError(mesg=mesg)
                         raise self.kids[0].addExcInfo(exc)
 
                     arry = node.get(name)
@@ -4260,9 +4342,7 @@ class EditPropDel(Edit):
 
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         async for node, path in genr:
             propname = await self.kids[0].compute(runt, path)
@@ -4288,9 +4368,7 @@ class EditUnivDel(Edit):
 
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         univprop = self.kids[0]
         assert isinstance(univprop, UnivProp)
@@ -4466,9 +4544,7 @@ def __init__(self, astinfo, kids=(), n2=False):
 
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         # SubQuery -> Query
         query = self.kids[1].kids[0]
@@ -4531,9 +4607,7 @@ def __init__(self, astinfo, kids=(), n2=False):
 
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         query = self.kids[1].kids[0]
 
@@ -4589,9 +4663,7 @@ class EditTagAdd(Edit):
 
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         if len(self.kids) > 1 and isinstance(self.kids[0], Const) and (await self.kids[0].compute(runt, None)) == '?':
             oper_offset = 1
@@ -4635,9 +4707,7 @@ class EditTagDel(Edit):
 
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         async for node, path in genr:
 
@@ -4661,9 +4731,7 @@ class EditTagPropSet(Edit):
     '''
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         oper = await self.kids[1].compute(runt, None)
         excignore = s_exc.BadTypeValu if oper == '?=' else ()
@@ -4697,9 +4765,7 @@ class EditTagPropDel(Edit):
     '''
     async def run(self, runt, genr):
 
-        if runt.readonly:
-            mesg = 'Storm runtime is in readonly mode, cannot create or edit nodes and other graph data.'
-            raise self.addExcInfo(s_exc.IsReadOnly(mesg=mesg))
+        self.reqNotReadOnly(runt)
 
         async for node, path in genr:
 

synapse/lib/auth.py

@@ -1545,6 +1545,7 @@ async def setPasswd(self, passwd, nexs=True, enforce_policy=True):
         # Prevent empty string or non-string values
         if passwd is None:
             shadow = None
+            enforce_policy = False
         elif passwd and isinstance(passwd, str):
             shadow = await s_passwd.getShadowV2(passwd=passwd)
         else: