Merge branch 'master' into visi-aha-gather

changes/15f2326e94fb0c6a0dff8325285ef354.yaml

@@ -0,0 +1,6 @@
+---
+desc: Fixed an issue where the deprecated types ``edge`` and ``timeedge`` were not annotated
+  as such by the ``getModelDict()`` API.
+prs: []
+type: bug
+...

changes/3c38d224d8b0667a7eec4b3a5ea8f89d.yaml

@@ -0,0 +1,6 @@
+---
+desc: Fixed an issue where certain User and Role properties could be modified via
+  Storm and adversely affect the in-memory representation of those objects.
+prs: []
+type: bug
+...

changes/4a4d016df5220dbe11c742d168326ee0.yaml

@@ -0,0 +1,7 @@
+---
+desc: Added ``$lib.cell.iden`` to retrieve the iden of the Cortex which the Storm
+  query is executing on. Unlike ``$lib.cell.getCellInfo().cell.iden``, this value
+  is available to non-admin users.
+prs: []
+type: feat
+...

changes/4dc8e5d43f4cd25953b363a3ef469d7a.yaml

@@ -0,0 +1,5 @@
+---
+desc: Deprecated ``$lib.infosec.cvss.calculate()`` and ``$lib.infosec.cvss.calculateFromProps()``.
+prs: []
+type: deprecation
+...

changes/cd12a6feba9161fe9651a4e3bf00bb04.yaml

@@ -0,0 +1,6 @@
+---
+desc: Added ``--find`` option to ``auth.perms.list`` to easily filter permission
+  results.
+prs: []
+type: feat
+...

docs/synapse/datamodel_deprecation.rst

@@ -10,8 +10,7 @@ to deprecate model elements which are no longer useful. These elements may repre
 relationships which are better captured with newer elements; concepts which are better
 represented by convention; or other issues. As such, model elements (types, forms,
 and properties) which are deprecated should no longer be used for new data modeling.
-Deprecated model elements will be removed in a future Synapse release, no earlier than
-``v3.0.0``.
+Deprecated model elements will be removed in a future major version Synapse release.
 
 For deprecated model elements, suggested alternatives will be provided and example Storm
 queries which can be used to migrate data in such a fashion.
@@ -38,115 +37,3 @@ When Deprecated model elements are used in a Cortex, the following log events wi
 Deleting nodes which use deprecated model elements does not trigger warnings, since that
 would normally be done after an associated data migration and would be excessive in
 the event of a large migration.
-
-Deprecated Model Elements
--------------------------
-
-The following elements are deprecated.
-
-Types
-+++++
-
-- `file:string`
-    - -(refs)> it:dev:str
-- `it:reveng:funcstr`
-    - Please use the `:strings` array property on the `it:reveng:function` form.
-- `lang:idiom`
-    - Please use `lang:translation` instead.
-- `lang:trans`
-    - Please use `lang:translation` instead.
-- `ou:hasalias`
-    - `ou:hasalias` is deprecated in favor of the `:alias` property on `ou:org` nodes.
-- `ou:meet:attendee`
-    - `ou:meet:attendee` has been superseded by `ou:attendee`. `ou:attendee` has the `:meet` property to denote what meeting the attendee attended.
-- `ou:conference:attendee`
-    - `ou:conference:attendee` has been superseded by `ou:attendee`. `ou:attendee` has the `:conference` property to denote what conference the attendee attended.
-- `ou:conference:event:attendee`
-    - `ou:conference:attendee` has been superseded by `ou:attendee`. `ou:attendee` has the `:conference` property to denote what conference event the attendee attended.
-- `ou:member`
-    - `ou:member` has been superseded by `ou:position`.
-- `ps:persona`
-    - Please use the `ps:person` or `ps:contact` types.
-- `ps:person:has`
-    - Please use `edge:has` or a light edge.
-- `ps:persona:has`
-    - Please use `ps:person` or `ps:context` in combination with an `edge:has` or a light edge.
-- `inet:ssl:cert`
-    - `inet:ssl:cert` is deprecated in favor of `inet:tls:servercert` and `inet:tls:clientcert`.
-
-Forms
-+++++
-
-Consistent with the deprecated types, the following forms are deprecated:
-- `file:string`
-- `it:reveng:funcstr`
-- `lang:idiom`
-- `lang:trans`
-- `ou:hasalias`
-- `ou:meet:attendee`
-- `ou:conference:attendee`
-- `ou:conference:event:attendee`
-- `ou:member`
-- `ps:person:has`
-- `ps:persona`
-- `ps:persona:has`
-- `inet:ssl:cert`
-
-Properties
-++++++++++
-
-- `ps:person`
-    - `:img`
-        - `ps:person:img` has been renamed to `ps:person:photo`.
-
-- `it:prod:soft`
-    - `author:org`, `author:acct`, `author:email`, and `author:person`
-        - These properties have been collected into the `it:prod:soft:author` property, which is typed as a `ps:contact`.
-
-- `media:news`
-    - `:author`
-        - The `media:news:author` property has been superseded by the array property of `media:news:authors`, which is an array of type `ps:contact`.
-
-- `file:subfile`
-    - `:name`
-        - The `file:subfile:name` property has been superseded by the property `file:subfile:path`, which is typed as `file:path`.
-
-- `ou:org`
-    - `:naics` and `:sic`
-        - The `ou:org:naics` and `ou:org:sic` properties has been collected into the `ou:org:industries` property, which is an array of type `ou:industry`.
-    - `:has`
-        - Please use an `edge:has` node or a light edge.
-
-- `risk:attack`
-    - `:actor:org`
-        - Please use the `:attacker` `ps:contact` property to allow entity resolution.
-    - `:actor:person`
-        - Please use the `:attacker` `ps:contact` property to allow entity resolution.
-    - `:target:org`
-        - Please use the `:target` `ps:contact` property to allow entity resolution.
-    - `:target:person`
-        - Please use the `:target` `ps:contact` property to allow entity resolution.
-
-- `ou:campaign`
-    - `:type`
-        - Please use the `:camptype` `taxonomy` property.
-
-- `it:host`
-    - `:manu`
-        - This property has been superseded by the `it:prod:hardware:make` property, which is typed as `ou:name`.
-    - `:model`
-        - This property has been superseded by the `it:prod:hardware:model` property, which is typed as string.
-
-- `it:exec:proc`
-    - `:user`
-        - Please use the `:account` `it:exec:proc` property to link processes to users.
-
-- `it:prod:hardware`
-    - `:make`
-        - The `:make` property has been superseded by the properties `it:prod:hardware:manufacturer` and  `it:prod:hardware:manufacturer:name`, which are typed as `ou:org` and `ou:name` respectively.
-
-Edges
-+++++
-
-- `* -(seenat)> geo:telem`
-    - Please use the `:node` `geo:telem` property to record the observed node.

synapse/datamodel.py

@@ -583,11 +583,11 @@ def __init__(self, core=None):
         item = s_types.Array(self, 'array', info, {'type': 'int'})
         self.addBaseType(item)
 
-        info = {'doc': 'An digraph edge base type.'}
+        info = {'doc': 'An digraph edge base type.', 'deprecated': True}
         item = s_types.Edge(self, 'edge', info, {})
         self.addBaseType(item)
 
-        info = {'doc': 'An digraph edge base type with a unique time.'}
+        info = {'doc': 'An digraph edge base type with a unique time.', 'deprecated': True}
         item = s_types.TimeEdge(self, 'timeedge', info, {})
         self.addBaseType(item)
 

synapse/lib/auth.py

@@ -881,13 +881,14 @@ class Role(Ruler):
     set of rules can be applied to multiple users.
     '''
     def pack(self):
-        return {
+        ret = {
             'type': 'role',
             'iden': self.iden,
             'name': self.name,
             'rules': self.info.get('rules'),
             'authgates': self.authgates,
         }
+        return s_msgpack.deepcopy(ret)
 
     async def _setRulrInfo(self, name, valu, gateiden=None, nexs=True, mesg=None):
         if nexs:
@@ -956,7 +957,7 @@ def pack(self, packroles=False):
                 _roles.append(role.pack())
             roles = _roles
 
-        return {
+        ret = {
             'type': 'user',
             'iden': self.iden,
             'name': self.name,
@@ -968,6 +969,7 @@ def pack(self, packroles=False):
             'archived': self.info.get('archived'),
             'authgates': self.authgates,
         }
+        return s_msgpack.deepcopy(ret)
 
     async def _setRulrInfo(self, name, valu, gateiden=None, nexs=True, mesg=None):
         if nexs:

synapse/lib/stormlib/auth.py

@@ -583,12 +583,26 @@
     {
         'name': 'auth.perms.list',
         'descr': 'Display a list of the current permissions defined within the Cortex.',
-        'cmdargs': (),
+        'cmdargs': (
+            ('--find', {'type': 'str', 'help': 'A search string for permissions.'}),
+        ),
         'storm': '''
 
             for $pdef in $lib.auth.getPermDefs() {
                 $perm = $lib.str.join(".", $pdef.perm)
 
+                if $cmdopts.find {
+                    $find = $cmdopts.find.lower()
+                    $match = (
+                        $perm.lower().find($find) != (null) or
+                        $pdef.desc.lower().find($find) != (null) or
+                        $pdef.gate.lower().find($find) != (null) or
+                        ($pdef.ex and $pdef.ex.lower().find($find) != (null))
+                    )
+
+                    if (not $match) { continue }
+                }
+
                 $lib.print($perm)
                 $lib.print(`    {$pdef.desc}`)
                 $lib.print(`    gate: {$pdef.gate}`)

synapse/lib/stormlib/cell.py

@@ -120,6 +120,9 @@ class CellLib(s_stormtypes.Lib):
     A Storm Library for interacting with the Cortex.
     '''
     _storm_locals = (
+        {'name': 'iden', 'desc': 'The Cortex service identifier.',
+         'type': {'type': 'gtor', '_gtorfunc': '_getCellIden',
+                  'returns': {'type': 'str', 'desc': 'The Cortex service identifier.'}}},
         {'name': 'getCellInfo', 'desc': 'Return metadata specific for the Cortex.',
          'type': {'type': 'function', '_funcname': '_getCellInfo', 'args': (),
                   'returns': {'type': 'dict', 'desc': 'A dictionary containing metadata.', }}},
@@ -174,6 +177,10 @@ class CellLib(s_stormtypes.Lib):
     )
     _storm_lib_path = ('cell',)
 
+    def __init__(self, runt, name=()):
+        s_stormtypes.Lib.__init__(self, runt, name=name)
+        self.gtors['iden'] = self._getCellIden
+
     def getObjLocals(self):
         return {
             'getCellInfo': self._getCellInfo,
@@ -187,6 +194,10 @@ def getObjLocals(self):
             'uptime': self._uptime,
         }
 
+    @s_stormtypes.stormfunc(readonly=True)
+    async def _getCellIden(self):
+        return self.runt.snap.core.getCellIden()
+
     async def _hotFixesApply(self):
         if not self.runt.isAdmin():
             mesg = '$lib.cell.stormFixesApply() requires admin privs.'

synapse/lib/stormlib/infosec.py

@@ -515,6 +515,7 @@ class CvssLib(s_stormtypes.Lib):
     '''
     _storm_locals = (
         {'name': 'calculate', 'desc': 'Calculate the CVSS score values for an input risk:vuln node.',
+         'deprecated': {'eolvers': 'v3.0.0'},
          'type': {'type': 'function', '_funcname': 'calculate',
                   'args': (
                       {'name': 'node', 'type': 'node',
@@ -527,6 +528,7 @@ class CvssLib(s_stormtypes.Lib):
                   'returns': {'type': 'dict', 'desc': 'A dictionary containing the computed score and subscores.', }
         }},
         {'name': 'calculateFromProps', 'desc': 'Calculate the CVSS score values from a props dict.',
+         'deprecated': {'eolvers': 'v3.0.0'},
          'type': {'type': 'function', '_funcname': 'calculateFromProps',
                   'args': (
                       {'name': 'props', 'type': 'dict',

synapse/tests/test_cortex.py

@@ -3048,7 +3048,7 @@ async def test_cortex_coreinfo(self):
 
             depr = [x for x in coreinfo['stormdocs']['libraries'] if x['path'] == ('lib', 'infosec', 'cvss')]
             self.len(1, depr)
-            self.len(2, [x for x in depr[0]['locals'] if x.get('deprecated')])
+            self.len(4, [x for x in depr[0]['locals'] if x.get('deprecated')])
 
     async def test_cortex_model_dict(self):
 
@@ -3107,6 +3107,9 @@ async def test_cortex_model_dict(self):
             self.nn(model['univs'].get('.created'))
             self.nn(model['univs'].get('.seen'))
 
+            self.true(model['types']['edge']['info'].get('deprecated'))
+            self.true(model['types']['timeedge']['info'].get('deprecated'))
+
     async def test_storm_graph(self):
 
         async with self.getTestCoreAndProxy() as (core, prox):

synapse/tests/test_lib_cell.py

@@ -1127,9 +1127,19 @@ async def test_cell_authv2(self):
                 await proxy.addUserRole(visi['iden'], ninjas['iden'])
                 await proxy.setUserEmail(visi['iden'], '[email protected]')
 
+                def1 = await core.getUserDef(visi['iden'])
+                def2 = await core.getUserDef(visi['iden'])
+                self.false(def1['authgates'] is def2['authgates'])
+                self.eq(def1, def2)
+
                 visi = await proxy.getUserDefByName('visi')
                 self.eq(visi['email'], '[email protected]')
 
+                def1 = await core.getRoleDef(ninjas['iden'])
+                def2 = await core.getRoleDef(ninjas['iden'])
+                self.false(def1['authgates'] is def2['authgates'])
+                self.eq(def1, def2)
+
                 self.true(await proxy.isUserAllowed(visi['iden'], ('foo', 'bar')))
                 self.true(await proxy.isUserAllowed(visi['iden'], ('hehe', 'haha')))
 

synapse/tests/test_lib_stormlib_auth.py

@@ -320,6 +320,17 @@ async def test_stormlib_auth(self):
             self.stormIsInPrint('Controls access to add a new view including forks.', msgs)
             self.stormIsInPrint('default: false', msgs)
 
+            msgs = await core.stormlist('auth.perms.list --find macro.')
+            self.stormIsInPrint('storm.macro.add', msgs)
+            self.stormIsInPrint('storm.macro.admin', msgs)
+            self.stormIsInPrint('storm.macro.edit', msgs)
+            self.stormNotInPrint('node.add.<form>', msgs)
+
+            msgs = await core.stormlist('auth.perms.list --find url')
+            self.stormIsInPrint('storm.lib.telepath.open.<scheme>', msgs)
+            self.stormIsInPrint('Controls the ability to open a telepath URL with a specific URI scheme.', msgs)
+            self.stormNotInPrint('node.add.<form>', msgs)
+
     async def test_stormlib_auth_default_allow(self):
         async with self.getTestCore() as core:
 
@@ -1003,6 +1014,21 @@ async def test_stormlib_auth_base(self):
             with self.raises(s_exc.DupIden):
                 await core.callStorm('$lib.auth.roles.add(walkers, iden=$iden)', opts=opts)
 
+            # The role & user.authgates local is a passthrough to the getRoleDef & getUserDef
+            # results, which are a pack()'d structure. Modifying the results of that structure
+            # does not persist.
+            q = '$u = $lib.auth.users.byname(root) $u.authgates.newp = ({}) return ($u)'
+            udef = await core.callStorm(q)
+            self.notin('newp', udef.get('authgates'))
+            q = '$u = $lib.auth.users.byname(root) return ( $lib.dict.has($u.authgates, newp) )'
+            self.false(await core.callStorm(q))
+
+            q = '$r = $lib.auth.roles.byname(all) $r.authgates.newp = ({}) return ($r)'
+            rdef = await core.callStorm(q)
+            self.notin('newp', rdef.get('authgates'))
+            q = '$r = $lib.auth.roles.byname(all) return ( $lib.dict.has($r.authgates, newp) )'
+            self.false(await core.callStorm(q))
+
     async def test_stormlib_auth_gateadmin(self):
 
         async with self.getTestCore() as core:

synapse/tests/test_lib_stormlib_cell.py

@@ -14,6 +14,9 @@ async def test_stormlib_cell(self):
 
         async with self.getTestCore() as core:
 
+            ret = await core.callStorm('return ( $lib.cell.iden )')
+            self.eq(ret, core.getCellIden())
+
             ret = await core.callStorm('return ( $lib.cell.getCellInfo() )')
             self.eq(ret, await core.getCellInfo())