Merge branch 'master' into rakuyo/tag-lift-optimize-7180

synapse/lib/cell.py

@@ -1189,16 +1189,24 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
         self.cellinfo = await node.dict()
         self.onfini(node)
 
-        node = await self.hive.open(('cellvers',))
-        self.cellvers = await node.dict(nexs=True)
+        # Check the cell version didn't regress
+        if (lastver := self.cellinfo.get('cell:version')) is not None and self.VERSION < lastver:
+            mesg = f'Cell version regression ({self.getCellType()}) is not allowed! Stored version: {lastver}, current version: {self.VERSION}.'
+            logger.error(mesg)
+            raise s_exc.BadVersion(mesg=mesg, currver=self.VERSION, lastver=lastver)
 
-        if self.inaugural:
-            await self.cellinfo.set('synapse:version', s_version.version)
+        await self.cellinfo.set('cell:version', self.VERSION)
 
-        synvers = self.cellinfo.get('synapse:version')
+        # Check the synapse version didn't regress
+        if (lastver := self.cellinfo.get('synapse:version')) is not None and s_version.version < lastver:
+            mesg = f'Synapse version regression ({self.getCellType()}) is not allowed! Stored version: {lastver}, current version: {s_version.version}.'
+            logger.error(mesg)
+            raise s_exc.BadVersion(mesg=mesg, currver=s_version.version, lastver=lastver)
 
-        if synvers is None or synvers < s_version.version:
-            await self.cellinfo.set('synapse:version', s_version.version)
+        await self.cellinfo.set('synapse:version', s_version.version)
+
+        node = await self.hive.open(('cellvers',))
+        self.cellvers = await node.dict(nexs=True)
 
         self.auth = await self._initCellAuth()
 

synapse/models/infotech.py

@@ -713,6 +713,13 @@ def getModelDefs(self):
                     'doc': 'A MITRE ATT&CK Campaign ID.',
                     'ex': 'C0028',
                 }),
+                ('it:mitre:attack:datasource', ('str', {'regex': r'^DS[0-9]{4}$'}), {
+                    'doc': 'A MITRE ATT&CK Datasource ID.',
+                    'ex': 'DS0026',
+                }),
+                ('it:mitre:attack:data:component', ('guid', {}), {
+                        'doc': 'A MITRE ATT&CK data component.',
+                }),
                 ('it:mitre:attack:flow', ('guid', {}), {
                     'doc': 'A MITRE ATT&CK Flow diagram.',
                 }),
@@ -1517,6 +1524,10 @@ def getModelDefs(self):
                                            'uniq': True, 'sorted': True, 'split': ','}), {
                         'doc': 'An array of ATT&CK tactics that include this technique.',
                     }),
+                    ('data:components', ('array', {'type': 'it:mitre:attack:data:component',
+                                                   'uniq': True, 'sorted': True}), {
+                        'doc': 'An array of MITRE ATT&CK data components that detect the ATT&CK technique.',
+                    }),
                 )),
                 ('it:mitre:attack:software', {}, (
                     ('software', ('it:prod:soft', {}), {
@@ -1636,6 +1647,27 @@ def getModelDefs(self):
                     ('author:contact', ('ps:contact', {}), {
                         'doc': 'The contact information for the author of the ATT&CK Flow diagram.'}),
                 )),
+                ('it:mitre:attack:datasource', {}, (
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'doc': 'The name of the datasource.'}),
+                    ('description', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'A description of the datasource.'}),
+                    ('references', ('array', {'type': 'inet:url', 'uniq': True, 'sorted': True}), {
+                        'doc': 'An array of URLs that document the datasource.',
+                    }),
+                )),
+                ('it:mitre:attack:data:component', {}, (
+                    ('name', ('str', {'lower': True, 'onespace': True}), {
+                        'ro': True,
+                        'doc': 'The name of the data component.'}),
+                    ('description', ('str', {}), {
+                        'disp': {'hint': 'text'},
+                        'doc': 'A description of the data component.'}),
+                    ('datasource', ('it:mitre:attack:datasource', {}), {
+                        'ro': True,
+                        'doc': 'The datasource this data component belongs to.'}),
+                )),
                 ('it:dev:int', {}, ()),
                 ('it:dev:pipe', {}, ()),
                 ('it:dev:mutex', {}, ()),

synapse/tests/test_lib_cell.py

@@ -2668,3 +2668,58 @@ async def test_cell_check_sysctl(self):
         stream.seek(0)
         data = stream.read()
         self.len(0, data, msg=data)
+
+    async def test_cell_version_regression(self):
+        oldver = (0, 1, 0)
+        newver = (0, 2, 0)
+
+        class TestCell(s_cell.Cell):
+            VERSION = newver
+
+        with self.getTestDir() as dirn:
+            async with self.getTestCell(TestCell, dirn=dirn):
+                pass
+
+            with self.raises(s_exc.BadVersion) as exc:
+                with mock.patch.object(TestCell, 'VERSION', oldver):
+                    with self.getLoggerStream('synapse.lib.cell') as stream:
+                        async with self.getTestCell(TestCell, dirn=dirn):
+                            pass
+
+            mesg = f'Cell version regression (testcell) is not allowed! Stored version: {newver}, current version: {oldver}.'
+            self.eq(exc.exception.get('mesg'), mesg)
+            self.eq(exc.exception.get('currver'), oldver)
+            self.eq(exc.exception.get('lastver'), newver)
+
+            stream.seek(0)
+            data = stream.read()
+            self.isin(mesg, data)
+
+            async with self.getTestCell(TestCell, dirn=dirn):
+                pass
+
+        with self.getTestDir() as dirn:
+            async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                pass
+
+            synver = list(s_version.version)
+            synver[1] -= 1
+            synver = tuple(synver)
+
+            with self.raises(s_exc.BadVersion) as exc:
+                with mock.patch.object(s_version, 'version', synver):
+                    with self.getLoggerStream('synapse.lib.cell') as stream:
+                        async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                            pass
+
+            mesg = f'Synapse version regression (cell) is not allowed! Stored version: {s_version.version}, current version: {synver}.'
+            self.eq(exc.exception.get('mesg'), mesg)
+            self.eq(exc.exception.get('currver'), synver)
+            self.eq(exc.exception.get('lastver'), s_version.version)
+
+            stream.seek(0)
+            data = stream.read()
+            self.isin(mesg, data)
+
+            async with self.getTestCell(s_cell.Cell, dirn=dirn):
+                pass

synapse/tests/test_model_infotech.py

@@ -58,6 +58,42 @@ async def test_infotech_basics(self):
             self.eq(nodes[0].get('techniques'), ('T0100', 'T0200'))
             self.eq(nodes[0].get('isnow'), 'G0110')
 
+            desc = 'A database and set of services that allows administrators to manage permissions, access to network '
+            desc += 'resources, and stored data objects (user, group, application, or devices)(Citation: Microsoft AD '
+            desc += 'DS Getting Started)'
+            refs = (
+                'https://attack.mitre.org/datasources/DS0026',
+                'https://docs.microsoft.com/en-us/windows-server/identity/ad-ds/ad-ds-getting-started',
+            )
+            q = f'''
+            [ it:mitre:attack:datasource=DS0026
+                :name="Active Directory"
+                :description="{desc}"
+                :references=({",".join(refs)})
+            ]
+            '''
+            nodes = await core.nodes(q)
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef, ('it:mitre:attack:datasource', 'DS0026'))
+            self.eq(nodes[0].get('name'), 'active directory')
+            self.eq(nodes[0].get('description'), desc)
+            self.eq(nodes[0].get('references'), refs)
+
+            q = f'''
+            [ it:mitre:attack:data:component=(DS0026, "Active Directory Credential Request")
+                :name="Active Directory Credential Request"
+                :description="{desc}"
+                :datasource=DS0026
+            ] -+> it:mitre:attack:datasource
+            '''
+            nodes = await core.nodes(q)
+            self.len(2, nodes)
+            self.eq(nodes[0].get('name'), 'active directory credential request')
+            self.eq(nodes[0].get('description'), desc)
+            self.eq(nodes[0].get('datasource'), 'DS0026')
+            self.eq(nodes[1].ndef, ('it:mitre:attack:datasource', 'DS0026'))
+            dcguid = nodes[0].ndef[1]
+
             nodes = await core.nodes('''[
                 it:mitre:attack:tactic=TA0100
                     :name=tactilneck
@@ -88,8 +124,10 @@ async def test_infotech_basics(self):
                     :isnow=T1110
                     :tactics=(TA0200,TA0100,TA0100)
                     :matrix=enterprise
-            ]''')
-            self.len(1, nodes)
+                    :data:components+={ it:mitre:attack:data:component=(DS0026, "Active Directory Credential Request") }
+            ] -+> it:mitre:attack:data:component
+            ''')
+            self.len(2, nodes)
             self.eq(nodes[0].ndef, ('it:mitre:attack:technique', 'T0100'))
             self.eq(nodes[0].get('name'), 'lockpicking')
             self.eq(nodes[0].get('desc'), 'speedhackers')
@@ -101,6 +139,8 @@ async def test_infotech_basics(self):
             self.eq(nodes[0].get('status'), 'deprecated')
             self.eq(nodes[0].get('isnow'), 'T1110')
             self.eq(nodes[0].get('matrix'), 'enterprise')
+            self.eq(nodes[0].get('data:components'), [dcguid])
+            self.eq(nodes[1].ndef, ('it:mitre:attack:data:component', dcguid))
 
             nodes = await core.nodes('''[
                 it:mitre:attack:software=S0100