Merge branch 'release/1.2.0'

mitre · May 1, 2018 · bf0ef9e · bf0ef9e
2 parents eca9e2d + 3e89001
commit bf0ef9e
Show file tree

Hide file tree

Showing 17 changed files with 849 additions and 43 deletions.
diff --git a/multiscanner.py b/multiscanner.py
@@ -6,6 +6,8 @@
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals, with_statement)
 
+__version__ = '1.2.0'
+
 import codecs
 import configparser
 import datetime

diff --git a/requirements.txt b/requirements.txt
@@ -15,6 +15,8 @@ requests
 ssdeep
 tika
 yara-python
+#Required for STIX2 content
+stix2
 #Required for PDF
 reportlab
 #Required by API

diff --git a/setup.cfg b/setup.cfg
@@ -1,9 +1,12 @@
 [bumpversion]
-current_version = 1.1.1
+current_version = 1.2.0
 commit = False
 tag = False
 
+[bumpversion:file:multiscanner.py]
+
 [flake8]
 ignore = E126,E127,E128,E402
 max-line-length = 120
 exclude = .git,__pycache__,libs/pdfparser.py,libs/office_meta.py
+
diff --git a/setup.py b/setup.py
@@ -29,7 +29,7 @@ def recursive_dir_list(path, exclude=['.pyc', '__pycache__']):
 
 setup(
     name='multiscanner',
-    version='1.1.1',
+    version='1.2.0',
     url='https://github.com/MITRECND/multiscanner',
     license='MPL 2.0',
     author='Drew Bonasera',
@@ -49,6 +49,8 @@ def recursive_dir_list(path, exclude=['.pyc', '__pycache__']):
         'python-magic',
         'requests',
         'ssdeep',
+        # Required for STIX2 content
+        'stix2',
         # Required by PDF
         'reportlab',
         # Required by API

diff --git a/storage/elasticsearch_storage.py b/storage/elasticsearch_storage.py
@@ -281,7 +281,7 @@ def get_report(self, sample_id, timestamp):
                         }},
                         {
                             "term": {
-                                "Scan Time": ts
+                                "Scan Metadata.Scan Time": ts
                             }
                         }
                     ]
@@ -490,6 +490,24 @@ def delete(self, report_id):
             # TODO: log exception
             return False
 
+    def delete_by_task_id(self, task_id):
+        query = {
+            "query": {
+                "term": {
+                    "Scan Metadata.Task ID": task_id
+                }
+            }
+        }
+
+        try:
+            self.es.delete_by_query(
+                index=self.index, doc_type=self.doc_type, body=query
+            )
+            return True
+        except Exception as e:
+            # TODO: log exception
+            return False
+
     def teardown(self):
         pass
 

diff --git a/tests/test_api.py b/tests/test_api.py
@@ -165,12 +165,6 @@ def test_get_updated_task(self):
         self.assertEqual(resp.status_code, api.HTTP_OK)
         self.assertDictEqual(json.loads(resp.get_data().decode()), expected_response)
 
-    def test_delete_nonexistent_task(self):
-        expected_response = api.TASK_NOT_FOUND
-        resp = self.app.delete('/api/v1/tasks/2')
-        self.assertEqual(resp.status_code, api.HTTP_NOT_FOUND)
-        self.assertDictEqual(json.loads(resp.get_data().decode()), expected_response)
-
 
 class TestTaskDeleteCase(APITestCase):
     def setUp(self):
@@ -179,13 +173,17 @@ def setUp(self):
         # populate the DB w/ a task
         post_file(self.app)
 
-    def test_delete_task(self):
+    @mock.patch('api.handler')
+    def test_delete_task(self, mock_handler):
+        mock_handler.delete_by_task_id.return_value = True
         expected_response = {'Message': 'Deleted'}
         resp = self.app.delete('/api/v1/tasks/1')
         self.assertEqual(resp.status_code, api.HTTP_OK)
         self.assertDictEqual(json.loads(resp.get_data().decode()), expected_response)
 
-    def test_delete_nonexistent_task(self):
+    @mock.patch('api.handler')
+    def test_delete_nonexistent_task(self, mock_handler):
+        mock_handler.delete_by_task_id.return_value = False
         expected_response = api.TASK_NOT_FOUND
         resp = self.app.delete('/api/v1/tasks/2')
         self.assertEqual(resp.status_code, api.HTTP_NOT_FOUND)

diff --git a/tests/test_elasticsearch.py b/tests/test_elasticsearch.py
@@ -66,7 +66,7 @@ def test_get_report(self, mock_get, mock_search):
         args, kwargs = mock_search.call_args_list[0]
         self.assertEqual(kwargs['index'], ElasticSearchStorage.DEFAULTCONF['index'])
         self.assertEqual(kwargs['body']['query']['bool']['must'][0]['has_parent']['query']['term']['_id'], TEST_ID)
-        self.assertEqual(kwargs['body']['query']['bool']['must'][1]['term']['Scan Time'], TEST_TS)
+        self.assertEqual(kwargs['body']['query']['bool']['must'][1]['term']['Scan Metadata.Scan Time'], TEST_TS)
         self.assertEqual(kwargs['doc_type'], ElasticSearchStorage.DEFAULTCONF['doc_type'])
 
         mock_get.assert_any_call(index=ElasticSearchStorage.DEFAULTCONF['index'], id=TEST_ID, doc_type='sample')

diff --git a/tests/utils/stix2_generator/sample_report.json b/tests/utils/stix2_generator/sample_report.json
@@ -0,0 +1,189 @@
+{
+  "Report": {
+    "Cuckoo Sandbox": {
+      "dropped": [
+        {
+          "crc32": "EDB36492",
+          "filepath": "C:\\Users\\some_user\\AppData\\Local\\Temp\\n2422\\s2429.exe.zip",
+          "md5": "d659e8900ea3fabe425882debed0c494",
+          "name": "1acf42374fb021fd_s2429.exe.zip",
+          "path": "/opt/cuckoo/.cuckoo/storage/analyses/22605/files/1acf42374fb021fd_s2429.exe.zip",
+          "pids": [
+            2272
+          ],
+          "sha1": "388e6816aff442e13cb546cfacd0c1d75b59b5b1",
+          "sha256": "1acf42374fb021fd1172df27a06f72e0e59f69a0bfaaaaea56f28dff6af01110",
+          "sha512": "f7e2de13afe330c96be43320968fc1152ef30562cd5e51a2b60306caffdea50745b1d515112cd09b0aaf1ba33c64bdd835d9999ec01f09aae8dbe01407d98e82",
+          "size": 173228,
+          "ssdeep": "3072:v8O0PPXlpAmOvDtu31DunkJdmAOIAT3B/WAyU98SJ4MWFYAkOymiTG4czJE:kdPP1Cm+OKYdmoqH8SSpkOye4czO",
+          "type": "Zip archive data, at least v2.0 to extract"
+        },
+        {
+          "crc32": "D29343CF",
+          "filepath": "C:\\Users\\some_user\\AppData\\Local\\Temp\\n2422\\s2429.exe",
+          "md5": "13b0085a03720e67fb8c73db3f14609e",
+          "name": "f9449897f9ca99b9_s2429.exe",
+          "path": "/opt/cuckoo/.cuckoo/storage/analyses/22605/files/f9449897f9ca99b9_s2429.exe",
+          "pids": [
+            2272
+          ],
+          "sha1": "ddf811f21e6c066b644d03e6751e16efb0fbecce",
+          "sha256": "f9449897f9ca99b99837ad322c8b6737e7a47e3827b6a4c073c6ca8911d8c340",
+          "sha512": "39b95dce14b3eea6f191d4dbaaff87ebbc8f3b6982e7b4ee5ebeed83d3b7397441665f25dec5eb9f8a1f3b12f4ddcd604d5852b781f592488263161c0d620e82",
+          "size": 421056,
+          "ssdeep": "6144:63hJxWjDKn4yTxz12wj/CF6J2Os+WX+ugnZJFNpluJHA4:6RJWDsTxzIwj/CF6FR6+zcO4",
+          "type": "PE32 executable (GUI) Intel 80386 Mono/.Net assembly, for MS Windows"
+        }
+      ],
+      "signatures": [
+        {
+          "description": "This executable is signed",
+          "name": "has_authenticode",
+          "severity": 1
+        },
+        {
+          "description": "Checks amount of memory in system, this can be used to detect virtual machines that have a low amount of memory available",
+          "markcount": 1,
+          "marks": [
+            {
+              "call": {
+                "api": "GlobalMemoryStatusEx",
+                "category": "system",
+                "return_value": 1,
+                "status": 1,
+                "tid": 1156,
+                "time": 1508411224.064626
+              },
+              "cid": 4115,
+              "pid": 2272,
+              "type": "call"
+            }
+          ],
+          "name": "antivm_memory_available",
+          "severity": 1
+        },
+        {
+          "description": "Potentially malicious URLs were found in the process memory dump",
+          "markcount": 3,
+          "marks": [
+            {
+              "category": "url",
+              "description": null,
+              "ioc": "http://ns.adobe.com/xap/1.0/mm/",
+              "type": "ioc"
+            },
+            {
+              "category": "url",
+              "description": null,
+              "ioc": "http://ns.adobe.com/xap/1.0/sType/ResourceRef",
+              "type": "ioc"
+            },
+            {
+              "category": "url",
+              "description": null,
+              "ioc": "http://ns.adobe.com/xap/1.0/",
+              "type": "ioc"
+            }
+          ],
+          "name": "memdump_urls",
+          "severity": 2
+        },
+        {
+          "description": "Performs some HTTP requests",
+          "markcount": 14,
+          "marks": [
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://www.msftncsi.com/ncsi.txt",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab?ec38990cc55170ab",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "POST http://tools.google.com/service/update2?cup2key=6:2144477707&cup2hreq=a6c83ff1daef97153eb6f265f9181edc5cea9a80f527aea825c28f6307c1fdfc",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "POST http://tools.google.com/service/update2?cup2key=6:3255292227&cup2hreq=a6c83ff1daef97153eb6f265f9181edc5cea9a80f527aea825c28f6307c1fdfc",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "POST http://tools.google.com/service/update2?cup2key=6:1128284371&cup2hreq=a6c83ff1daef97153eb6f265f9181edc5cea9a80f527aea825c28f6307c1fdfc",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "POST http://tools.google.com/service/update2?cup2key=6:1439439368&cup2hreq=a6c83ff1daef97153eb6f265f9181edc5cea9a80f527aea825c28f6307c1fdfc",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/disallowedcertstl.cab?075dc50dacf9f2bb",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab?31308c2120fea4bc",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab?6ecb1b8de9d8006f",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab?91c8a9092e8cb67a",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab?1390637153eb96bd",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab?b16bed41061b4861",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/authrootstl.cab?9a8ede518893069d",
+              "type": "ioc"
+            },
+            {
+              "category": "request",
+              "description": null,
+              "ioc": "GET http://go.microsoft.com/fwlink/?LinkId=544713",
+              "type": "ioc"
+            }
+          ],
+          "name": "network_http",
+          "severity": 2
+        }
+      ]
+    },
+    "MD5": "34303fdb55e5d0f1142bb07eed2064cb",
+    "SHA1": "91fd2d2935aedcb47271b54cd22f8fe3b30c17fd",
+    "SHA256": "90b1e39282dbda2341d91b87ca161afe564b7d3b4f82f25b3f1dce3fa857226c"
+  }
+}