Improvements to testing

.travis.yml

@@ -31,4 +31,3 @@ install:
   - pip install tox-travis pre-commit
 script:
   - tox
-  - pre-commit run --all-files

multiscanner/analytics/ssdeep_analytics.py

@@ -43,7 +43,7 @@ class SSDeepAnalytic:
 
     def __init__(self, debug=False):
         storage_conf = utils.get_config_path(MS_CONFIG, 'storage')
-        config_object = configparser.SafeConfigParser()
+        config_object = configparser.ConfigParser()
         config_object.optionxform = str
         config_object.read(storage_conf)
         conf = utils.parse_config(config_object)

multiscanner/common/utils.py

@@ -95,7 +95,7 @@ def get_config_path(config_file, component):
         storage
         api
         web"""
-    conf = configparser.SafeConfigParser()
+    conf = configparser.ConfigParser()
     conf.read(config_file)
     conf = parse_config(conf)
     try:

multiscanner/distributed/api.py

@@ -106,7 +106,7 @@ def default(self, obj):
 
 app = Flask(__name__)
 app.json_encoder = CustomJSONEncoder
-api_config_object = configparser.SafeConfigParser()
+api_config_object = configparser.ConfigParser()
 api_config_object.optionxform = str
 # TODO: Why does this multiscanner.common instead of just common?
 api_config_file = utils.get_config_path(MS_CONFIG, 'api')
@@ -159,7 +159,7 @@ def default(self, obj):
 storage_handler = StorageHandler(configfile=storage_conf)
 handler = storage_handler.load_required_module('ElasticSearchStorage')
 
-ms_config_object = configparser.SafeConfigParser()
+ms_config_object = configparser.ConfigParser()
 ms_config_object.optionxform = str
 ms_configfile = MS_CONFIG
 ms_config_object.read(ms_configfile)
@@ -279,7 +279,7 @@ def modules():
     filenames = [os.path.splitext(os.path.basename(f)) for f in files]
     module_names = [m[0] for m in filenames if m[1] == '.py']
 
-    ms_config = configparser.SafeConfigParser()
+    ms_config = configparser.ConfigParser()
     ms_config.optionxform = str
     ms_config.read(MS_CONFIG)
     modules = {}

multiscanner/distributed/celery_worker.py

@@ -35,7 +35,7 @@
     'tz': 'US/Eastern',
 }
 
-config_object = configparser.SafeConfigParser()
+config_object = configparser.ConfigParser()
 config_object.optionxform = str
 configfile = utils.get_config_path(MS_CONFIG, 'api')
 config_object.read(configfile)
@@ -53,7 +53,7 @@
 worker_config = config.get('celery')
 db_config = config.get('Database')
 
-storage_config_object = configparser.SafeConfigParser()
+storage_config_object = configparser.ConfigParser()
 storage_config_object.optionxform = str
 storage_configfile = utils.get_config_path(MS_CONFIG, 'storage')
 storage_config_object.read(storage_configfile)
@@ -148,7 +148,7 @@ def multiscanner_celery(file_, original_filename, task_id, file_hash, metadata,
 
     # Get the Scan Config that the task was run with and
     # add it to the task metadata
-    scan_config_object = configparser.SafeConfigParser()
+    scan_config_object = configparser.ConfigParser()
     scan_config_object.optionxform = str
     scan_config_object.read(config)
     full_conf = utils.parse_config(scan_config_object)

multiscanner/distributed/distributed_worker.py

@@ -67,7 +67,7 @@ def multiscanner_process(work_queue, config, batch_size, wait_seconds, delete, e
 
 
 def _read_conf(file_path):
-    conf = configparser.SafeConfigParser()
+    conf = configparser.ConfigParser()
     conf.optionxform = str
     with codecs.open(file_path, 'r', encoding='utf-8') as fp:
         conf.readfp(fp)

multiscanner/ms.py

@@ -489,7 +489,7 @@ def config_init(filepath, module_list=parseDir(MODULESDIR, recursive=True, exclu
 
     filepath - The config file to create
     """
-    config = configparser.SafeConfigParser()
+    config = configparser.ConfigParser()
     config.optionxform = str
 
     if filepath:
@@ -583,7 +583,7 @@ def multiscan(Files, recursive=False, configregen=False, configfile=CONFIG, conf
 
     # Read in config
     if configfile:
-        config_object = configparser.SafeConfigParser()
+        config_object = configparser.ConfigParser()
         config_object.optionxform = str
         # Regen the config if needed or wanted
         if configregen or not os.path.isfile(configfile):
@@ -893,15 +893,15 @@ def _init(args):
         else:
             print('Checking for missing modules in configuration...')
             ModuleList = parseDir(MODULESDIR, recursive=True, exclude=["__init__"])
-            config = configparser.SafeConfigParser()
+            config = configparser.ConfigParser()
             config.optionxform = str
             config.read(args.config)
             _write_missing_module_configs(ModuleList, config, filepath=args.config)
     else:
         config_init(args.config)
 
     # Init storage
-    config = configparser.SafeConfigParser()
+    config = configparser.ConfigParser()
     config.optionxform = str
     config.read(args.config)
     config = _get_main_config(config)
@@ -1010,7 +1010,7 @@ def _main():
         results = multiscan(filelist, configfile=args.config)
 
         # We need to read in the config for the parseReports call
-        config = configparser.SafeConfigParser()
+        config = configparser.ConfigParser()
         config.optionxform = str
         config.read(args.config)
         config = _get_main_config(config)

multiscanner/storage/basic_elasticsearch_storage.py

@@ -121,13 +121,13 @@ def normalize_list(self, array):
                 array[i] = self.same_type_lists(array[i])
         elif not self.check_same_types(array):
             for i in range(0, len(array)):
-                    if isinstance(array[i], list):
-                        array[i] = self.normalize_list(array[i])
-                    elif isinstance(array[i], dict):
-                        array[i] = self.same_type_lists(array[i])
-                    else:
-                        array[i] = str(array[i])
-                        if not self.warned_changed:
-                            print("WARNING: We changed some of the data types so that Elasticsearch wouldn't get angry")
-                            self.warned_changed = True
+                if isinstance(array[i], list):
+                    array[i] = self.normalize_list(array[i])
+                elif isinstance(array[i], dict):
+                    array[i] = self.same_type_lists(array[i])
+                else:
+                    array[i] = str(array[i])
+                    if not self.warned_changed:
+                        print("WARNING: We changed some of the data types so that Elasticsearch wouldn't get angry")
+                        self.warned_changed = True
         return array

multiscanner/storage/file.py

@@ -48,7 +48,7 @@ def store(self, results):
                     writedata = {}
             if writedata:
                 if metadata:
-                        writedata = {'Files': writedata, 'Metadata': metadata}
+                    writedata = {'Files': writedata, 'Metadata': metadata}
                 if self.config['gzip'] is True:
                     self.file_handle.write(
                         json.dumps(writedata, sort_keys=True, separators=(',', ':'),

multiscanner/storage/sql_driver.py

@@ -67,7 +67,7 @@ def __init__(self, config=None, configfile=CONFIG_FILE, regenconfig=False):
         self.db_engine = None
 
         # Configuration parsing
-        config_parser = configparser.SafeConfigParser()
+        config_parser = configparser.ConfigParser()
         config_parser.optionxform = str
 
         # (re)generate conf file if necessary

multiscanner/storage/storage.py

@@ -96,7 +96,7 @@ def __init__(self, configfile=MS_CONFIG, config=None, configregen=False):
         # Read in config
         if configfile:
             configfile = utils.get_config_path(MS_CONFIG, 'storage')
-            config_object = configparser.SafeConfigParser()
+            config_object = configparser.ConfigParser()
             config_object.optionxform = str
             # Regen the config if needed or wanted
             if configregen or not os.path.isfile(configfile):
@@ -262,7 +262,7 @@ def is_done(self, wait=False):
 def config_init(filepath, overwrite=False, storage_classes=None):
     if storage_classes is None:
         storage_classes = _get_storage_classes()
-    config_object = configparser.SafeConfigParser()
+    config_object = configparser.ConfigParser()
     config_object.optionxform = str
     if overwrite or not os.path.isfile(filepath):
         _write_main_config(config_object)

multiscanner/tests/module_tests/Metadefender/test_metadefender_module.py

@@ -154,9 +154,9 @@ def test_submit_sample_success(self, mock_get):
         '''
         print('Running test_submit_sample_success')
         submit_resp = Metadefender._submit_sample(RANDOM_INPUT_FILES[0], 'scan_url', 'user_agent')
-        self.assertEquals(submit_resp['status_code'], 200)
-        self.assertEquals(submit_resp['error'], None)
-        self.assertEquals(submit_resp['scan_id'], generate_scan_id(RANDOM_INPUT_FILES[0]))
+        self.assertEqual(submit_resp['status_code'], 200)
+        self.assertEqual(submit_resp['error'], None)
+        self.assertEqual(submit_resp['scan_id'], generate_scan_id(RANDOM_INPUT_FILES[0]))
 
     @mock.patch('Metadefender.requests.post', side_effect=mocked_requests_post_sample_failed_w_msg)
     def test_submit_sample_fail_unavailable(self, mock_get):
@@ -166,9 +166,9 @@ def test_submit_sample_fail_unavailable(self, mock_get):
         '''
         print('Running test_submit_sample_fail_unavailable')
         submit_resp = Metadefender._submit_sample(RANDOM_INPUT_FILES[1], 'scan_url', 'user_agent')
-        self.assertEquals(submit_resp['status_code'], 500)
-        self.assertEquals(submit_resp['error'], MSG_SERVER_UNAVAILABLE)
-        self.assertEquals(submit_resp['scan_id'], None)
+        self.assertEqual(submit_resp['status_code'], 500)
+        self.assertEqual(submit_resp['error'], MSG_SERVER_UNAVAILABLE)
+        self.assertEqual(submit_resp['scan_id'], None)
 
     @mock.patch('Metadefender.requests.post', side_effect=mocked_requests_post_sample_failed_no_msg)
     def test_submit_sample_fail_unavailable_no_msg(self, mock_get):
@@ -178,9 +178,9 @@ def test_submit_sample_fail_unavailable_no_msg(self, mock_get):
         '''
         print('Running test_submit_sample_fail_unavailable_no_msg')
         submit_resp = Metadefender._submit_sample(RANDOM_INPUT_FILES[1], 'scan_url', 'user_agent')
-        self.assertEquals(submit_resp['status_code'], 500)
-        self.assertEquals(submit_resp['error'], Metadefender.MD_HTTP_ERR_CODES[500])
-        self.assertEquals(submit_resp['scan_id'], None)
+        self.assertEqual(submit_resp['status_code'], 500)
+        self.assertEqual(submit_resp['error'], Metadefender.MD_HTTP_ERR_CODES[500])
+        self.assertEqual(submit_resp['scan_id'], None)
 
     # ---------------------------------------------------------------------
     # This section tests the logic for parsing Metadefender's responses
@@ -195,26 +195,26 @@ def test_get_results_200_success(self, mock_get):
         print('Running test_get_results_200_success')
         report_resp = Metadefender._retrieve_scan_results('results_url', SCAN_IDS[0])
         is_scan_complete, parsed_resp = Metadefender._parse_scan_result(report_resp)
-        self.assertEquals(is_scan_complete, True)
-        self.assertEquals(parsed_resp['overall_status'], Metadefender.STATUS_SUCCESS)
+        self.assertEqual(is_scan_complete, True)
+        self.assertEqual(parsed_resp['overall_status'], Metadefender.STATUS_SUCCESS)
 
         engine_results = parsed_resp['engine_results']
         for engine_result in engine_results:
             engine_name = engine_result['engine_name']
             scan_result = engine_result['scan_result']
             threat_found = engine_result['threat_found']
             if engine_name == 'ClamAV':
-                self.assertEquals(scan_result, 'Infected/Known')
-                self.assertEquals(threat_found, 'Heuristics.PDF.ObfuscatedNameObject')
+                self.assertEqual(scan_result, 'Infected/Known')
+                self.assertEqual(threat_found, 'Heuristics.PDF.ObfuscatedNameObject')
             elif engine_name == 'Ahnlab':
-                self.assertEquals(scan_result, 'Infected/Known')
-                self.assertEquals(threat_found, 'Trojan/Win32.Inject.C1515213')
+                self.assertEqual(scan_result, 'Infected/Known')
+                self.assertEqual(threat_found, 'Trojan/Win32.Inject.C1515213')
             elif engine_name == 'ESET':
-                self.assertEquals(scan_result, 'No threats Found')
-                self.assertEquals(threat_found, '')
+                self.assertEqual(scan_result, 'No threats Found')
+                self.assertEqual(threat_found, '')
             elif engine_name == 'Avira':
-                self.assertEquals(scan_result, 'No threats Found')
-                self.assertEquals(threat_found, '')
+                self.assertEqual(scan_result, 'No threats Found')
+                self.assertEqual(threat_found, '')
             else:
                 self.fail('Unexpected Engine: %s' % engine_name)
 
@@ -228,8 +228,8 @@ def test_get_results_200_not_found(self, mock_get):
         print('Running test_get_results_200_not_found')
         report_resp = Metadefender._retrieve_scan_results('results_url', SCAN_IDS[0])
         is_scan_complete, parsed_resp = Metadefender._parse_scan_result(report_resp)
-        self.assertEquals(is_scan_complete, False)
-        self.assertEquals(parsed_resp['overall_status'], Metadefender.STATUS_PENDING)
+        self.assertEqual(is_scan_complete, False)
+        self.assertEqual(parsed_resp['overall_status'], Metadefender.STATUS_PENDING)
 
         engine_results = parsed_resp['engine_results']
         if len(engine_results) != 0:
@@ -244,8 +244,8 @@ def test_get_results_200_succes_in_progress(self, mock_get):
         print('Running test_get_results_200_succes_in_progress')
         report_resp = Metadefender._retrieve_scan_results('results_url', SCAN_IDS[0])
         is_scan_complete, parsed_resp = Metadefender._parse_scan_result(report_resp)
-        self.assertEquals(is_scan_complete, False)
-        self.assertEquals(parsed_resp['overall_status'], Metadefender.STATUS_PENDING)
+        self.assertEqual(is_scan_complete, False)
+        self.assertEqual(parsed_resp['overall_status'], Metadefender.STATUS_PENDING)
         msg = parsed_resp['msg']
         if 'percent complete: 10' not in msg:
             self.fail('Progress percentage not present')
@@ -265,9 +265,9 @@ def test_scan_complete_success(self, mock_post, mock_get):
         print('Running test_scan_complete_success')
         resultlist, metadata = Metadefender.scan(RANDOM_INPUT_FILES,
                                                  conf=self.create_conf_short_timeout())
-        self.assertEquals(len(resultlist), len(RANDOM_INPUT_FILES))
+        self.assertEqual(len(resultlist), len(RANDOM_INPUT_FILES))
         for scan_res in resultlist:
-            self.assertEquals(scan_res[1]['overall_status'], Metadefender.STATUS_SUCCESS)
+            self.assertEqual(scan_res[1]['overall_status'], Metadefender.STATUS_SUCCESS)
 
     @mock.patch('Metadefender.requests.get', side_effect=mocked_requests_get_sample_200_in_progress)
     @mock.patch('Metadefender.requests.post', side_effect=mocked_requests_post_sample_submitted)
@@ -278,9 +278,9 @@ def test_scan_timeout_scan_in_progress(self, mock_post, mock_get):
         print('Running test_scan_timeout_scan_in_progress')
         resultlist, metadata = Metadefender.scan(RANDOM_INPUT_FILES,
                                                  conf=self.create_conf_short_timeout())
-        self.assertEquals(len(resultlist), len(RANDOM_INPUT_FILES))
+        self.assertEqual(len(resultlist), len(RANDOM_INPUT_FILES))
         for scan_res in resultlist:
-            self.assertEquals(scan_res[1]['overall_status'], Metadefender.STATUS_TIMEOUT)
+            self.assertEqual(scan_res[1]['overall_status'], Metadefender.STATUS_TIMEOUT)
 
 
 if __name__ == "__main__":

multiscanner/tests/test_frontend.py

@@ -26,7 +26,7 @@
 try:
     opts = Options()
     opts.add_argument('-headless')
-    driver = webdriver.Firefox(firefox_options=opts)
+    driver = webdriver.Firefox(options=opts)
 except Exception as e:
     pytestmark = pytest.mark.skip
 

multiscanner/web/app.py

@@ -32,7 +32,7 @@
 app = Flask(__name__)
 
 # Finagle Flask to read config from .ini file instead of .py file
-web_config_object = configparser.SafeConfigParser()
+web_config_object = configparser.ConfigParser()
 web_config_object.optionxform = str
 web_config_file = utils.get_config_path(MS_CONFIG, 'web')
 web_config_object.read(web_config_file)

tox.ini

@@ -6,6 +6,7 @@ deps = -rrequirements-dev.txt
 commands =
   multiscanner init
   pytest
+  pre-commit run --all-files
 
 passenv = CI TRAVIS TRAVIS_*