Skip to content

Commit 74ef29d

Browse files
emmanvgemmanvg
authored
Merge pull request #184 from clenk/tox-improvements
Improvements to testing, use ConfigParser, and Tox configuration
2 parents 86e0145 + 406231c commit 74ef29d

File tree

15 files changed

+57
-57
lines changed

15 files changed

+57
-57
lines changed

‎.travis.yml

-1
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,3 @@ install:
3131
- pip install tox-travis pre-commit
3232
script:
3333
- tox
34-
- pre-commit run --all-files

‎multiscanner/analytics/ssdeep_analytics.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ class SSDeepAnalytic:
4343

4444
def __init__(self, debug=False):
4545
storage_conf = utils.get_config_path(MS_CONFIG, 'storage')
46-
config_object = configparser.SafeConfigParser()
46+
config_object = configparser.ConfigParser()
4747
config_object.optionxform = str
4848
config_object.read(storage_conf)
4949
conf = utils.parse_config(config_object)

‎multiscanner/common/utils.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ def get_config_path(config_file, component):
9595
storage
9696
api
9797
web"""
98-
conf = configparser.SafeConfigParser()
98+
conf = configparser.ConfigParser()
9999
conf.read(config_file)
100100
conf = parse_config(conf)
101101
try:

‎multiscanner/distributed/api.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def default(self, obj):
106106

107107
app = Flask(__name__)
108108
app.json_encoder = CustomJSONEncoder
109-
api_config_object = configparser.SafeConfigParser()
109+
api_config_object = configparser.ConfigParser()
110110
api_config_object.optionxform = str
111111
# TODO: Why does this multiscanner.common instead of just common?
112112
api_config_file = utils.get_config_path(MS_CONFIG, 'api')
@@ -159,7 +159,7 @@ def default(self, obj):
159159
storage_handler = StorageHandler(configfile=storage_conf)
160160
handler = storage_handler.load_required_module('ElasticSearchStorage')
161161

162-
ms_config_object = configparser.SafeConfigParser()
162+
ms_config_object = configparser.ConfigParser()
163163
ms_config_object.optionxform = str
164164
ms_configfile = MS_CONFIG
165165
ms_config_object.read(ms_configfile)
@@ -279,7 +279,7 @@ def modules():
279279
filenames = [os.path.splitext(os.path.basename(f)) for f in files]
280280
module_names = [m[0] for m in filenames if m[1] == '.py']
281281

282-
ms_config = configparser.SafeConfigParser()
282+
ms_config = configparser.ConfigParser()
283283
ms_config.optionxform = str
284284
ms_config.read(MS_CONFIG)
285285
modules = {}

‎multiscanner/distributed/celery_worker.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535
'tz': 'US/Eastern',
3636
}
3737

38-
config_object = configparser.SafeConfigParser()
38+
config_object = configparser.ConfigParser()
3939
config_object.optionxform = str
4040
configfile = utils.get_config_path(MS_CONFIG, 'api')
4141
config_object.read(configfile)
@@ -53,7 +53,7 @@
5353
worker_config = config.get('celery')
5454
db_config = config.get('Database')
5555

56-
storage_config_object = configparser.SafeConfigParser()
56+
storage_config_object = configparser.ConfigParser()
5757
storage_config_object.optionxform = str
5858
storage_configfile = utils.get_config_path(MS_CONFIG, 'storage')
5959
storage_config_object.read(storage_configfile)
@@ -148,7 +148,7 @@ def multiscanner_celery(file_, original_filename, task_id, file_hash, metadata,
148148

149149
# Get the Scan Config that the task was run with and
150150
# add it to the task metadata
151-
scan_config_object = configparser.SafeConfigParser()
151+
scan_config_object = configparser.ConfigParser()
152152
scan_config_object.optionxform = str
153153
scan_config_object.read(config)
154154
full_conf = utils.parse_config(scan_config_object)

‎multiscanner/distributed/distributed_worker.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def multiscanner_process(work_queue, config, batch_size, wait_seconds, delete, e
6767

6868

6969
def _read_conf(file_path):
70-
conf = configparser.SafeConfigParser()
70+
conf = configparser.ConfigParser()
7171
conf.optionxform = str
7272
with codecs.open(file_path, 'r', encoding='utf-8') as fp:
7373
conf.readfp(fp)

‎multiscanner/ms.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -489,7 +489,7 @@ def config_init(filepath, module_list=parseDir(MODULESDIR, recursive=True, exclu
489489
490490
filepath - The config file to create
491491
"""
492-
config = configparser.SafeConfigParser()
492+
config = configparser.ConfigParser()
493493
config.optionxform = str
494494

495495
if filepath:
@@ -583,7 +583,7 @@ def multiscan(Files, recursive=False, configregen=False, configfile=CONFIG, conf
583583

584584
# Read in config
585585
if configfile:
586-
config_object = configparser.SafeConfigParser()
586+
config_object = configparser.ConfigParser()
587587
config_object.optionxform = str
588588
# Regen the config if needed or wanted
589589
if configregen or not os.path.isfile(configfile):
@@ -893,15 +893,15 @@ def _init(args):
893893
else:
894894
print('Checking for missing modules in configuration...')
895895
ModuleList = parseDir(MODULESDIR, recursive=True, exclude=["__init__"])
896-
config = configparser.SafeConfigParser()
896+
config = configparser.ConfigParser()
897897
config.optionxform = str
898898
config.read(args.config)
899899
_write_missing_module_configs(ModuleList, config, filepath=args.config)
900900
else:
901901
config_init(args.config)
902902

903903
# Init storage
904-
config = configparser.SafeConfigParser()
904+
config = configparser.ConfigParser()
905905
config.optionxform = str
906906
config.read(args.config)
907907
config = _get_main_config(config)
@@ -1010,7 +1010,7 @@ def _main():
10101010
results = multiscan(filelist, configfile=args.config)
10111011

10121012
# We need to read in the config for the parseReports call
1013-
config = configparser.SafeConfigParser()
1013+
config = configparser.ConfigParser()
10141014
config.optionxform = str
10151015
config.read(args.config)
10161016
config = _get_main_config(config)

‎multiscanner/storage/basic_elasticsearch_storage.py

+9-9
Original file line numberDiff line numberDiff line change
@@ -121,13 +121,13 @@ def normalize_list(self, array):
121121
array[i] = self.same_type_lists(array[i])
122122
elif not self.check_same_types(array):
123123
for i in range(0, len(array)):
124-
if isinstance(array[i], list):
125-
array[i] = self.normalize_list(array[i])
126-
elif isinstance(array[i], dict):
127-
array[i] = self.same_type_lists(array[i])
128-
else:
129-
array[i] = str(array[i])
130-
if not self.warned_changed:
131-
print("WARNING: We changed some of the data types so that Elasticsearch wouldn't get angry")
132-
self.warned_changed = True
124+
if isinstance(array[i], list):
125+
array[i] = self.normalize_list(array[i])
126+
elif isinstance(array[i], dict):
127+
array[i] = self.same_type_lists(array[i])
128+
else:
129+
array[i] = str(array[i])
130+
if not self.warned_changed:
131+
print("WARNING: We changed some of the data types so that Elasticsearch wouldn't get angry")
132+
self.warned_changed = True
133133
return array

‎multiscanner/storage/file.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ def store(self, results):
4848
writedata = {}
4949
if writedata:
5050
if metadata:
51-
writedata = {'Files': writedata, 'Metadata': metadata}
51+
writedata = {'Files': writedata, 'Metadata': metadata}
5252
if self.config['gzip'] is True:
5353
self.file_handle.write(
5454
json.dumps(writedata, sort_keys=True, separators=(',', ':'),

‎multiscanner/storage/sql_driver.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ def __init__(self, config=None, configfile=CONFIG_FILE, regenconfig=False):
6767
self.db_engine = None
6868

6969
# Configuration parsing
70-
config_parser = configparser.SafeConfigParser()
70+
config_parser = configparser.ConfigParser()
7171
config_parser.optionxform = str
7272

7373
# (re)generate conf file if necessary

‎multiscanner/storage/storage.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ def __init__(self, configfile=MS_CONFIG, config=None, configregen=False):
9696
# Read in config
9797
if configfile:
9898
configfile = utils.get_config_path(MS_CONFIG, 'storage')
99-
config_object = configparser.SafeConfigParser()
99+
config_object = configparser.ConfigParser()
100100
config_object.optionxform = str
101101
# Regen the config if needed or wanted
102102
if configregen or not os.path.isfile(configfile):
@@ -262,7 +262,7 @@ def is_done(self, wait=False):
262262
def config_init(filepath, overwrite=False, storage_classes=None):
263263
if storage_classes is None:
264264
storage_classes = _get_storage_classes()
265-
config_object = configparser.SafeConfigParser()
265+
config_object = configparser.ConfigParser()
266266
config_object.optionxform = str
267267
if overwrite or not os.path.isfile(filepath):
268268
_write_main_config(config_object)

‎multiscanner/tests/module_tests/Metadefender/test_metadefender_module.py

+27-27
Original file line numberDiff line numberDiff line change
@@ -154,9 +154,9 @@ def test_submit_sample_success(self, mock_get):
154154
'''
155155
print('Running test_submit_sample_success')
156156
submit_resp = Metadefender._submit_sample(RANDOM_INPUT_FILES[0], 'scan_url', 'user_agent')
157-
self.assertEquals(submit_resp['status_code'], 200)
158-
self.assertEquals(submit_resp['error'], None)
159-
self.assertEquals(submit_resp['scan_id'], generate_scan_id(RANDOM_INPUT_FILES[0]))
157+
self.assertEqual(submit_resp['status_code'], 200)
158+
self.assertEqual(submit_resp['error'], None)
159+
self.assertEqual(submit_resp['scan_id'], generate_scan_id(RANDOM_INPUT_FILES[0]))
160160

161161
@mock.patch('Metadefender.requests.post', side_effect=mocked_requests_post_sample_failed_w_msg)
162162
def test_submit_sample_fail_unavailable(self, mock_get):
@@ -166,9 +166,9 @@ def test_submit_sample_fail_unavailable(self, mock_get):
166166
'''
167167
print('Running test_submit_sample_fail_unavailable')
168168
submit_resp = Metadefender._submit_sample(RANDOM_INPUT_FILES[1], 'scan_url', 'user_agent')
169-
self.assertEquals(submit_resp['status_code'], 500)
170-
self.assertEquals(submit_resp['error'], MSG_SERVER_UNAVAILABLE)
171-
self.assertEquals(submit_resp['scan_id'], None)
169+
self.assertEqual(submit_resp['status_code'], 500)
170+
self.assertEqual(submit_resp['error'], MSG_SERVER_UNAVAILABLE)
171+
self.assertEqual(submit_resp['scan_id'], None)
172172

173173
@mock.patch('Metadefender.requests.post', side_effect=mocked_requests_post_sample_failed_no_msg)
174174
def test_submit_sample_fail_unavailable_no_msg(self, mock_get):
@@ -178,9 +178,9 @@ def test_submit_sample_fail_unavailable_no_msg(self, mock_get):
178178
'''
179179
print('Running test_submit_sample_fail_unavailable_no_msg')
180180
submit_resp = Metadefender._submit_sample(RANDOM_INPUT_FILES[1], 'scan_url', 'user_agent')
181-
self.assertEquals(submit_resp['status_code'], 500)
182-
self.assertEquals(submit_resp['error'], Metadefender.MD_HTTP_ERR_CODES[500])
183-
self.assertEquals(submit_resp['scan_id'], None)
181+
self.assertEqual(submit_resp['status_code'], 500)
182+
self.assertEqual(submit_resp['error'], Metadefender.MD_HTTP_ERR_CODES[500])
183+
self.assertEqual(submit_resp['scan_id'], None)
184184

185185
# ---------------------------------------------------------------------
186186
# This section tests the logic for parsing Metadefender's responses
@@ -195,26 +195,26 @@ def test_get_results_200_success(self, mock_get):
195195
print('Running test_get_results_200_success')
196196
report_resp = Metadefender._retrieve_scan_results('results_url', SCAN_IDS[0])
197197
is_scan_complete, parsed_resp = Metadefender._parse_scan_result(report_resp)
198-
self.assertEquals(is_scan_complete, True)
199-
self.assertEquals(parsed_resp['overall_status'], Metadefender.STATUS_SUCCESS)
198+
self.assertEqual(is_scan_complete, True)
199+
self.assertEqual(parsed_resp['overall_status'], Metadefender.STATUS_SUCCESS)
200200

201201
engine_results = parsed_resp['engine_results']
202202
for engine_result in engine_results:
203203
engine_name = engine_result['engine_name']
204204
scan_result = engine_result['scan_result']
205205
threat_found = engine_result['threat_found']
206206
if engine_name == 'ClamAV':
207-
self.assertEquals(scan_result, 'Infected/Known')
208-
self.assertEquals(threat_found, 'Heuristics.PDF.ObfuscatedNameObject')
207+
self.assertEqual(scan_result, 'Infected/Known')
208+
self.assertEqual(threat_found, 'Heuristics.PDF.ObfuscatedNameObject')
209209
elif engine_name == 'Ahnlab':
210-
self.assertEquals(scan_result, 'Infected/Known')
211-
self.assertEquals(threat_found, 'Trojan/Win32.Inject.C1515213')
210+
self.assertEqual(scan_result, 'Infected/Known')
211+
self.assertEqual(threat_found, 'Trojan/Win32.Inject.C1515213')
212212
elif engine_name == 'ESET':
213-
self.assertEquals(scan_result, 'No threats Found')
214-
self.assertEquals(threat_found, '')
213+
self.assertEqual(scan_result, 'No threats Found')
214+
self.assertEqual(threat_found, '')
215215
elif engine_name == 'Avira':
216-
self.assertEquals(scan_result, 'No threats Found')
217-
self.assertEquals(threat_found, '')
216+
self.assertEqual(scan_result, 'No threats Found')
217+
self.assertEqual(threat_found, '')
218218
else:
219219
self.fail('Unexpected Engine: %s' % engine_name)
220220

@@ -228,8 +228,8 @@ def test_get_results_200_not_found(self, mock_get):
228228
print('Running test_get_results_200_not_found')
229229
report_resp = Metadefender._retrieve_scan_results('results_url', SCAN_IDS[0])
230230
is_scan_complete, parsed_resp = Metadefender._parse_scan_result(report_resp)
231-
self.assertEquals(is_scan_complete, False)
232-
self.assertEquals(parsed_resp['overall_status'], Metadefender.STATUS_PENDING)
231+
self.assertEqual(is_scan_complete, False)
232+
self.assertEqual(parsed_resp['overall_status'], Metadefender.STATUS_PENDING)
233233

234234
engine_results = parsed_resp['engine_results']
235235
if len(engine_results) != 0:
@@ -244,8 +244,8 @@ def test_get_results_200_succes_in_progress(self, mock_get):
244244
print('Running test_get_results_200_succes_in_progress')
245245
report_resp = Metadefender._retrieve_scan_results('results_url', SCAN_IDS[0])
246246
is_scan_complete, parsed_resp = Metadefender._parse_scan_result(report_resp)
247-
self.assertEquals(is_scan_complete, False)
248-
self.assertEquals(parsed_resp['overall_status'], Metadefender.STATUS_PENDING)
247+
self.assertEqual(is_scan_complete, False)
248+
self.assertEqual(parsed_resp['overall_status'], Metadefender.STATUS_PENDING)
249249
msg = parsed_resp['msg']
250250
if 'percent complete: 10' not in msg:
251251
self.fail('Progress percentage not present')
@@ -265,9 +265,9 @@ def test_scan_complete_success(self, mock_post, mock_get):
265265
print('Running test_scan_complete_success')
266266
resultlist, metadata = Metadefender.scan(RANDOM_INPUT_FILES,
267267
conf=self.create_conf_short_timeout())
268-
self.assertEquals(len(resultlist), len(RANDOM_INPUT_FILES))
268+
self.assertEqual(len(resultlist), len(RANDOM_INPUT_FILES))
269269
for scan_res in resultlist:
270-
self.assertEquals(scan_res[1]['overall_status'], Metadefender.STATUS_SUCCESS)
270+
self.assertEqual(scan_res[1]['overall_status'], Metadefender.STATUS_SUCCESS)
271271

272272
@mock.patch('Metadefender.requests.get', side_effect=mocked_requests_get_sample_200_in_progress)
273273
@mock.patch('Metadefender.requests.post', side_effect=mocked_requests_post_sample_submitted)
@@ -278,9 +278,9 @@ def test_scan_timeout_scan_in_progress(self, mock_post, mock_get):
278278
print('Running test_scan_timeout_scan_in_progress')
279279
resultlist, metadata = Metadefender.scan(RANDOM_INPUT_FILES,
280280
conf=self.create_conf_short_timeout())
281-
self.assertEquals(len(resultlist), len(RANDOM_INPUT_FILES))
281+
self.assertEqual(len(resultlist), len(RANDOM_INPUT_FILES))
282282
for scan_res in resultlist:
283-
self.assertEquals(scan_res[1]['overall_status'], Metadefender.STATUS_TIMEOUT)
283+
self.assertEqual(scan_res[1]['overall_status'], Metadefender.STATUS_TIMEOUT)
284284

285285

286286
if __name__ == "__main__":

‎multiscanner/tests/test_frontend.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
try:
2727
opts = Options()
2828
opts.add_argument('-headless')
29-
driver = webdriver.Firefox(firefox_options=opts)
29+
driver = webdriver.Firefox(options=opts)
3030
except Exception as e:
3131
pytestmark = pytest.mark.skip
3232

‎multiscanner/web/app.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
app = Flask(__name__)
3333

3434
# Finagle Flask to read config from .ini file instead of .py file
35-
web_config_object = configparser.SafeConfigParser()
35+
web_config_object = configparser.ConfigParser()
3636
web_config_object.optionxform = str
3737
web_config_file = utils.get_config_path(MS_CONFIG, 'web')
3838
web_config_object.read(web_config_file)

‎tox.ini

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ deps = -rrequirements-dev.txt
66
commands =
77
multiscanner init
88
pytest
9+
pre-commit run --all-files
910

1011
passenv = CI TRAVIS TRAVIS_*
1112

0 commit comments

Comments
 (0)