Skip to content

Commit eb66ddf

Browse files
mlodicmlodic
authored
Merge pull request #49 from certego/develop
mainly added "run_all_available_analyzers" option + get_analyzer_conf API
2 parents 8d61e37 + 8a14917 commit eb66ddf

File tree

21 files changed

+477
-34
lines changed

21 files changed

+477
-34
lines changed

‎.lgtm.yml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ queries:
1010
- include: py/unused-global-variable
1111
- include: py/hardcoded-credentials
1212
- include: py/import-of-mutable-attribute
13-
- include: py/cyclic-import
1413
- include: py/empty-except
1514
- include: py/unnecessary-lambda
1615
- include: py/print-during-import

‎README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,13 @@ Main features:
4545
* Intezer
4646
* Farsight DNSDB
4747
* Hunter.io - Email Hunting
48+
* ONYPHE
49+
* Censys.io
4850
#### required free api key
4951
* GoogleSafeBrowsing
5052
* AbuseIPDB
5153
* Shodan
52-
* HoneyDB - Twitter Threat Feed Scan
54+
* HoneyDB
5355
* AlienVault OTX
5456
* MaxMind
5557
#### needed access request
@@ -60,6 +62,8 @@ Main features:
6062
* Talos Reputation
6163
* Tor Project
6264
* Robtex
65+
* Threatminer
66+
* Abuse.ch MalwareBazaar
6367

6468
### Documentation
6569
[![Documentation Status](https://readthedocs.org/projects/intelowl/badge/?version=latest)](https://intelowl.readthedocs.io/en/latest/?badge=latest)

‎api_app/models.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ class Job(models.Model):
1919
file_name = models.CharField(max_length=50, blank=True)
2020
file_mimetype = models.CharField(max_length=50, blank=True)
2121
status = models.CharField(max_length=50, blank=False, choices=STATUS, default="pending")
22-
analyzers_requested = postgres_fields.ArrayField(models.CharField(max_length=900), blank=False)
22+
analyzers_requested = postgres_fields.ArrayField(models.CharField(max_length=900), blank=True, default=list)
23+
run_all_available_analyzers = models.BooleanField(blank=False, default=False)
2324
analyzers_to_execute = postgres_fields.ArrayField(models.CharField(max_length=900), blank=True, default=list)
2425
analysis_reports = postgres_fields.JSONField(default=list, null=True, blank=True)
2526
received_request_time = models.DateTimeField(auto_now_add=True)

‎api_app/script_analyzers/file_analyzers/cuckoo_scan.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ def _cuckoo_poll_result(cuckoo_analysis, filename, md5, additional_config_params
117117
logger.info("polling result for {} {}, task_id {}".format(filename, md5, cuckoo_analysis.task_id))
118118

119119
# poll for the result
120-
max_get_tries = additional_config_params.get('max_poll_tries', 50)
120+
max_get_tries = additional_config_params.get('max_poll_tries', 20)
121121
poll_time = 15
122122
get_success = False
123123
for chance in range(max_get_tries):

‎api_app/script_analyzers/observable_analyzers/censys.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import traceback
2+
import logging
3+
import requests
4+
5+
from api_app.exceptions import AnalyzerRunException
6+
from api_app.script_analyzers import general
7+
from intel_owl import secrets
8+
9+
logger = logging.getLogger(__name__)
10+
11+
base_url = 'https://www.censys.io/api/v1'
12+
13+
14+
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
15+
logger.info("started analyzer {} job_id {} observable {}"
16+
"".format(analyzer_name, job_id, observable_name))
17+
report = general.get_basic_report_template(analyzer_name)
18+
try:
19+
api_id_name = additional_config_params.get('api_id_name', '')
20+
api_secret_name = additional_config_params.get('api_secret_name', '')
21+
if not api_id_name:
22+
api_id_name = "CENSYS_API_ID"
23+
api_secret_name = "CENSYS_API_SECRET"
24+
api_id = secrets.get_secret(api_id_name)
25+
api_secret = secrets.get_secret(api_secret_name)
26+
if not (api_id and api_secret):
27+
raise AnalyzerRunException("no api credentials retrieved")
28+
29+
result = _censys_get_report((api_id, api_secret), observable_name, observable_classification,
30+
additional_config_params)
31+
32+
# pprint.pprint(result)
33+
report['report'] = result
34+
except AnalyzerRunException as e:
35+
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
36+
"".format(job_id, analyzer_name, observable_name, e)
37+
logger.error(error_message)
38+
report['errors'].append(error_message)
39+
report['success'] = False
40+
except Exception as e:
41+
traceback.print_exc()
42+
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
43+
"".format(job_id, analyzer_name, observable_name, e)
44+
logger.exception(error_message)
45+
report['errors'].append(str(e))
46+
report['success'] = False
47+
else:
48+
report['success'] = True
49+
50+
general.set_report_and_cleanup(job_id, report)
51+
52+
logger.info("ended analyzer {} job_id {} observable {}"
53+
"".format(analyzer_name, job_id, observable_name))
54+
55+
return report
56+
57+
58+
def _censys_get_report(api_creds, observable_name, observable_classification, additional_config_params):
59+
censys_analysis = additional_config_params.get('censys_analysis', 'search')
60+
if censys_analysis == 'search':
61+
uri = '/view/ipv4/{}'.format(observable_name)
62+
else:
63+
raise AnalyzerRunException("not supported observable type {}. Supported is IP"
64+
"".format(observable_classification))
65+
try:
66+
response = requests.get(base_url + uri, auth=api_creds)
67+
response.raise_for_status()
68+
except requests.RequestException as e:
69+
raise AnalyzerRunException(e)
70+
result = response.json()
71+
return result

‎api_app/script_analyzers/observable_analyzers/mb_get.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import traceback
2+
import requests
3+
4+
from celery.utils.log import get_task_logger
5+
6+
from api_app.exceptions import AnalyzerRunException
7+
from api_app.script_analyzers import general
8+
9+
logger = get_task_logger(__name__)
10+
11+
12+
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
13+
logger.info("started analyzer {} job_id {} observable {}"
14+
"".format(analyzer_name, job_id, observable_name))
15+
report = general.get_basic_report_template(analyzer_name)
16+
try:
17+
if observable_classification=="hash":
18+
filehash = observable_name
19+
else:
20+
raise AnalyzerRunException(f"not supported observable type {observable_classification}. Supported: hash only")
21+
22+
post_data = {
23+
"query": "get_info",
24+
"hash": filehash
25+
}
26+
27+
url = 'https://mb-api.abuse.ch/api/v1/'
28+
response = requests.post(url, data=post_data)
29+
response.raise_for_status()
30+
31+
json_response = response.json()
32+
report['report'] = json_response
33+
34+
except AnalyzerRunException as e:
35+
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
36+
"".format(job_id, analyzer_name, observable_name, e)
37+
logger.error(error_message)
38+
report['errors'].append(error_message)
39+
report['success'] = False
40+
except Exception as e:
41+
traceback.print_exc()
42+
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
43+
"".format(job_id, analyzer_name, observable_name, e)
44+
logger.exception(error_message)
45+
report['errors'].append(str(e))
46+
report['success'] = False
47+
else:
48+
report['success'] = True
49+
50+
general.set_report_and_cleanup(job_id, report)
51+
52+
logger.info("ended analyzer {} job_id {} observable {}"
53+
"".format(analyzer_name, job_id, observable_name))
54+
55+
return report

‎api_app/script_analyzers/observable_analyzers/onyphe.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import traceback
2+
3+
import requests
4+
from celery.utils.log import get_task_logger
5+
6+
from api_app.exceptions import AnalyzerRunException
7+
from api_app.script_analyzers import general
8+
from intel_owl import secrets
9+
10+
logger = get_task_logger(__name__)
11+
12+
base_url = "https://www.onyphe.io/api/v2/summary/"
13+
14+
15+
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
16+
logger.info("started analyzer {} job_id {} observable {}"
17+
"".format(analyzer_name, job_id, observable_name))
18+
report = general.get_basic_report_template(analyzer_name)
19+
try:
20+
api_key_name = additional_config_params.get('api_key_name', 'ONYPHE_KEY')
21+
api_key = secrets.get_secret(api_key_name)
22+
if not api_key:
23+
raise AnalyzerRunException("no ONYPHE's API key retrieved")
24+
25+
result = _onyphe_get_report(api_key, observable_name, observable_classification)
26+
27+
report['report'] = result
28+
29+
except AnalyzerRunException as e:
30+
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
31+
"".format(job_id, analyzer_name, observable_name, e)
32+
logger.error(error_message)
33+
report['errors'].append(error_message)
34+
report['success'] = False
35+
except Exception as e:
36+
traceback.print_exc()
37+
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
38+
"".format(job_id, analyzer_name, observable_name, e)
39+
logger.exception(error_message)
40+
report['errors'].append(str(e))
41+
report['success'] = False
42+
else:
43+
report['success'] = True
44+
45+
general.set_report_and_cleanup(job_id, report)
46+
47+
logger.info("ended analyzer {} job_id {} observable {}"
48+
"".format(analyzer_name, job_id, observable_name))
49+
50+
return report
51+
52+
53+
def _onyphe_get_report(api_key, observable_name, observable_classification):
54+
headers = {
55+
'Authorization': f'apikey {api_key}',
56+
'Content-Type': 'application/json'
57+
}
58+
if observable_classification == 'domain':
59+
uri = f'domain/{observable_name}'
60+
elif observable_classification == 'ip':
61+
uri = f'ip/{observable_name}'
62+
elif observable_classification == 'url':
63+
uri = f'hostname/{observable_name}'
64+
else:
65+
raise AnalyzerRunException(f"not supported observable type {observable_classification}. Supported are: ip, domain and url.")
66+
67+
try:
68+
response = requests.get(base_url + uri, headers=headers)
69+
response.raise_for_status()
70+
except requests.RequestException as e:
71+
raise AnalyzerRunException(e)
72+
result = response.json()
73+
return result

‎api_app/script_analyzers/observable_analyzers/threatminer.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import traceback
2+
import logging
3+
import requests
4+
5+
from api_app.exceptions import AnalyzerRunException
6+
from api_app.script_analyzers import general
7+
8+
logger = logging.getLogger(__name__)
9+
10+
base_url = "https://api.threatminer.org/v2/"
11+
12+
13+
def run(analyzer_name, job_id, observable_name, observable_classification, additional_config_params):
14+
logger.info("started analyzer {} job_id {} observable {}"
15+
"".format(analyzer_name, job_id, observable_name))
16+
report = general.get_basic_report_template(analyzer_name)
17+
try:
18+
rt_value = additional_config_params.get('rt_value', '')
19+
params = {
20+
'q': observable_name
21+
}
22+
if rt_value:
23+
params['rt'] = rt_value
24+
25+
if observable_classification == 'domain':
26+
uri = 'domain.php'
27+
elif observable_classification == 'ip':
28+
uri = 'host.php'
29+
else:
30+
raise AnalyzerRunException("not supported observable type {}. Supported are IP and Domain"
31+
"".format(observable_classification))
32+
33+
try:
34+
response = requests.get(base_url + uri, params=params)
35+
response.raise_for_status()
36+
except requests.RequestException as e:
37+
raise AnalyzerRunException(e)
38+
result = response.json()
39+
40+
# pprint.pprint(result)
41+
report['report'] = result
42+
except AnalyzerRunException as e:
43+
error_message = "job_id:{} analyzer:{} observable_name:{} Analyzer error {}" \
44+
"".format(job_id, analyzer_name, observable_name, e)
45+
logger.error(error_message)
46+
report['errors'].append(error_message)
47+
report['success'] = False
48+
except Exception as e:
49+
traceback.print_exc()
50+
error_message = "job_id:{} analyzer:{} observable_name:{} Unexpected error {}" \
51+
"".format(job_id, analyzer_name, observable_name, e)
52+
logger.exception(error_message)
53+
report['errors'].append(str(e))
54+
report['success'] = False
55+
else:
56+
report['success'] = True
57+
58+
general.set_report_and_cleanup(job_id, report)
59+
60+
logger.info("ended analyzer {} job_id {} observable {}"
61+
"".format(analyzer_name, job_id, observable_name))
62+
63+
return report

‎api_app/utilities.py‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ def file_directory_path(instance, filename):
3232
return 'job_{}_{}'.format(get_now_str(), filename)
3333

3434

35-
def filter_analyzers(serialized_data, analyzers_config, warnings):
35+
def filter_analyzers(serialized_data, analyzers_requested, analyzers_config, warnings, run_all=False):
3636
cleaned_analyzer_list = []
37-
for analyzer in serialized_data['analyzers_requested']:
37+
for analyzer in analyzers_requested:
3838
try:
3939
if analyzer not in analyzers_config:
4040
raise NotRunnableAnalyzer("{} not available in configuration".format(analyzer))
@@ -65,8 +65,12 @@ def filter_analyzers(serialized_data, analyzers_config, warnings):
6565
if serialized_data['disable_external_analyzers'] and analyzers_config[analyzer].get('external_service', ''):
6666
raise NotRunnableAnalyzer("{} won't be run because you filtered external analyzers".format(analyzer))
6767
except NotRunnableAnalyzer as e:
68-
logger.warning(e)
69-
warnings.append(str(e))
68+
if run_all:
69+
# in this case, they are not warnings but excepted and wanted behavior
70+
logger.debug(e)
71+
else:
72+
logger.warning(e)
73+
warnings.append(str(e))
7074
else:
7175
cleaned_analyzer_list.append(analyzer)
7276

0 commit comments

Comments
 (0)