diff --git a/examples/conffile.conf b/examples/conffile.conf new file mode 100644 index 0000000..46052d0 --- /dev/null +++ b/examples/conffile.conf @@ -0,0 +1,54 @@ +[Elasticsearch] +hosts=hostname1.com,hostname2.com +port=9200 +username=ll +password=ll +setup_index=True +use_tls=True +cacert="/path/to/cert" +tls_verify=True +index="elastic-tip" + +[URLhaus] +enabled=True + +[MalwareBazaar] +enabled=True + +[FeodoTracker] +enabled=True + +[SSLBlacklist] +enabled=True + +[EmergingThreats-Blocklist] +enabled=True + +[ESET-MalwareIOC] +enabled=True + +[AbuseIPdb] +enabled=True +apikey= +confidenceminimum=90 + +[Spamhaus-Drop] +enabled=True + +[Spamhaus-ExtendedDrop] +enabled=True + +[Spamhaus-IPv6Drop] +enabled=True + +[Botvrij-filenames] +enabled=True + +[Botvrij-domains] +enabled=True + +[Botvrij-destinations] +enabled=True + +[Botvrij-urls] +enabled=True diff --git a/tip/abuse_bazaar.py b/tip/abuse_bazaar.py index 9d23891..03b7d6f 100644 --- a/tip/abuse_bazaar.py +++ b/tip/abuse_bazaar.py @@ -6,11 +6,12 @@ class URLhaus: - def __init__(self): + def __init__(self, conf=None): self._raw_threat_intel = None self.intel = [] self._retrieved = None self._feed_url = "https://urlhaus.abuse.ch/downloads/csv_recent/" + self.conf = conf def run(self): self._download() @@ -50,11 +51,12 @@ def _parse(self): class MalwareBazaar: - def __init__(self): + def __init__(self, conf=None): self._raw_threat_intel = None self.intel = [] self._retrieved = None self._feed_url = "https://bazaar.abuse.ch/export/csv/recent/" + self.conf = conf def run(self): self._download() @@ -95,11 +97,12 @@ def _parse(self): class FeodoTracker: - def __init__(self): + def __init__(self, conf=None): self._raw_threat_intel = None self.intel = [] self._retrieved = None self._feed_url = "https://feodotracker.abuse.ch/downloads/ipblocklist.csv" + self.conf = conf def run(self): self._download() @@ -141,11 +144,12 @@ def _parse(self): class SSLBlacklist: - def __init__(self): + def __init__(self, conf=None): self._raw_threat_intel = None self.intel = [] self._retrieved = None self._feed_url = "https://sslbl.abuse.ch/blacklist/sslblacklist.csv" + self.conf = conf def run(self): self._download() diff --git a/tip/abuseipdb.py b/tip/abuseipdb.py index d4be87b..5008dd5 100644 --- a/tip/abuseipdb.py +++ b/tip/abuseipdb.py @@ -7,12 +7,13 @@ class AbuseIPDB: - def __init__(self): + def __init__(self, conf=None): self.intel = [] self._retrieved = None self._feed_url = "https://api.abuseipdb.com/api/v2/blacklist" - self.confidenceminimum = '90' - self.key = None + self._conf = conf + self.confidenceminimum = self._conf["AbuseIPdb"].getint("confidenceminimum") + self.key = self._conf["AbuseIPdb"].getint("apikey") self._raw_threat_intel = { "data": [] } diff --git a/tip/botvrij.py b/tip/botvrij.py index 639fe99..eb6771b 100644 --- a/tip/botvrij.py +++ b/tip/botvrij.py @@ -7,12 +7,13 @@ class BotvrijFileNames: - def __init__(self): + def __init__(self, conf=None): self.intel = [] self._retrieved = None self._feed_url = "https://botvrij.eu/data/ioclist.filename.raw" self.key = None self._raw_threat_intel = "" + self.conf = conf def run(self): self._download() @@ -51,12 +52,13 @@ def _parse(self): class BotvrijDomains: - def __init__(self): + def __init__(self, conf=None): self.intel = [] self._retrieved = None self._feed_url = "https://botvrij.eu/data/ioclist.domain.raw" self.key = None self._raw_threat_intel = "" + self.conf = conf def run(self): self._download() @@ -95,12 +97,13 @@ def _parse(self): class BotvrijDstIP: - def __init__(self): + def __init__(self, conf=None): self.intel = [] self._retrieved = None self._feed_url = "https://botvrij.eu/data/ioclist.ip-dst.raw" self.key = None self._raw_threat_intel = "" + self.conf = conf def run(self): self._download() @@ -139,12 +142,13 @@ def _parse(self): class BotvrijUrl: - def __init__(self): + def __init__(self, conf=None): self.intel = [] self._retrieved = None self._feed_url = "https://botvrij.eu/data/ioclist.url.raw" self.key = None self._raw_threat_intel = "" + self.conf = conf def run(self): self._download() diff --git a/tip/elastic_tip.py b/tip/elastic_tip.py index 9b4112c..849ff9a 100644 --- a/tip/elastic_tip.py +++ b/tip/elastic_tip.py @@ -9,112 +9,127 @@ from elasticsearch import Elasticsearch from spamhaus import SpamhausDrop, SpamhausExtendedDrop, SpamhausDropIpv6 from botvrij import BotvrijFileNames, BotvrijDomains, BotvrijDstIP, BotvrijUrl +import configparser +from os import path class ElasticTip: def __init__(self): - self.index = "elastic-tip" - self.eshosts = [] - self.esport = 9200 - self.esuser = None - self.espass = None - self.setup_index = True - self.tls = { - "use": True, - "cacert": None, - "verify": True - } + self.conf = None self._es = None self._total_count = 0 - self.modules = { + self.modules = {} + + def load_conf_file(self, conffile): + """ + Method to load the values from a configuration file provided + :param conffile: Path to the conf file + :return: + """ + # Verify the file exists + if not path.exists(conffile): + raise FileNotFoundError("Provided config file does not exists") + else: + conf = configparser.ConfigParser() + conf.read(conffile) + self.conf = conf + + def _load_modules(self): + modules = { "URLhaus": { "enabled": False, - "class": URLhaus(), + "class": URLhaus(self.conf), "ref": "https://urlhaus.abuse.ch/", "note": None }, "MalwareBazaar": { "enabled": False, - "class": MalwareBazaar(), + "class": MalwareBazaar(self.conf), "ref": "https://bazaar.abuse.ch/", "note": None }, "FeodoTracker": { "enabled": False, - "class": FeodoTracker(), + "class": FeodoTracker(self.conf), "ref": "https://feodotracker.abuse.ch/", "note": None }, "SSLBlacklist": { "enabled": False, - "class": SSLBlacklist(), + "class": SSLBlacklist(self.conf), "ref": "https://sslbl.abuse.ch/", "note": None }, "EmergingThreats-Blocklist": { "enabled": False, - "class": ETFireWallBlockIps(), + "class": ETFireWallBlockIps(self.conf), "ref": "https://rules.emergingthreats.net/", "note": None }, "ESET-MalwareIOC": { "enabled": False, - "class": EsetMalwareIOC(), + "class": EsetMalwareIOC(self.conf), "ref": "https://github.com/eset/malware-ioc", "note": None }, "AbuseIPdb": { "enabled": False, - "class": AbuseIPDB(), + "class": AbuseIPDB(self.conf), "ref": "https://www.abuseipdb.com/", "note": "AbuseIPdb requires an API key to work, this can be set through the 'ABUSE_IP_KEY' environment variable or will be requested upon runtime" }, "Spamhaus-Drop": { "enabled": False, - "class": SpamhausDrop(), + "class": SpamhausDrop(self.conf), "ref": "https://www.spamhaus.org/drop/", "note": None }, "Spamhaus-ExtendedDrop": { "enabled": False, - "class": SpamhausExtendedDrop(), + "class": SpamhausExtendedDrop(self.conf), "ref": "https://www.spamhaus.org/drop/", "note": None }, "Spamhaus-IPv6Drop": { "enabled": False, - "class": SpamhausDropIpv6(), + "class": SpamhausDropIpv6(self.conf), "ref": "https://www.spamhaus.org/drop/", "note": None }, "Botvrij-filenames": { "enabled": False, - "class": BotvrijFileNames(), + "class": BotvrijFileNames(self.conf), "ref": "https://botvrij.eu/data/ioclist.filename.raw", "note": None }, "Botvrij-domains": { "enabled": False, - "class": BotvrijDomains(), + "class": BotvrijDomains(self.conf), "ref": "https://botvrij.eu/data/ioclist.domain.raw", "note": None }, "Botvrij-destinations": { "enabled": False, - "class": BotvrijDstIP(), + "class": BotvrijDstIP(self.conf), "ref": "https://botvrij.eu/data/ioclist.ip-dst.raw", "note": None }, "Botvrij-urls": { "enabled": False, - "class": BotvrijUrl(), + "class": BotvrijUrl(self.conf), "ref": "https://botvrij.eu/data/ioclist.url.raw", "note": None } } + for mod in modules: + modules[mod]["enabled"] = self.conf[mod].getboolean("enabled") + + self.modules = modules + def run(self): + self._load_modules() self._build_es_conn() self.verify_tip() print("Running TIP") @@ -127,7 +142,7 @@ def run(self): except AttributeError: if len(mod.intel) > 0: self._ingest(mod.intel, module, True) - self._es.indices.refresh(index=self.index) + self._es.indices.refresh(index=self.conf["Elasticsearch"].get("index")) print("Ingested a total of {} IOC's".format(self._total_count)) def init_tip(self): @@ -150,14 +165,14 @@ def verify_tip(self): with open("tip/elasticsearch/index_mapping.json", "r") as file: index_mapping = json.loads(file.read()) # Verify the index exists - if self._es.indices.exists(index=self.index): - print("Index {} exists".format(self.index)) + if self._es.indices.exists(index=self.conf["Elasticsearch"].get("index")): + print("Index {} exists".format(self.conf["Elasticsearch"].get("index"))) else: - print("Index {} does not exists, creating...".format(self.index)) - if self.setup_index: + print("Index {} does not exists, creating...".format(self.conf["Elasticsearch"].get("index"))) + if self.conf["Elasticsearch"].getboolean("setup_index"): try: self._es.indices.create( - index=self.index, + index=self.conf["Elasticsearch"].get("index"), body={ "settings": index_settings, "mappings": index_mapping @@ -172,7 +187,7 @@ def verify_tip(self): def _build_es_conn(self): if not self._es: eshosts = [] - for hoststring in self.eshosts: + for hoststring in self.conf["Elasticsearch"].get("hosts"): # Determine host and port host, port = self._parse_hosts(hoststring) @@ -181,23 +196,28 @@ def _build_es_conn(self): 'host': host, 'port': port } - if not self.tls["use"]: + if not self.conf["Elasticsearch"].getboolean("use_tls"): host_block["use_ssl"] = False else: host_block["use_ssl"] = True - if self.tls["cacert"]: - host_block["ca_certs"] = self.tls["cacert"] + if "cacert" in self.conf["Elasticsearch"]: + host_block["ca_certs"] = self.conf["Elasticsearch"].get("cacert") - if not self.tls["verify"]: + if not self.conf["Elasticsearch"].getboolean("tls_verify"): host_block["verify_certs"] = False host_block["ssl_show_warn"] = False eshosts.append(host_block) - self.eshosts = eshosts - if self.esuser: - self._es = Elasticsearch(hosts=self.eshosts, http_auth=(self.esuser, self.espass)) + if "username" in self.conf["Elasticsearch"]: + self._es = Elasticsearch( + hosts=eshosts, + http_auth=( + self.conf["Elasticsearch"].get("username"), + self.conf["Elasticsearch"].get("password") + ) + ) else: - self._es = Elasticsearch(hosts=self.eshosts) + self._es = Elasticsearch(hosts=eshosts) print("Connection: {}".format(self._es)) def _parse_hosts(self, hoststring): @@ -211,7 +231,7 @@ def _parse_hosts(self, hoststring): port = int(float(arr[1])) else: host = hoststring - port = self.esport + port = self.conf["Elasticsearch"].getint("port") return host, port @@ -219,11 +239,11 @@ def _ingest(self, iocs, mod="", intel=False): """Ingest IOC's into Elasticsearch""" tens_of_thousands = "(^[1-9]*0{4,}$|^[0-9]{2,}0{3,}$)" - print("Ingesting {} iocs from {} into {}".format(len(iocs), mod, self.eshosts)) + print("Ingesting {} iocs from {}".format(len(iocs), mod)) self._total_count += len(iocs) bulk_body = "" for ioc in iocs: - bulk_body += "{ \"update\" : { \"_index\" : \"%s\", \"_id\" : \"%s\" } }\n" % (self.index, ioc.id) + bulk_body += "{ \"update\" : { \"_index\" : \"%s\", \"_id\" : \"%s\" } }\n" % (self.conf["Elasticsearch"].get("index"), ioc.id) if intel: ioc.intel["@timestamp"] = datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), bulk_body += '{ "doc_as_upsert": true, "doc": %s }\n' % json.dumps(ioc.intel) diff --git a/tip/elastic_tip_cli.py b/tip/elastic_tip_cli.py index c6d1078..d087214 100644 --- a/tip/elastic_tip_cli.py +++ b/tip/elastic_tip_cli.py @@ -1,6 +1,8 @@ from sys import argv import getopt +from os import path from elastic_tip import ElasticTip +import configparser class CLI: @@ -38,14 +40,49 @@ def _run_cli(self): exit() try: - opts, args = getopt.getopt(argv[2:], "hm:e:Tu:p:P:i:c:", - ["help", "modules=", "modules-list", "es-hosts=", "es-port=", "tls", "user=", "passwd=", "index=", "ca-cert=", "no-verify"]) + opts, args = getopt.getopt(argv[2:], "hm:e:Tu:p:P:i:c:C:", + ["help", "modules=", "modules-list", "es-hosts=", "es-port=", "tls", "user=", "passwd=", "index=", "ca-cert=", "no-verify", "config-file="]) except getopt.GetoptError as err: print(err) exit(1) else: self._tip = ElasticTip() + # If a conf file is provided load it first + for opt, arg in opts: + if opt in ["-C", "--config-file"]: + self._tip.load_conf_file(arg) + + # Either create a new conf or use the one from the file + if not self._tip.conf: + config = configparser.ConfigParser() + config["Elasticsearch"] = { + "tls_verify": "True", + "use_tls": "True", + "setup_index": "True" + } + config["URLhaus"] = {"enabled": "False"} + config["MalwareBazaar"] = {"enabled": "False"} + config["FeodoTracker"] = {"enabled": "False"} + config["SSLBlacklist"] = {"enabled": "False"} + config["EmergingThreats-Blocklist"] = {"enabled": "False"} + config["ESET-MalwareIOC"] = {"enabled": "False"} + config["AbuseIPdb"] = { + "enabled": "False", + "apikey": "", + "confidenceminimum": 90 + } + config["Spamhaus-Drop"] = {"enabled": "False"} + config["Spamhaus-ExtendedDrop"] = {"enabled": "False"} + config["Spamhaus-IPv6Drop"] = {"enabled": "False"} + config["Botvrij-filenames"] = {"enabled": "False"} + config["Botvrij-domains"] = {"enabled": "False"} + config["Botvrij-destinations"] = {"enabled": "False"} + config["Botvrij-urls"] = {"enabled": "False"} + else: + # Load the existing one, provided commandline args will overwrite the file + config = self._tip.conf + for opt, arg in opts: if opt in ["-h", "--help"]: self._run_help() @@ -63,40 +100,40 @@ def _run_cli(self): exit() print(self._cli_footer) elif opt in ["-m", "--modules"]: + # Build a config if "*" in arg: - for mod in self._tip.modules: - self._tip.modules[mod]["enabled"] = True + for section in config.sections(): + if "enabled" in section: + config[section]["enabled"] = "True" else: for mod in arg.split(","): try: # Enable the module - self._tip.modules["{}".format(mod)]["enabled"] = True + config[mod]["enabled"] = "True" except KeyError: print("Module {} does not exist".format(mod)) elif opt in ["-e", "--es-hosts"]: - hosts = arg.split(",") - for host in hosts: - if "://" in host: - parsedhost = host.split("://")[1] - else: - parsedhost = host - self._tip.eshosts.append(parsedhost) + config["Elasticsearch"]["hosts"] = arg elif opt in ["-P", "--es-port"]: - self._tip.esport = int(float(arg)) + config["Elasticsearch"]["port"] = arg elif opt in ["-u", "--user"]: - self._tip.esuser = arg + config["Elasticsearch"]["username"] = arg elif opt in ["-p", "--passwd"]: - self._tip.espass = arg + config["Elasticsearch"]["password"] = arg elif opt in ["-i", "--index"]: - self._tip.index = arg + config["Elasticsearch"]["index"] = arg elif opt in ["-T", "--tls"]: - self._tip.tls["use"] = False + config["Elasticsearch"]["use_tls"] = arg elif opt in ["-c", "--ca-cert"]: - self._tip.tls["cacert"] = arg + # make sure the file exists + if not path.exists(arg): + raise FileNotFoundError("The provided cacert file cannot be found") + else: + config["Elasticsearch"]["cacert"] = arg elif opt in ["--no-verify"]: - self._tip.tls["verify"] = False + config["Elasticsearch"]["tls_verify"] = arg elif opt in ["--no-setup"]: - self._tip.setup_index = False + config["Elasticsearch"]["setup_index"] = arg self._tip.run() diff --git a/tip/emergingthreats.py b/tip/emergingthreats.py index c261226..1bc81e7 100644 --- a/tip/emergingthreats.py +++ b/tip/emergingthreats.py @@ -5,10 +5,11 @@ class ETFireWallBlockIps: - def __init__(self): + def __init__(self, conf=None): self.intel = [] self._retrieved = None self._feed_url = "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt" + self.conf = conf def run(self): self._download() diff --git a/tip/eset.py b/tip/eset.py index 5913c03..1caba9a 100644 --- a/tip/eset.py +++ b/tip/eset.py @@ -7,10 +7,11 @@ class EsetMalwareIOC: - def __init__(self): + def __init__(self, conf=None): self.intel = [] self._retrieved = None self._feed_url = "https://github.com/eset/malware-ioc.git" + self.conf = conf def run(self): self._download() diff --git a/tip/spamhaus.py b/tip/spamhaus.py index 5d43c87..94508af 100644 --- a/tip/spamhaus.py +++ b/tip/spamhaus.py @@ -5,11 +5,12 @@ class SpamhausDrop: - def __init__(self): + def __init__(self, conf=None): self._raw_threat_intel = None self.intel = [] self._retrieved = None self._feed_url = "https://www.spamhaus.org/drop/drop.txt" + self.conf = conf def run(self): self._download() @@ -50,11 +51,12 @@ def _parse(self): class SpamhausExtendedDrop: - def __init__(self): + def __init__(self, conf=None): self._raw_threat_intel = None self.intel = [] self._retrieved = None self._feed_url = "https://www.spamhaus.org/drop/edrop.txt" + self.conf = conf def run(self): self._download() @@ -95,11 +97,12 @@ def _parse(self): class SpamhausDropIpv6: - def __init__(self): + def __init__(self, conf=None): self._raw_threat_intel = None self.intel = [] self._retrieved = None self._feed_url = "https://www.spamhaus.org/drop/dropv6.txt" + self.conf = conf def run(self): self._download()