Merge pull request #508 from intelowlproject/develop

v2.4.2 release

Author: eshaan7

.github/CHANGELOG.md

@@ -1,6 +1,13 @@
 # Changelog
 
 [**Upgrade Guide**](https://intelowl.readthedocs.io/en/latest/Installation.html#update-to-the-most-recent-version)
+
+## [v2.4.2](https://github.com/intelowlproject/IntelOwl/releases/tag/v2.4.2)
+- darksearch.io search API analyzer
+- improved abuseipdb analyzer to show matched categories in a human readable form too
+- improved HoneyDB analyzer 
+- as always: fixes, tweaks and dependencies upgrades.
+
 ## [v2.4.1](https://github.com/intelowlproject/IntelOwl/releases/tag/v2.4.1)
 A lot of different fixes, tweaks and dependencies upgrades. Also the documentation was updated
 

.github/workflows/pull_request_automation.yml

@@ -2,7 +2,7 @@ name: Build & Tests
 
 on:
   pull_request:
-    branches: [ master, develop ]
+    branches: [ master, develop, develop-2 ]
 
 jobs:
   build:

api_app/script_analyzers/observable_analyzers/abuseipdb.py

@@ -32,4 +32,45 @@ def run(self):
         response = requests.get(self.url, params=params, headers=headers)
         response.raise_for_status()
 
-        return response.json()
+        result = response.json()
+        reports = result.get("data", {}).get("reports", {})
+        mapping = self._get_mapping()
+        for report in reports:
+            report["categories_human_readable"] = []
+            for category in report.get("categories", []):
+                if category in mapping:
+                    category_human_readable = mapping[category]
+                else:
+                    category_human_readable = "unknown category"
+                report["categories_human_readable"].append(category_human_readable)
+
+        return result
+
+    @staticmethod
+    def _get_mapping():
+        mapping = {
+            1: "DNS Compromise",
+            2: "DNS Poisoning",
+            3: "Fraud Orders",
+            4: "DDOS Attack",
+            5: "FTP Brute-Force",
+            6: "Ping of Death",
+            7: "Phishing",
+            8: "Fraud VOIP",
+            9: "Open Proxy",
+            10: "Web Spam",
+            11: "Email Spam",
+            12: "Blog Spam",
+            13: "VPN IP",
+            14: "Port Scan",
+            15: "Hacking",
+            16: "SQL Injection",
+            17: "Spoofing",
+            18: "Brute Force",
+            19: "Bad Web Bot",
+            20: "Exploited Host",
+            21: "Web App Attack",
+            22: "SSH",
+            23: "IoT Targeted",
+        }
+        return mapping

api_app/script_analyzers/observable_analyzers/darksearch.py

@@ -0,0 +1,28 @@
+# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
+# See the file 'LICENSE' for copying permission.
+
+from api_app.script_analyzers.classes import ObservableAnalyzer
+
+
+class DarkSearchQuery(ObservableAnalyzer):
+    name: str = "DarkSearchQuery"
+
+    def set_config(self, config_params):
+        self.num_pages = int(config_params.get("pages", 5))
+        self.proxies = config_params.get("proxies", None)
+
+    def run(self):
+        from darksearch import Client
+
+        c = Client(proxies=self.proxies)
+        responses = c.search(self.observable_name, pages=self.num_pages)
+        report = {
+            "total": responses[0]["total"],
+            "total_pages": responses[0]["last_page"],
+            "requested_pages": self.num_pages,
+            "data": [],
+        }
+        for resp in responses:
+            report["data"].extend(resp["data"])
+
+        return report

api_app/script_analyzers/observable_analyzers/honeydb.py

@@ -2,43 +2,73 @@
 # See the file 'LICENSE' for copying permission.
 
 import requests
+import logging
 
-from api_app.exceptions import AnalyzerRunException
+from api_app.exceptions import AnalyzerConfigurationException
 from api_app.script_analyzers import classes
 from intel_owl import secrets
 
+logger = logging.getLogger(__name__)
+
 
 class HoneyDB(classes.ObservableAnalyzer):
     base_url = "https://honeydb.io/api"
 
     def set_config(self, additional_config_params):
         api_key_name = additional_config_params.get("api_key_name", "HONEYDB_API_KEY")
         api_id_name = additional_config_params.get("api_id_name", "HONEYDB_API_ID")
-        self.analysis_type = additional_config_params.get(
-            "honeydb_analysis", "ip_query"
-        )
+        self.analysis_type = additional_config_params.get("honeydb_analysis", "all")
+        self.endpoints = [
+            "scan_twitter",
+            "ip_query",
+            "ip_history",
+            "internet_scanner",
+            "ip_info",
+        ]
+        if self.analysis_type not in self.endpoints and self.analysis_type != "all":
+            raise AnalyzerConfigurationException(
+                f"analysis_type is not valid: {self.analysis_type}"
+            )
         self.__api_key = secrets.get_secret(api_key_name)
         self.__api_id = secrets.get_secret(api_id_name)
-
-    def run(self):
         if not self.__api_key:
-            raise AnalyzerRunException("No HoneyDB API Key retrieved")
+            raise AnalyzerConfigurationException("No HoneyDB API Key retrieved")
         if not self.__api_id:
-            raise AnalyzerRunException("No HoneyDB API ID retrieved")
-        headers = {"X-HoneyDb-ApiKey": self.__api_key, "X-HoneyDb-ApiId": self.__api_id}
+            raise AnalyzerConfigurationException("No HoneyDB API ID retrieved")
+        self.headers = {
+            "X-HoneyDb-ApiKey": self.__api_key,
+            "X-HoneyDb-ApiId": self.__api_id,
+        }
+        self.result = {}
 
-        if self.analysis_type == "scan_twitter":
-            url = f"{self.base_url}/twitter-threat-feed/{self.observable_name}"
-        elif self.analysis_type == "ip_query":
-            url = f"{self.base_url}/netinfo/lookup/{self.observable_name}"
+    def run(self):
+        if self.analysis_type == "all":
+            for endpoint in self.endpoints:
+                self._request_analysis(endpoint)
         else:
-            raise AnalyzerRunException(
-                """invalid analyzer name specified.
-                 Supported: HONEYDB_Scan_Twitter, HONEYDB_Get"""
-            )
+            self._request_analysis(self.analysis_type)
 
-        response = requests.get(url, headers=headers)
-        response.raise_for_status()
+        return self.result
 
-        result = response.json()
-        return result
+    def _request_analysis(self, endpoint):
+        if endpoint == "scan_twitter":
+            url = f"{self.base_url}/twitter-threat-feed/{self.observable_name}"
+        elif endpoint == "ip_query":
+            url = f"{self.base_url}/netinfo/lookup/{self.observable_name}"
+        elif endpoint == "ip_history":
+            url = f"{self.base_url}/ip-history/{self.observable_name}"
+        elif endpoint == "internet_scanner":
+            url = f"{self.base_url}/internet-scanner/info/{self.observable_name}"
+        elif endpoint == "ip_info":
+            url = f"{self.base_url}/ipinfo/{self.observable_name}"
+        else:
+            logger.error(f"endpoint {endpoint} not supported")
+            return
+        try:
+            response = requests.get(url, headers=self.headers)
+            response.raise_for_status()
+        except Exception as e:
+            logger.exception(e)
+            self.result[endpoint] = {"error": e}
+        else:
+            self.result[endpoint] = response.json()

configuration/analyzer_config.json

@@ -158,6 +158,19 @@
     "python_module": "cymru.Cymru",
     "soft_time_limit": 50
   },
+  "Darksearch_Query": {
+    "type": "observable",
+    "observable_supported": ["ip", "url", "domain", "hash", "generic"],
+    "external_service": true,
+    "description": "Query against https://darksearch.io/api/search API",
+    "python_module": "darksearch.DarkSearchQuery",
+    "soft_time_limit": 50,
+    "queue": "long",
+    "additional_config_params": {
+      "pages": 10,
+      "proxies": {}
+    }
+  },
   "DNSDB": {
     "type": "observable",
     "observable_supported": [
@@ -365,7 +378,7 @@
     },
     "soft_time_limit": 30
   },
-  "HoneyDB_Get": {
+  "HoneyDB": {
     "type": "observable",
     "observable_supported": [
       "ip"
@@ -377,23 +390,7 @@
     "additional_config_params": {
       "api_key_name": "HONEYDB_API_KEY",
       "api_id_name": "HONEYDB_API_ID",
-      "honeydb_analysis": "ip_query"
-    },
-    "soft_time_limit": 200
-  },
-  "HoneyDB_Scan_Twitter": {
-    "type": "observable",
-    "observable_supported": [
-      "ip"
-    ],
-    "external_service": true,
-    "description": "scan an IP against HoneyDB.io’s Twitter Threat Feed",
-    "requires_configuration": true,
-    "python_module": "honeydb.HoneyDB",
-    "additional_config_params": {
-      "api_key_name": "HONEYDB_API_KEY",
-      "api_id_name": "HONEYDB_API_ID",
-      "honeydb_analysis": "scan_twitter"
+      "honeydb_analysis": "all"
     },
     "soft_time_limit": 200
   },

docker/.env

@@ -1,3 +1,3 @@
 ### DO NOT CHANGE THIS VALUE !!
 ### It should be updated only when you pull latest changes off from the 'master' branch of IntelOwl.
-INTELOWL_TAG_VERSION=v2.4.1
+INTELOWL_TAG_VERSION=v2.4.2

docker/Dockerfile_nginx

@@ -1,5 +1,5 @@
 # Stage 1: Get build artifacts from intelowl-ng
-FROM intelowlproject/intelowl_ng:v2.1.0 AS angular-prod-build
+FROM intelowlproject/intelowl_ng:v2.1.1 AS angular-prod-build
 
 # Stage 2: Inject the build artifacts into nginx container
 FROM library/nginx:1.19-alpine

docs/source/Advanced-Usage.md

@@ -149,7 +149,7 @@ List of some of the analyzers with optional configuration:
   * `search_type` (default `WiFi Network`). Supported are: `WiFi Network`, `CDMA Network`, `Bluetooth Network`, `GSM/LTE/WCDMA Network`
   * Above mentioned `search_type` is just different routes mentioned in [docs](https://api.wigle.net/swagger#/v3_ALPHA). Also, the string to be passed in input field of generic analyzers have a format. Different variables are separated by semicolons(`;`) and the field-name and value are separated by equals sign(`=`). Example string for search_type `CDMA Network` is `sid=12345;nid=12345;bsid=12345`
 * `CRXcavator`:
-  * Every Chrome-Extension has a unique alpha=numerc identifier. That's the only Input necessary. Eg: `Norton Safe Search Enhanced`'s identifier is `eoigllimhcllmhedfbmahegmoakcdakd`.
+  * Every Chrome-Extension has a unique alpha=numeric identifier. That's the only Input necessary. Eg: `Norton Safe Search Enhanced`'s identifier is `eoigllimhcllmhedfbmahegmoakcdakd`.
 * `SSAPINet`:
   *  `use_proxy` (default `false`) and `proxy` (default `""`) - use these options to pass your request through a proxy server.
   *  `output` (default `"image"`) (available options `"image"`, `"json"`) - this specifies whether the result would be a raw image or json (containing link to the image stored on their server).
@@ -162,6 +162,8 @@ List of some of the analyzers with optional configuration:
      Refer to the [docs](https://screenshotapi.net/documentation) for a reference to what other parameters are and their default values.
 * `FireHol_IPList`:
   * `list_names`: Add [FireHol's IPList](https://iplists.firehol.org/) names as comma separated values in `list_names` array.
+* `Honey_DB`:
+  * `honeydb_analysis`(default `all`): choose which endpoint to query from the HoneyDB service (options are `scan_twitter`, `ip_query`, `ip_history`, `internet_scanner`, `ip_info`)
 
 There are two ways to do this:
 

docs/source/Usage.md

@@ -152,11 +152,12 @@ The following is the list of the available analyzers you can run out-of-the-box:
 * `FireHol_IPList`: check if an IP is in [FireHol's IPList](https://iplists.firehol.org/)
 * `ThreatFox`: search for an IOC in [ThreatFox](https://threatfox.abuse.ch/api/)'s database
 
-#### Generic analyzers (email, phone number, ...)
+#### Generic analyzers (email, phone number, etc.; anything really)
 Some Analyzers require details other than just IP, URL, Domain etc... We classified them as Generic Analyzers. Since the type of field is not known, there is a format for strings to be followed.
 * `EmailRep`: search an email address on emailrep.io
 * `WiGLE`: Maps and database of 802.11 wireless networks, with statistics, submitted by wardrivers, netstumblers, and net huggers.
 * `CRXcavator`: scans a chrome extension against crxcavator.io
+* `Darksearch_Query`: Search a keyword against darksearch.io's search API. It's possible to make complex queries using boolean logic. For example, `OSINT AND CTI OR intelowl NOT hack` is a valid observable name.
 
 #### [Additional analyzers](https://intelowl.readthedocs.io/en/develop/Advanced-Usage.html#optional-analyzers) that can be enabled per your wish.
 
@@ -169,7 +170,7 @@ The following are all the keys that you can change without touching the source c
 * `external_service`: if set, in the case you specify via the API to exclude external services, the specific analyzer won't be executed
 * `supported_filetypes`: can be populated as a list. If set, if you ask to analyze a file with a different mimetype from the ones you specified, it won't be executed
 * `not_supported_filetypes`: can be populated as a list. If set, if you ask to analyze a file with a mimetype from the ones you specified, it won't be executed
-* `observable_supported`: can be populated as a list. If set, if you ask to analyze an observable that is not in this list, it won't be executed. Valid values are: `ip`, `domain`, `url`, `hash`
+* `observable_supported`: can be populated as a list. If set, if you ask to analyze an observable that is not in this list, it won't be executed. Valid values are: `ip`, `domain`, `url`, `hash`, `generic`.
 * `soft_time_limit`: this is the maximum time (in seconds) of execution for an analyzer. Once reached, the task will be killed (or managed in the code by a custom Exception). Default 300s
 * `queue`: this takes effects only when [multi-queue](https://intelowl.readthedocs.io/en/develop/Advanced-Usage.html#multi-queue) is enabled. Choose which celery worker would execute the task: `local` (ideal for tasks that leverage local applications like Yara), `long` (ideal for long tasks) or `default` (ideal for simple webAPI-based analyzers).