output-json/output-json-flow: Geoip enrichment [v2]

doc/userguide/output/eve/eve-json-output.rst

@@ -58,6 +58,9 @@ Output types::
 
       filetype: regular #regular|syslog|unix_dgram|unix_stream|redis
       filename: eve.json
+      # Enable GeoIP on source ip and destination ip, default is disabled
+      # Make sure that you have set geoip-database path
+      #geoip-enrichment: yes
       # Enable for multi-threaded eve.json output; output files are amended
       # with an identifier, e.g., eve.9.json. Default: off
       #threaded: off

doc/userguide/partials/eve-log.yaml

@@ -4,6 +4,9 @@ outputs:
       enabled: yes
       filetype: regular #regular|syslog|unix_dgram|unix_stream|redis
       filename: eve.json
+      # Enable GeoIP on source ip and destination ip
+      # Disable geoip enrichment by commenting geoip-enrichment
+      #geoip-enrichment: yes
       # Enable for multi-threaded eve.json output; output files are amended with
       # an identifier, e.g., eve.9.json
       #threaded: false

etc/schema.json

@@ -6,6 +6,90 @@
         "timestamp"
     ],
     "properties": {
+        "geoip_src": {
+            "type": "object",
+            "properties": {
+                "ip":{
+                    "type": "string"
+                },
+                "geo": {
+                    "type": "object",
+                    "properties": {
+                        "continent_code": {
+                            "type": "string"
+                        },
+                        "country_iso_code": {
+                            "type": "string"
+                        },
+                        "city_name": {
+                            "type": "string"
+                        },
+                        "country_name": {
+                            "type": "string"
+                        },
+                        "continent_name": {
+                            "type": "string"
+                        },
+                        "timezone": {
+                            "type": "string"
+                        },
+                        "location": {
+                            "type": "object",
+                            "properties": {
+                                "latitude": {
+                                    "type": "number"
+                                },
+                                "longitude": {
+                                    "type": "number"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
+        "geoip_dst": {
+            "type": "object",
+            "properties": {
+                "ip":{
+                    "type": "string"
+                },
+                "geo": {
+                    "type": "object",
+                    "properties": {
+                        "continent_code": {
+                            "type": "string"
+                        },
+                        "country_iso_code": {
+                            "type": "string"
+                        },
+                        "city_name": {
+                            "type": "string"
+                        },
+                        "country_name": {
+                            "type": "string"
+                        },
+                        "continent_name": {
+                            "type": "string"
+                        },
+                        "timezone": {
+                            "type": "string"
+                        },
+                        "location": {
+                            "type": "object",
+                            "properties": {
+                                "latitude": {
+                                    "type": "number"
+                                },
+                                "longitude": {
+                                    "type": "number"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
         "alert": {
             "type": "object",
             "additionalProperties": false,
@@ -3499,7 +3583,7 @@
                                 "$comment": "keyword: app-layer-event:mdns.name_too_long (https://redmine.openinfosecfoundation.org/issues/7784)"
                             },
                             "rrtype": {
-                                "type": "string", 
+                                "type": "string",
                                 "description": "Type of resource being requested"
                             }
                         }

src/output-json-flow.c

@@ -53,7 +53,7 @@
 #include "flow-storage.h"
 #include "util-exception-policy.h"
 
-static SCJsonBuilder *CreateEveHeaderFromFlow(const Flow *f)
+static SCJsonBuilder *CreateEveHeaderFromFlow(const Flow *f, const OutputJsonCommonSettings *cfg)
 {
     char timebuf[64];
     char srcip[46] = {0}, dstip[46] = {0};
@@ -93,6 +93,13 @@ static SCJsonBuilder *CreateEveHeaderFromFlow(const Flow *f)
     /* time */
     SCJbSetString(jb, "timestamp", timebuf);
 
+#ifdef HAVE_GEOIP
+    if (cfg != NULL && cfg->geoip_enabled) {
+        SCGeoIPGet(jb, srcip, "geoip_src");
+        SCGeoIPGet(jb, dstip, "geoip_dst");
+    }
+#endif /* HAVE_GEOIP */
+
     CreateEveFlowId(jb, (const Flow *)f);
 
 #if 0 // TODO
@@ -432,7 +439,7 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f)
     /* reset */
     MemBufferReset(thread->buffer);
 
-    SCJsonBuilder *jb = CreateEveHeaderFromFlow(f);
+    SCJsonBuilder *jb = CreateEveHeaderFromFlow(f, &thread->ctx->cfg);
     if (unlikely(jb == NULL)) {
         SCReturnInt(TM_ECODE_OK);
     }

src/output-json.c

@@ -68,6 +68,162 @@
 
 #define MAX_JSON_SIZE 2048
 
+#ifdef HAVE_GEOIP
+#include <maxminddb.h>
+
+static MMDB_s mmdb;
+static int mmdb_status = MMDB_FILE_OPEN_ERROR;
+
+#define GeoIPSetString(js, entry_data, key)                                                        \
+    {                                                                                              \
+        if (entry_data.has_data && entry_data.utf8_string != NULL) {                               \
+            SCJbSetStringFromBytes(                                                                \
+                    js, key, (const uint8_t *)entry_data.utf8_string, entry_data.data_size);       \
+        }                                                                                          \
+    }
+
+static bool MMDB_fetch_and_set_json_string(
+        SCJsonBuilder *js, MMDB_lookup_result_s result, const char *key, ...);
+
+static bool MMDB_fetch_and_set_json_double(
+        SCJsonBuilder *js, MMDB_lookup_result_s result, const char *key, ...);
+
+void SCGeoIPGet(SCJsonBuilder *js, const char *ip_address, const char *key);
+
+static bool GeoEnrichmentInit(SCConfNode *conf);
+
+static bool MMDB_fetch_and_set_json_string(
+        SCJsonBuilder *js, MMDB_lookup_result_s result, const char *key, ...)
+{
+    MMDB_entry_data_s entry_data;
+    bool success = false;
+    va_list args;
+    va_start(args, key);
+    if (MMDB_vget_value(&result.entry, &entry_data, args) == MMDB_SUCCESS) {
+        GeoIPSetString(js, entry_data, key);
+        success = true;
+    }
+    va_end(args);
+
+    return success;
+}
+
+static bool MMDB_fetch_and_set_json_double(
+        SCJsonBuilder *js, MMDB_lookup_result_s result, const char *key, ...)
+{
+    MMDB_entry_data_s entry_data;
+    bool success = false;
+    va_list args;
+    va_start(args, key);
+    if (MMDB_vget_value(&result.entry, &entry_data, args) == MMDB_SUCCESS) {
+        SCJbSetFloat(js, key, entry_data.double_value);
+        success = true;
+    }
+    va_end(args);
+
+    return success;
+}
+
+void SCGeoIPGet(SCJsonBuilder *js, const char *ip_address, const char *key)
+{
+    int gai_error, mmdb_error;
+    SCJsonBuilderMark mark_geo = { 0, 0, 0 };
+    SCJsonBuilderMark mark_location = { 0, 0, 0 };
+    bool non_empty_location = false;
+    bool non_empty_geo = false;
+
+    if (mmdb_status != MMDB_SUCCESS) {
+        return;
+    }
+    MMDB_lookup_result_s result = MMDB_lookup_string(&mmdb, ip_address, &gai_error, &mmdb_error);
+    if (MMDB_SUCCESS != gai_error) {
+        return;
+    }
+
+    if (!result.found_entry) {
+        return;
+    }
+
+    SCJbOpenObject(js, key);
+    SCJbSetString(js, "ip", ip_address);
+
+    /* Create geo object */
+    SCJbGetMark(js, &mark_geo);
+    SCJbOpenObject(js, "geo");
+
+    non_empty_geo |=
+            MMDB_fetch_and_set_json_string(js, result, "continent_code", "continent", "code", NULL);
+    non_empty_geo |= MMDB_fetch_and_set_json_string(
+            js, result, "country_iso_code", "country", "iso_code", NULL);
+    non_empty_geo |=
+            MMDB_fetch_and_set_json_string(js, result, "city_name", "city", "names", "en", NULL);
+    non_empty_geo |= MMDB_fetch_and_set_json_string(
+            js, result, "country_name", "country", "names", "en", NULL);
+    non_empty_geo |= MMDB_fetch_and_set_json_string(
+            js, result, "continent_name", "continent", "names", "en", NULL);
+    non_empty_geo |=
+            MMDB_fetch_and_set_json_string(js, result, "timezone", "location", "time_zone", NULL);
+
+    /* Create location object */
+    SCJbGetMark(js, &mark_location);
+    SCJbOpenObject(js, "location");
+    non_empty_location |=
+            MMDB_fetch_and_set_json_double(js, result, "lat", "location", "latitude", NULL);
+    non_empty_location |=
+            MMDB_fetch_and_set_json_double(js, result, "lon", "location", "longitude", NULL);
+
+    if (!non_empty_location) {
+        SCJbRestoreMark(js, &mark_location);
+    } else {
+        SCJbClose(js); /* close location */
+    }
+
+    non_empty_geo |= non_empty_location;
+    if (!non_empty_geo) {
+        SCJbRestoreMark(js, &mark_geo);
+    } else {
+        SCJbClose(js); /* close geo */
+    }
+
+    SCJbClose(js); /* close key */
+}
+
+static bool GeoEnrichmentInit(SCConfNode *conf)
+{
+    const SCConfNode *geoip_enrichment = SCConfNodeLookupChild(conf, "geoip-enrichment");
+    /* Only enable if explicitly set to true/yes */
+    if (geoip_enrichment == NULL || geoip_enrichment->val == NULL ||
+            !SCConfValIsTrue(geoip_enrichment->val)) {
+        SCLogConfig("GeoIP enrichment is disabled.");
+        return false;
+    }
+
+    const char *geoip_db_s = NULL;
+
+    SCLogConfig("GeoIP enrichment is enabled.");
+    (void)SCConfGet("geoip-database", &geoip_db_s);
+    if (geoip_db_s == NULL) {
+        SCLogWarning("geoip-database should be set for geoip-enrichment functionality");
+        return false;
+    } else if (mmdb_status == MMDB_SUCCESS) {
+        /* mmdb already opened by another eve-log output */
+        return true;
+    } else {
+        /* Attempt to open MaxMind DB and save file handle if successful */
+        int status = MMDB_open(geoip_db_s, MMDB_MODE_MMAP, &mmdb);
+        mmdb_status = status;
+        if (mmdb_status == MMDB_SUCCESS) {
+            SCLogConfig("Opened GeoLite2 database successfully, path %s", geoip_db_s);
+            return true;
+        } else {
+            SCLogWarning("Failed to open GeoLite2 database, path %s, error message %s", geoip_db_s,
+                    MMDB_strerror(mmdb_status));
+            return false;
+        }
+    }
+}
+#endif /* HAVE_GEOIP */
+
 static void OutputJsonDeInitCtx(OutputCtx *);
 static void CreateEveCommunityFlowId(SCJsonBuilder *js, const Flow *f, const uint16_t seed);
 static int CreateJSONEther(
@@ -909,6 +1065,13 @@ SCJsonBuilder *CreateEveHeader(const Packet *p, enum SCOutputJsonLogDirection di
         SCJbSetUint(js, "ip_v", 6);
     }
 
+#ifdef HAVE_GEOIP
+    if (eve_ctx != NULL && eve_ctx->cfg.geoip_enabled) {
+        SCGeoIPGet(js, addr->src_ip, "geoip_src");
+        SCGeoIPGet(js, addr->dst_ip, "geoip_dst");
+    }
+#endif /* HAVE_GEOIP */
+
     /* icmp */
     switch (p->proto) {
         case IPPROTO_ICMP:
@@ -1184,6 +1347,10 @@ OutputInitResult OutputJsonInitCtx(SCConfNode *conf)
                 FatalError("Invalid JSON output option: %s", output_s);
         }
 
+#ifdef HAVE_GEOIP
+        json_ctx->cfg.geoip_enabled = GeoEnrichmentInit(conf);
+#endif /* HAVE_GEOIP */
+
         const char *prefix = SCConfNodeLookupChildValue(conf, "prefix");
         if (prefix != NULL)
         {
@@ -1309,6 +1476,12 @@ static void OutputJsonDeInitCtx(OutputCtx *output_ctx)
                      "disconnected socket",
                 logfile_ctx->dropped);
     }
+#ifdef HAVE_GEOIP
+    if (mmdb_status == MMDB_SUCCESS) {
+        MMDB_close(&mmdb);
+        SCLogDebug("GeoLite2 database is closed");
+    }
+#endif /* HAVE_GEOIP */
     if (json_ctx->xff_cfg != NULL) {
         SCFree(json_ctx->xff_cfg);
     }

src/output-json.h

@@ -66,6 +66,7 @@ typedef struct OutputJsonCommonSettings_ {
     bool include_community_id;
     bool include_ethernet;
     bool include_suricata_version;
+    bool geoip_enabled;
     uint16_t community_id_seed;
 } OutputJsonCommonSettings;
 
@@ -118,4 +119,8 @@ void FreeEveThreadCtx(OutputJsonThreadCtx *ctx);
 void JSONFormatAndAddMACAddr(SCJsonBuilder *js, const char *key, const uint8_t *val, bool is_array);
 void OutputJsonFlush(OutputJsonThreadCtx *ctx);
 
+#ifdef HAVE_GEOIP
+void SCGeoIPGet(SCJsonBuilder *js, const char *ip_address, const char *key);
+#endif
+
 #endif /* SURICATA_OUTPUT_JSON_H */

suricata.yaml.in

@@ -99,6 +99,9 @@ outputs:
       enabled: yes
       filetype: regular #regular|syslog|unix_dgram|unix_stream|redis
       filename: eve.json
+      # Enable GeoIP on source ip and destination ip, default is disabled
+      # Make sure that you have set geoip-database path
+      #geoip-enrichment: yes
       # Enable for multi-threaded eve.json output; output files are amended with
       # an identifier, e.g., eve.9.json
       #threaded: false
@@ -1410,6 +1413,8 @@ unix-command:
 
 # GeoIP2 database file. Specify path and filename of GeoIP2 database
 # if using rules with "geoip" rule option.
+# This database is also used by geoip-enrichment in EVE output.
+# If you wish to enrich in city level please use GeoLite2 City database.
 #geoip-database: /usr/local/share/GeoLite2/GeoLite2-Country.mmdb
 
 legacy: