diff --git a/doc/userguide/output/eve/eve-json-output.rst b/doc/userguide/output/eve/eve-json-output.rst index a6c88d1e640a..863367198f94 100644 --- a/doc/userguide/output/eve/eve-json-output.rst +++ b/doc/userguide/output/eve/eve-json-output.rst @@ -58,6 +58,9 @@ Output types:: filetype: regular #regular|syslog|unix_dgram|unix_stream|redis filename: eve.json + # Enable GeoIP on source ip and destination ip, default is disabled + # Make sure that you have set geoip-database path + #geoip-enrichment: yes # Enable for multi-threaded eve.json output; output files are amended # with an identifier, e.g., eve.9.json. Default: off #threaded: off diff --git a/doc/userguide/partials/eve-log.yaml b/doc/userguide/partials/eve-log.yaml index 845d7e1157c1..f1ba82a590ad 100644 --- a/doc/userguide/partials/eve-log.yaml +++ b/doc/userguide/partials/eve-log.yaml @@ -4,6 +4,9 @@ outputs: enabled: yes filetype: regular #regular|syslog|unix_dgram|unix_stream|redis filename: eve.json + # Enable GeoIP on source ip and destination ip + # Disable geoip enrichment by commenting geoip-enrichment + #geoip-enrichment: yes # Enable for multi-threaded eve.json output; output files are amended with # an identifier, e.g., eve.9.json #threaded: false diff --git a/etc/schema.json b/etc/schema.json index ce25e458a2ad..8b1e002a64b8 100644 --- a/etc/schema.json +++ b/etc/schema.json @@ -6,6 +6,90 @@ "timestamp" ], "properties": { + "geoip_src": { + "type": "object", + "properties": { + "ip":{ + "type": "string" + }, + "geo": { + "type": "object", + "properties": { + "continent_code": { + "type": "string" + }, + "country_iso_code": { + "type": "string" + }, + "city_name": { + "type": "string" + }, + "country_name": { + "type": "string" + }, + "continent_name": { + "type": "string" + }, + "timezone": { + "type": "string" + }, + "location": { + "type": "object", + "properties": { + "latitude": { + "type": "number" + }, + "longitude": { + "type": "number" + } + } + } + } + } + } + }, + "geoip_dst": { + "type": "object", + "properties": { + "ip":{ + "type": "string" + }, + "geo": { + "type": "object", + "properties": { + "continent_code": { + "type": "string" + }, + "country_iso_code": { + "type": "string" + }, + "city_name": { + "type": "string" + }, + "country_name": { + "type": "string" + }, + "continent_name": { + "type": "string" + }, + "timezone": { + "type": "string" + }, + "location": { + "type": "object", + "properties": { + "latitude": { + "type": "number" + }, + "longitude": { + "type": "number" + } + } + } + } + } + } + }, "alert": { "type": "object", "additionalProperties": false, @@ -3499,7 +3583,7 @@ "$comment": "keyword: app-layer-event:mdns.name_too_long (https://redmine.openinfosecfoundation.org/issues/7784)" }, "rrtype": { - "type": "string", + "type": "string", "description": "Type of resource being requested" } } diff --git a/src/output-json-flow.c b/src/output-json-flow.c index 2bd5d0129b1f..b2ca03a45ca6 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -53,7 +53,7 @@ #include "flow-storage.h" #include "util-exception-policy.h" -static SCJsonBuilder *CreateEveHeaderFromFlow(const Flow *f) +static SCJsonBuilder *CreateEveHeaderFromFlow(const Flow *f, const OutputJsonCommonSettings *cfg) { char timebuf[64]; char srcip[46] = {0}, dstip[46] = {0}; @@ -93,6 +93,13 @@ static SCJsonBuilder *CreateEveHeaderFromFlow(const Flow *f) /* time */ SCJbSetString(jb, "timestamp", timebuf); +#ifdef HAVE_GEOIP + if (cfg != NULL && cfg->geoip_enabled) { + SCGeoIPGet(jb, srcip, "geoip_src"); + SCGeoIPGet(jb, dstip, "geoip_dst"); + } +#endif /* HAVE_GEOIP */ + CreateEveFlowId(jb, (const Flow *)f); #if 0 // TODO @@ -432,7 +439,7 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) /* reset */ MemBufferReset(thread->buffer); - SCJsonBuilder *jb = CreateEveHeaderFromFlow(f); + SCJsonBuilder *jb = CreateEveHeaderFromFlow(f, &thread->ctx->cfg); if (unlikely(jb == NULL)) { SCReturnInt(TM_ECODE_OK); } diff --git a/src/output-json.c b/src/output-json.c index dfea4602c201..d1974b3edc81 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -68,6 +68,162 @@ #define MAX_JSON_SIZE 2048 +#ifdef HAVE_GEOIP +#include + +static MMDB_s mmdb; +static int mmdb_status = MMDB_FILE_OPEN_ERROR; + +#define GeoIPSetString(js, entry_data, key) \ + { \ + if (entry_data.has_data && entry_data.utf8_string != NULL) { \ + SCJbSetStringFromBytes( \ + js, key, (const uint8_t *)entry_data.utf8_string, entry_data.data_size); \ + } \ + } + +static bool MMDB_fetch_and_set_json_string( + SCJsonBuilder *js, MMDB_lookup_result_s result, const char *key, ...); + +static bool MMDB_fetch_and_set_json_double( + SCJsonBuilder *js, MMDB_lookup_result_s result, const char *key, ...); + +void SCGeoIPGet(SCJsonBuilder *js, const char *ip_address, const char *key); + +static bool GeoEnrichmentInit(SCConfNode *conf); + +static bool MMDB_fetch_and_set_json_string( + SCJsonBuilder *js, MMDB_lookup_result_s result, const char *key, ...) +{ + MMDB_entry_data_s entry_data; + bool success = false; + va_list args; + va_start(args, key); + if (MMDB_vget_value(&result.entry, &entry_data, args) == MMDB_SUCCESS) { + GeoIPSetString(js, entry_data, key); + success = true; + } + va_end(args); + + return success; +} + +static bool MMDB_fetch_and_set_json_double( + SCJsonBuilder *js, MMDB_lookup_result_s result, const char *key, ...) +{ + MMDB_entry_data_s entry_data; + bool success = false; + va_list args; + va_start(args, key); + if (MMDB_vget_value(&result.entry, &entry_data, args) == MMDB_SUCCESS) { + SCJbSetFloat(js, key, entry_data.double_value); + success = true; + } + va_end(args); + + return success; +} + +void SCGeoIPGet(SCJsonBuilder *js, const char *ip_address, const char *key) +{ + int gai_error, mmdb_error; + SCJsonBuilderMark mark_geo = { 0, 0, 0 }; + SCJsonBuilderMark mark_location = { 0, 0, 0 }; + bool non_empty_location = false; + bool non_empty_geo = false; + + if (mmdb_status != MMDB_SUCCESS) { + return; + } + MMDB_lookup_result_s result = MMDB_lookup_string(&mmdb, ip_address, &gai_error, &mmdb_error); + if (MMDB_SUCCESS != gai_error) { + return; + } + + if (!result.found_entry) { + return; + } + + SCJbOpenObject(js, key); + SCJbSetString(js, "ip", ip_address); + + /* Create geo object */ + SCJbGetMark(js, &mark_geo); + SCJbOpenObject(js, "geo"); + + non_empty_geo |= + MMDB_fetch_and_set_json_string(js, result, "continent_code", "continent", "code", NULL); + non_empty_geo |= MMDB_fetch_and_set_json_string( + js, result, "country_iso_code", "country", "iso_code", NULL); + non_empty_geo |= + MMDB_fetch_and_set_json_string(js, result, "city_name", "city", "names", "en", NULL); + non_empty_geo |= MMDB_fetch_and_set_json_string( + js, result, "country_name", "country", "names", "en", NULL); + non_empty_geo |= MMDB_fetch_and_set_json_string( + js, result, "continent_name", "continent", "names", "en", NULL); + non_empty_geo |= + MMDB_fetch_and_set_json_string(js, result, "timezone", "location", "time_zone", NULL); + + /* Create location object */ + SCJbGetMark(js, &mark_location); + SCJbOpenObject(js, "location"); + non_empty_location |= + MMDB_fetch_and_set_json_double(js, result, "lat", "location", "latitude", NULL); + non_empty_location |= + MMDB_fetch_and_set_json_double(js, result, "lon", "location", "longitude", NULL); + + if (!non_empty_location) { + SCJbRestoreMark(js, &mark_location); + } else { + SCJbClose(js); /* close location */ + } + + non_empty_geo |= non_empty_location; + if (!non_empty_geo) { + SCJbRestoreMark(js, &mark_geo); + } else { + SCJbClose(js); /* close geo */ + } + + SCJbClose(js); /* close key */ +} + +static bool GeoEnrichmentInit(SCConfNode *conf) +{ + const SCConfNode *geoip_enrichment = SCConfNodeLookupChild(conf, "geoip-enrichment"); + /* Only enable if explicitly set to true/yes */ + if (geoip_enrichment == NULL || geoip_enrichment->val == NULL || + !SCConfValIsTrue(geoip_enrichment->val)) { + SCLogConfig("GeoIP enrichment is disabled."); + return false; + } + + const char *geoip_db_s = NULL; + + SCLogConfig("GeoIP enrichment is enabled."); + (void)SCConfGet("geoip-database", &geoip_db_s); + if (geoip_db_s == NULL) { + SCLogWarning("geoip-database should be set for geoip-enrichment functionality"); + return false; + } else if (mmdb_status == MMDB_SUCCESS) { + /* mmdb already opened by another eve-log output */ + return true; + } else { + /* Attempt to open MaxMind DB and save file handle if successful */ + int status = MMDB_open(geoip_db_s, MMDB_MODE_MMAP, &mmdb); + mmdb_status = status; + if (mmdb_status == MMDB_SUCCESS) { + SCLogConfig("Opened GeoLite2 database successfully, path %s", geoip_db_s); + return true; + } else { + SCLogWarning("Failed to open GeoLite2 database, path %s, error message %s", + geoip_db_s, MMDB_strerror(mmdb_status)); + return false; + } + } +} +#endif /* HAVE_GEOIP */ + static void OutputJsonDeInitCtx(OutputCtx *); static void CreateEveCommunityFlowId(SCJsonBuilder *js, const Flow *f, const uint16_t seed); static int CreateJSONEther( @@ -909,6 +1065,13 @@ SCJsonBuilder *CreateEveHeader(const Packet *p, enum SCOutputJsonLogDirection di SCJbSetUint(js, "ip_v", 6); } +#ifdef HAVE_GEOIP + if (eve_ctx != NULL && eve_ctx->cfg.geoip_enabled) { + SCGeoIPGet(js, addr->src_ip, "geoip_src"); + SCGeoIPGet(js, addr->dst_ip, "geoip_dst"); + } +#endif /* HAVE_GEOIP */ + /* icmp */ switch (p->proto) { case IPPROTO_ICMP: @@ -1184,6 +1347,10 @@ OutputInitResult OutputJsonInitCtx(SCConfNode *conf) FatalError("Invalid JSON output option: %s", output_s); } +#ifdef HAVE_GEOIP + json_ctx->cfg.geoip_enabled = GeoEnrichmentInit(conf); +#endif /* HAVE_GEOIP */ + const char *prefix = SCConfNodeLookupChildValue(conf, "prefix"); if (prefix != NULL) { @@ -1309,6 +1476,12 @@ static void OutputJsonDeInitCtx(OutputCtx *output_ctx) "disconnected socket", logfile_ctx->dropped); } +#ifdef HAVE_GEOIP + if (mmdb_status == MMDB_SUCCESS) { + MMDB_close(&mmdb); + SCLogDebug("GeoLite2 database is closed"); + } +#endif /* HAVE_GEOIP */ if (json_ctx->xff_cfg != NULL) { SCFree(json_ctx->xff_cfg); } diff --git a/src/output-json.h b/src/output-json.h index 1f4fec70d041..da2449953400 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -66,6 +66,7 @@ typedef struct OutputJsonCommonSettings_ { bool include_community_id; bool include_ethernet; bool include_suricata_version; + bool geoip_enabled; uint16_t community_id_seed; } OutputJsonCommonSettings; @@ -118,4 +119,8 @@ void FreeEveThreadCtx(OutputJsonThreadCtx *ctx); void JSONFormatAndAddMACAddr(SCJsonBuilder *js, const char *key, const uint8_t *val, bool is_array); void OutputJsonFlush(OutputJsonThreadCtx *ctx); +#ifdef HAVE_GEOIP +void SCGeoIPGet(SCJsonBuilder *js, const char *ip_address, const char *key); +#endif + #endif /* SURICATA_OUTPUT_JSON_H */ diff --git a/suricata.yaml.in b/suricata.yaml.in index 86d2ef9f3868..6d70380fb63b 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -99,6 +99,9 @@ outputs: enabled: yes filetype: regular #regular|syslog|unix_dgram|unix_stream|redis filename: eve.json + # Enable GeoIP on source ip and destination ip, default is disabled + # Make sure that you have set geoip-database path + #geoip-enrichment: yes # Enable for multi-threaded eve.json output; output files are amended with # an identifier, e.g., eve.9.json #threaded: false @@ -1410,6 +1413,8 @@ unix-command: # GeoIP2 database file. Specify path and filename of GeoIP2 database # if using rules with "geoip" rule option. +# This database is also used by geoip-enrichment in EVE output. +# If you wish to enrich in city level please use GeoLite2 City database. #geoip-database: /usr/local/share/GeoLite2/GeoLite2-Country.mmdb legacy: