From 4f0d9c055c3b22376d19ef90241eaaf334cec063 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Tue, 14 Jun 2022 09:29:04 +0200 Subject: [PATCH] http: logs custom headers in a subobject This subobject is request_headers or response_headers This especially avoids json keys collisions. Ticket: #5320 Also fixes typo referrer/referer --- doc/userguide/output/eve/eve-json-format.rst | 4 +- doc/userguide/output/eve/eve-json-output.rst | 2 +- src/output-json-http.c | 121 +++++++++---------- 3 files changed, 58 insertions(+), 69 deletions(-) diff --git a/doc/userguide/output/eve/eve-json-format.rst b/doc/userguide/output/eve/eve-json-format.rst index b872e5da2975..89377096566b 100644 --- a/doc/userguide/output/eve/eve-json-format.rst +++ b/doc/userguide/output/eve/eve-json-format.rst @@ -286,7 +286,7 @@ suricata.yaml file the following fields are (can) also included: * "status": HTTP status code * "protocol": Protocol / Version of HTTP (ex: HTTP/1.1) * "http_method": The HTTP method (ex: GET, POST, HEAD) -* "http_refer": The referrer for this action +* "http_refer": The referer for this action In addition to the extended logging fields one can also choose to enable/add from more than 50 additional custom logging HTTP fields enabled in the @@ -318,7 +318,7 @@ suricata.yaml file. The additional fields can be enabled as following: allow, connection, content-encoding, content-language, content-length, content-location, content-md5, content-range, content-type, date, etags, expires, last-modified, link, location, - proxy-authenticate, referrer, refresh, retry-after, server, + proxy-authenticate, referer, refresh, retry-after, server, set-cookie, trailer, transfer-encoding, upgrade, vary, warning, www-authenticate, x-flash-version, x-authenticated-user] diff --git a/doc/userguide/output/eve/eve-json-output.rst b/doc/userguide/output/eve/eve-json-output.rst index 17e7ea2bab00..09ca09a65ff0 100644 --- a/doc/userguide/output/eve/eve-json-output.rst +++ b/doc/userguide/output/eve/eve-json-output.rst @@ -192,7 +192,7 @@ last_modified last-modified link link location location proxy_authenticate proxy-authenticate -referrer referrer +referer referer refresh refresh retry_after retry-after server server diff --git a/src/output-json-http.c b/src/output-json-http.c index 0dbbfee7d19e..824c04e2e7cf 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -138,7 +138,7 @@ struct { const char *config_field; const char *htp_field; uint32_t flags; -} http_fields[] = { +} http_fields[] = { { "accept", "accept", LOG_HTTP_REQUEST }, { "accept_charset", "accept-charset", LOG_HTTP_REQUEST }, { "accept_encoding", "accept-encoding", LOG_HTTP_REQUEST }, @@ -146,7 +146,7 @@ struct { { "accept_datetime", "accept-datetime", LOG_HTTP_REQUEST }, { "authorization", "authorization", LOG_HTTP_REQUEST }, { "cache_control", "cache-control", LOG_HTTP_REQUEST }, - { "cookie", "cookie", LOG_HTTP_REQUEST|LOG_HTTP_ARRAY }, + { "cookie", "cookie", LOG_HTTP_REQUEST | LOG_HTTP_ARRAY }, { "from", "from", LOG_HTTP_REQUEST }, { "max_forwards", "max-forwards", LOG_HTTP_REQUEST }, { "origin", "origin", LOG_HTTP_REQUEST }, @@ -173,12 +173,12 @@ struct { { "content_type", "content-type", 0 }, { "date", "date", 0 }, { "etag", "etags", 0 }, - { "expires", "expires" , 0 }, + { "expires", "expires", 0 }, { "last_modified", "last-modified", 0 }, { "link", "link", 0 }, { "location", "location", 0 }, { "proxy_authenticate", "proxy-authenticate", 0 }, - { "referrer", "referrer", LOG_HTTP_EXTENDED }, + { "referer", "referer", LOG_HTTP_EXTENDED }, { "refresh", "refresh", 0 }, { "retry_after", "retry-after", 0 }, { "server", "server", 0 }, @@ -264,46 +264,6 @@ static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx) } } -static void EveHttpLogJSONCustom(LogHttpFileCtx *http_ctx, JsonBuilder *js, htp_tx_t *tx) -{ - char *c; - HttpField f; - - for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++) - { - if ((http_ctx->fields & (1ULL<flags & LOG_HTTP_EXTENDED) == 0) || - ((http_ctx->flags & LOG_HTTP_EXTENDED) != - (http_fields[f].flags & LOG_HTTP_EXTENDED))) - { - htp_header_t *h_field = NULL; - if ((http_fields[f].flags & LOG_HTTP_REQUEST) != 0) - { - if (tx->request_headers != NULL) { - h_field = htp_table_get_c(tx->request_headers, - http_fields[f].htp_field); - } - } else { - if (tx->response_headers != NULL) { - h_field = htp_table_get_c(tx->response_headers, - http_fields[f].htp_field); - } - } - if (h_field != NULL) { - c = bstr_util_strdup_to_c(h_field->value); - if (c != NULL) { - jb_set_string(js, http_fields[f].config_field, c); - SCFree(c); - } - } - } - } - } -} - static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx) { /* referer */ @@ -348,19 +308,44 @@ static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx) jb_set_uint(js, "length", tx->response_message_len); } -static void EveHttpLogJSONHeaders(JsonBuilder *js, uint32_t direction, htp_tx_t *tx) +static void EveHttpLogJSONHeaders( + JsonBuilder *js, uint32_t direction, htp_tx_t *tx, LogHttpFileCtx *http_ctx) { htp_table_t * headers = direction & LOG_HTTP_REQ_HEADERS ? tx->request_headers : tx->response_headers; char name[MAX_SIZE_HEADER_NAME] = {0}; char value[MAX_SIZE_HEADER_VALUE] = {0}; size_t n = htp_table_size(headers); + JsonBuilderMark mark = { 0, 0, 0 }; + jb_get_mark(js, &mark); + bool array_empty = true; jb_open_array(js, direction & LOG_HTTP_REQ_HEADERS ? "request_headers" : "response_headers"); for (size_t i = 0; i < n; i++) { htp_header_t * h = htp_table_get_index(headers, i, NULL); if (h == NULL) { continue; } + if ((http_ctx->flags & direction) == 0 && http_ctx->fields != 0) { + bool tolog = false; + for (HttpField f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++) { + if ((http_ctx->fields & (1ULL << f)) != 0) { + /* prevent logging a field twice if extended logging is + enabled */ + if (((http_ctx->flags & LOG_HTTP_EXTENDED) == 0) || + ((http_ctx->flags & LOG_HTTP_EXTENDED) != + (http_fields[f].flags & LOG_HTTP_EXTENDED))) { + if (bstr_cmp_c_nocase(h->name, http_fields[f].htp_field) == 0) { + tolog = true; + break; + } + } + } + } + if (!tolog) { + continue; + } + } + array_empty = false; jb_start_object(js); size_t size_name = bstr_len(h->name) < MAX_SIZE_HEADER_NAME - 1 ? bstr_len(h->name) : MAX_SIZE_HEADER_NAME - 1; @@ -374,8 +359,12 @@ static void EveHttpLogJSONHeaders(JsonBuilder *js, uint32_t direction, htp_tx_t jb_set_string(js, "value", value); jb_close(js); } - // Close array. - jb_close(js); + if (array_empty) { + jb_restore_mark(js, &mark); + } else { + // Close array. + jb_close(js); + } } static void BodyPrintableBuffer(JsonBuilder *js, HtpBody *body, const char *key) @@ -454,15 +443,12 @@ static void EveHttpLogJSON(JsonHttpLogThread *aft, JsonBuilder *js, htp_tx_t *tx jb_open_object(js, "http"); EveHttpLogJSONBasic(js, tx); - /* log custom fields if configured */ - if (http_ctx->fields != 0) - EveHttpLogJSONCustom(http_ctx, js, tx); if (http_ctx->flags & LOG_HTTP_EXTENDED) EveHttpLogJSONExtended(js, tx); - if (http_ctx->flags & LOG_HTTP_REQ_HEADERS) - EveHttpLogJSONHeaders(js, LOG_HTTP_REQ_HEADERS, tx); - if (http_ctx->flags & LOG_HTTP_RES_HEADERS) - EveHttpLogJSONHeaders(js, LOG_HTTP_RES_HEADERS, tx); + if (http_ctx->flags & LOG_HTTP_REQ_HEADERS || http_ctx->fields != 0) + EveHttpLogJSONHeaders(js, LOG_HTTP_REQ_HEADERS, tx, http_ctx); + if (http_ctx->flags & LOG_HTTP_RES_HEADERS || http_ctx->fields != 0) + EveHttpLogJSONHeaders(js, LOG_HTTP_RES_HEADERS, tx, http_ctx); jb_close(js); } @@ -566,8 +552,23 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c } } + const char *all_headers = ConfNodeLookupChildValue(conf, "dump-all-headers"); + if (all_headers != NULL) { + if (strncmp(all_headers, "both", 4) == 0) { + http_ctx->flags |= LOG_HTTP_REQ_HEADERS; + http_ctx->flags |= LOG_HTTP_RES_HEADERS; + } else if (strncmp(all_headers, "request", 7) == 0) { + http_ctx->flags |= LOG_HTTP_REQ_HEADERS; + } else if (strncmp(all_headers, "response", 8) == 0) { + http_ctx->flags |= LOG_HTTP_RES_HEADERS; + } + } ConfNode *custom; if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) { + if ((http_ctx->flags & (LOG_HTTP_REQ_HEADERS | LOG_HTTP_RES_HEADERS)) == + (LOG_HTTP_REQ_HEADERS | LOG_HTTP_RES_HEADERS)) { + SCLogWarning("No need for custom as dump-all-headers is already present"); + } ConfNode *field; TAILQ_FOREACH(field, &custom->head, next) { @@ -588,18 +589,6 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c } } } - const char *all_headers = ConfNodeLookupChildValue( - conf, "dump-all-headers"); - if (all_headers != NULL) { - if (strncmp(all_headers, "both", 4) == 0) { - http_ctx->flags |= LOG_HTTP_REQ_HEADERS; - http_ctx->flags |= LOG_HTTP_RES_HEADERS; - } else if (strncmp(all_headers, "request", 7) == 0) { - http_ctx->flags |= LOG_HTTP_REQ_HEADERS; - } else if (strncmp(all_headers, "response", 8) == 0) { - http_ctx->flags |= LOG_HTTP_RES_HEADERS; - } - } } if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) {