From e4c20d0ce0285da82743d177c5230019a1285860 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 4 Jul 2024 13:55:27 +0200 Subject: [PATCH 1/4] util/thash: decrease memuse if array was allocated THashInitConfig may not allocate array and increase memuse. Such a failure leads to THashShutdown which should not decrease the memuse. Ticket: 7135 --- src/util-thash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util-thash.c b/src/util-thash.c index 649d8b98b845..74c74e245a69 100644 --- a/src/util-thash.c +++ b/src/util-thash.c @@ -378,8 +378,8 @@ void THashShutdown(THashTableContext *ctx) } SCFreeAligned(ctx->array); ctx->array = NULL; + (void)SC_ATOMIC_SUB(ctx->memuse, ctx->config.hash_size * sizeof(THashHashRow)); } - (void) SC_ATOMIC_SUB(ctx->memuse, ctx->config.hash_size * sizeof(THashHashRow)); THashDataQueueDestroy(&ctx->spare_q); DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(ctx->memuse) != 0); SCFree(ctx); From 61e839825ae741c1a891a10b062159453d0d01b3 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 4 Jul 2024 15:07:01 +0200 Subject: [PATCH 2/4] runmodes: use dynamic number of app-layer protos Ticket: 5053 --- src/runmodes.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/runmodes.c b/src/runmodes.c index 287567570c3c..b5c335176b06 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -540,7 +540,7 @@ static void RunOutputFreeList(void) static int file_logger_count = 0; static int filedata_logger_count = 0; -static LoggerId logger_bits[ALPROTO_MAX]; +static LoggerId *logger_bits = NULL; int RunModeOutputFiledataEnabled(void) { @@ -592,6 +592,7 @@ void RunModeShutDown(void) OutputClearActiveLoggers(); + SCFree(logger_bits); /* Reset logger counts. */ file_logger_count = 0; filedata_logger_count = 0; @@ -790,8 +791,11 @@ void RunModeInitializeOutputs(void) char tls_log_enabled = 0; char tls_store_present = 0; - memset(&logger_bits, 0, sizeof(logger_bits)); - + // ALPROTO_MAX is set to its final value + logger_bits = SCCalloc(ALPROTO_MAX, sizeof(LoggerId)); + if (unlikely(logger_bits == NULL)) { + FatalError("Failed to allocate logger_bits"); + } TAILQ_FOREACH(output, &outputs->head, next) { output_config = ConfNodeLookupChild(output, output->val); From 482f3c677aee184a9924ed4ce220ff5c4586bf32 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 4 Jul 2024 15:11:53 +0200 Subject: [PATCH 3/4] output/tx: use dynamic number of app-layer protos Ticket: 5053 --- src/output-tx.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/output-tx.c b/src/output-tx.c index 7bc569ef9016..449cff665062 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -61,7 +61,7 @@ typedef struct OutputTxLogger_ { void (*ThreadExitPrintStats)(ThreadVars *, void *); } OutputTxLogger; -static OutputTxLogger *list[ALPROTO_MAX] = { NULL }; +static OutputTxLogger **list = NULL; int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, TxLogger LogFunc, @@ -668,6 +668,12 @@ static uint32_t OutputTxLoggerGetActiveCount(void) void OutputTxLoggerRegister (void) { + BUG_ON(list); + list = SCCalloc(ALPROTO_MAX, sizeof(OutputTxLogger *)); + if (unlikely(list == NULL)) { + FatalError("Failed to allocate OutputTx list"); + } + OutputRegisterRootLogger(OutputTxLogThreadInit, OutputTxLogThreadDeinit, OutputTxLogExitPrintStats, OutputTxLog, OutputTxLoggerGetActiveCount); } @@ -683,4 +689,6 @@ void OutputTxShutdown(void) } list[alproto] = NULL; } + SCFree(list); + list = NULL; } From f5075b6e88d3f0feeb8a1f5a5a0543f27b16d8b9 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 4 Jul 2024 15:31:18 +0200 Subject: [PATCH 4/4] output: use dynamic number of app-layer protos Ticket: 5053 --- src/output.c | 137 +++++++++++++++++++++++++-------------------------- src/output.h | 2 +- 2 files changed, 68 insertions(+), 71 deletions(-) diff --git a/src/output.c b/src/output.c index 7b13913c831b..46a777e378e7 100644 --- a/src/output.c +++ b/src/output.c @@ -669,6 +669,8 @@ OutputModule *OutputGetModuleByConfName(const char *conf_name) return NULL; } +static EveJsonSimpleAppLayerLogger *simple_json_applayer_loggers; + /** * \brief Deregister all modules. Useful for a memory clean exit. */ @@ -680,6 +682,8 @@ void OutputDeregisterAll(void) TAILQ_REMOVE(&output_modules, module, entries); SCFree(module); } + SCFree(simple_json_applayer_loggers); + simple_json_applayer_loggers = NULL; } static int drop_loggers = 0; @@ -895,11 +899,73 @@ void TmModuleLoggerRegister(void) OutputRegisterLoggers(); } +EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto) +{ + if (alproto < ALPROTO_MAX) { + return &simple_json_applayer_loggers[alproto]; + } + return NULL; +} + +static void RegisterSimpleJsonApplayerLogger( + AppProto alproto, EveJsonSimpleTxLogFunc LogTx, const char *name) +{ + simple_json_applayer_loggers[alproto].LogTx = LogTx; + if (name) { + simple_json_applayer_loggers[alproto].name = name; + } else { + simple_json_applayer_loggers[alproto].name = AppProtoToString(alproto); + } +} + /** * \brief Register all root loggers. */ void OutputRegisterRootLoggers(void) { + simple_json_applayer_loggers = SCCalloc(ALPROTO_MAX, sizeof(EveJsonSimpleAppLayerLogger)); + if (unlikely(simple_json_applayer_loggers == NULL)) { + FatalError("Failed to allocate simple_json_applayer_loggers"); + } + // ALPROTO_HTTP1 special: uses some options flags + RegisterSimpleJsonApplayerLogger(ALPROTO_FTP, EveFTPLogCommand, NULL); + // ALPROTO_SMTP special: uses state + RegisterSimpleJsonApplayerLogger(ALPROTO_TLS, JsonTlsLogJSONExtended, NULL); + // no cast here but done in rust for SSHTransaction + RegisterSimpleJsonApplayerLogger(ALPROTO_SSH, rs_ssh_log_json, NULL); + // ALPROTO_SMB special: uses state + // ALPROTO_DCERPC special: uses state + RegisterSimpleJsonApplayerLogger(ALPROTO_DNS, AlertJsonDns, NULL); + // either need a cast here or in rust for ModbusTransaction, done here + RegisterSimpleJsonApplayerLogger( + ALPROTO_MODBUS, (EveJsonSimpleTxLogFunc)rs_modbus_to_json, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_ENIP, SCEnipLoggerLog, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_DNP3, AlertJsonDnp3, NULL); + // ALPROTO_NFS special: uses state + // underscore instead of dash for ftp_data + RegisterSimpleJsonApplayerLogger(ALPROTO_FTPDATA, EveFTPDataAddMetadata, "ftp_data"); + RegisterSimpleJsonApplayerLogger( + ALPROTO_TFTP, (EveJsonSimpleTxLogFunc)rs_tftp_log_json_request, NULL); + // ALPROTO_IKE special: uses state + RegisterSimpleJsonApplayerLogger( + ALPROTO_KRB5, (EveJsonSimpleTxLogFunc)rs_krb5_log_json_response, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_QUIC, rs_quic_to_json, NULL); + // ALPROTO_DHCP TODO missing + RegisterSimpleJsonApplayerLogger( + ALPROTO_SNMP, (EveJsonSimpleTxLogFunc)rs_snmp_log_json_response, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_SIP, (EveJsonSimpleTxLogFunc)rs_sip_log_json, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_RFB, rs_rfb_logger_log, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_MQTT, JsonMQTTAddMetadata, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_PGSQL, JsonPgsqlAddMetadata, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_WEBSOCKET, rs_websocket_logger_log, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_TEMPLATE, rs_template_logger_log, NULL); + RegisterSimpleJsonApplayerLogger(ALPROTO_RDP, (EveJsonSimpleTxLogFunc)rs_rdp_to_json, NULL); + // special case : http2 is logged in http object + RegisterSimpleJsonApplayerLogger(ALPROTO_HTTP2, rs_http2_log_json, "http"); + // underscore instead of dash for bittorrent_dht + RegisterSimpleJsonApplayerLogger( + ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log, "bittorrent_dht"); + OutputPacketLoggerRegister(); OutputFiledataLoggerRegister(); OutputFileLoggerRegister(); @@ -916,24 +982,7 @@ static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p, return TM_ECODE_FAILED; } - const char *name; - switch (al->proto) { - case ALPROTO_HTTP2: - // special case - name = "http"; - break; - case ALPROTO_FTPDATA: - // underscore instead of dash - name = "ftp_data"; - break; - case ALPROTO_BITTORRENT_DHT: - // underscore instead of dash - name = "bittorrent_dht"; - break; - default: - name = AppProtoToString(al->proto); - } - JsonBuilder *js = CreateEveHeader(p, dir, name, NULL, thread->ctx); + JsonBuilder *js = CreateEveHeader(p, dir, al->name, NULL, thread->ctx); if (unlikely(js == NULL)) { return TM_ECODE_FAILED; } @@ -1115,55 +1164,3 @@ void OutputRegisterLoggers(void) /* ARP JSON logger */ JsonArpLogRegister(); } - -static EveJsonSimpleAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = { - { ALPROTO_UNKNOWN, NULL }, - { ALPROTO_HTTP1, NULL }, // special: uses some options flags - { ALPROTO_FTP, EveFTPLogCommand }, - { ALPROTO_SMTP, NULL }, // special: uses state - { ALPROTO_TLS, JsonTlsLogJSONExtended }, - { ALPROTO_SSH, rs_ssh_log_json }, - { ALPROTO_IMAP, NULL }, // protocol detection only - { ALPROTO_JABBER, NULL }, // no parser, no logging - { ALPROTO_SMB, NULL }, // special: uses state - { ALPROTO_DCERPC, NULL }, // special: uses state - { ALPROTO_IRC, NULL }, // no parser, no logging - { ALPROTO_DNS, AlertJsonDns }, - { ALPROTO_MODBUS, (EveJsonSimpleTxLogFunc)rs_modbus_to_json }, - { ALPROTO_ENIP, SCEnipLoggerLog }, - { ALPROTO_DNP3, AlertJsonDnp3 }, - { ALPROTO_NFS, NULL }, // special: uses state - { ALPROTO_NTP, NULL }, // no logging - { ALPROTO_FTPDATA, EveFTPDataAddMetadata }, - { ALPROTO_TFTP, (EveJsonSimpleTxLogFunc)rs_tftp_log_json_request }, - { ALPROTO_IKE, NULL }, // special: uses state - { ALPROTO_KRB5, (EveJsonSimpleTxLogFunc)rs_krb5_log_json_response }, - { ALPROTO_QUIC, rs_quic_to_json }, - { ALPROTO_DHCP, NULL }, // TODO missing - { ALPROTO_SNMP, (EveJsonSimpleTxLogFunc)rs_snmp_log_json_response }, - { ALPROTO_SIP, (EveJsonSimpleTxLogFunc)rs_sip_log_json }, - { ALPROTO_RFB, rs_rfb_logger_log }, - { ALPROTO_MQTT, JsonMQTTAddMetadata }, - { ALPROTO_PGSQL, JsonPgsqlAddMetadata }, - { ALPROTO_TELNET, NULL }, // no logging - { ALPROTO_WEBSOCKET, rs_websocket_logger_log }, - { ALPROTO_TEMPLATE, rs_template_logger_log }, - { ALPROTO_RDP, (EveJsonSimpleTxLogFunc)rs_rdp_to_json }, - { ALPROTO_HTTP2, rs_http2_log_json }, - { ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log }, - { ALPROTO_POP3, NULL }, // protocol detection only - { ALPROTO_HTTP, NULL }, // signature protocol, not for app-layer logging - { ALPROTO_FAILED, NULL }, -#ifdef UNITTESTS - { ALPROTO_TEST, NULL }, -#endif /* UNITESTS */ -}; - -EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto) -{ - if (alproto < ALPROTO_MAX) { - BUG_ON(simple_json_applayer_loggers[alproto].proto != alproto); - return &simple_json_applayer_loggers[alproto]; - } - return NULL; -} diff --git a/src/output.h b/src/output.h index 6c477547e66f..074d20bf5bf2 100644 --- a/src/output.h +++ b/src/output.h @@ -192,8 +192,8 @@ void OutputClearActiveLoggers(void); typedef bool (*EveJsonSimpleTxLogFunc)(void *, struct JsonBuilder *); typedef struct EveJsonSimpleAppLayerLogger { - AppProto proto; EveJsonSimpleTxLogFunc LogTx; + const char *name; } EveJsonSimpleAppLayerLogger; EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto);