Skip to content

App-layer plugin preliminary work 5053 v2.4 #11426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
catenacyber wants to merge 4 commits into from
Closed

App-layer plugin preliminary work 5053 v2.4 #11426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e4c20d0
util/thash: decrease memuse if array was allocated
catenacyber Jul 4, 2024
61e8398
runmodes: use dynamic number of app-layer protos
catenacyber Jul 4, 2024
9c45c58
output/tx: use dynamic number of app-layer protos
catenacyber Jul 4, 2024
e2826d8
output: use dynamic number of app-layer protos
catenacyber Jul 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/output-tx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ typedef struct OutputTxLogger_ {
void (*ThreadExitPrintStats)(ThreadVars *, void *);
} OutputTxLogger;

static OutputTxLogger *list[ALPROTO_MAX] = { NULL };
static OutputTxLogger **list = NULL;

int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto,
TxLogger LogFunc,
Expand Down Expand Up @@ -668,6 +668,11 @@ static uint32_t OutputTxLoggerGetActiveCount(void)

void OutputTxLoggerRegister (void)
{
list = SCCalloc(ALPROTO_MAX, sizeof(OutputTxLogger *));
if (unlikely(list == NULL)) {
FatalError("Failed to allocate OutputTx list");
}

OutputRegisterRootLogger(OutputTxLogThreadInit, OutputTxLogThreadDeinit,
OutputTxLogExitPrintStats, OutputTxLog, OutputTxLoggerGetActiveCount);
}
Expand All @@ -683,4 +688,6 @@ void OutputTxShutdown(void)
}
list[alproto] = NULL;
}
SCFree(list);
list = NULL;
}
111 changes: 57 additions & 54 deletions src/output.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,8 @@ OutputModule *OutputGetModuleByConfName(const char *conf_name)
return NULL;
}

static EveJsonSimpleAppLayerLogger *simple_json_applayer_loggers;

/**
* \brief Deregister all modules. Useful for a memory clean exit.
*/
Expand All @@ -680,6 +682,8 @@ void OutputDeregisterAll(void)
TAILQ_REMOVE(&output_modules, module, entries);
SCFree(module);
}
SCFree(simple_json_applayer_loggers);
simple_json_applayer_loggers = NULL;
}

static int drop_loggers = 0;
Expand Down Expand Up @@ -895,11 +899,62 @@ void TmModuleLoggerRegister(void)
OutputRegisterLoggers();
}

EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto)
{
if (alproto < ALPROTO_MAX) {
return &simple_json_applayer_loggers[alproto];
}
return NULL;
}

static void RegisterSimpleJsonApplayerLogger(AppProto alproto, EveJsonSimpleTxLogFunc LogTx)
{
simple_json_applayer_loggers[alproto].LogTx = LogTx;
}

/**
* \brief Register all root loggers.
*/
void OutputRegisterRootLoggers(void)
{
simple_json_applayer_loggers = SCCalloc(ALPROTO_MAX, sizeof(EveJsonSimpleAppLayerLogger));
if (unlikely(simple_json_applayer_loggers == NULL)) {
FatalError("Failed to allocate simple_json_applayer_loggers");
}
// ALPROTO_HTTP1 special: uses some options flags
RegisterSimpleJsonApplayerLogger(ALPROTO_FTP, EveFTPLogCommand);
Copy link
Member

@victorjulien victorjulien Jul 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we move these into the respective parsers?

Copy link
Contributor Author

@catenacyber catenacyber Jul 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the end, we want to.

But this is not obvious now : when the respective parsers register themselves, we do not know yet ALPROTO_MAX

So, I would like to keep this for a later PR...

// ALPROTO_SMTP special: uses state
RegisterSimpleJsonApplayerLogger(ALPROTO_TLS, JsonTlsLogJSONExtended);
// no cast here but done in rust for SSHTransaction
RegisterSimpleJsonApplayerLogger(ALPROTO_SSH, rs_ssh_log_json);
// ALPROTO_SMB special: uses state
// ALPROTO_DCERPC special: uses state
RegisterSimpleJsonApplayerLogger(ALPROTO_DNS, AlertJsonDns);
// either need a cast here or in rust for ModbusTransaction, done here
RegisterSimpleJsonApplayerLogger(ALPROTO_MODBUS, (EveJsonSimpleTxLogFunc)rs_modbus_to_json);
RegisterSimpleJsonApplayerLogger(ALPROTO_ENIP, SCEnipLoggerLog);
RegisterSimpleJsonApplayerLogger(ALPROTO_DNP3, AlertJsonDnp3);
// ALPROTO_NFS special: uses state
RegisterSimpleJsonApplayerLogger(ALPROTO_FTPDATA, EveFTPDataAddMetadata);
RegisterSimpleJsonApplayerLogger(
ALPROTO_TFTP, (EveJsonSimpleTxLogFunc)rs_tftp_log_json_request);
// ALPROTO_IKE special: uses state
RegisterSimpleJsonApplayerLogger(
ALPROTO_KRB5, (EveJsonSimpleTxLogFunc)rs_krb5_log_json_response);
RegisterSimpleJsonApplayerLogger(ALPROTO_QUIC, rs_quic_to_json);
// ALPROTO_DHCP TODO missing
RegisterSimpleJsonApplayerLogger(
ALPROTO_SNMP, (EveJsonSimpleTxLogFunc)rs_snmp_log_json_response);
RegisterSimpleJsonApplayerLogger(ALPROTO_SIP, (EveJsonSimpleTxLogFunc)rs_sip_log_json);
RegisterSimpleJsonApplayerLogger(ALPROTO_RFB, rs_rfb_logger_log);
RegisterSimpleJsonApplayerLogger(ALPROTO_MQTT, JsonMQTTAddMetadata);
RegisterSimpleJsonApplayerLogger(ALPROTO_PGSQL, JsonPgsqlAddMetadata);
RegisterSimpleJsonApplayerLogger(ALPROTO_WEBSOCKET, rs_websocket_logger_log);
RegisterSimpleJsonApplayerLogger(ALPROTO_TEMPLATE, rs_template_logger_log);
RegisterSimpleJsonApplayerLogger(ALPROTO_RDP, (EveJsonSimpleTxLogFunc)rs_rdp_to_json);
RegisterSimpleJsonApplayerLogger(ALPROTO_HTTP2, rs_http2_log_json);
RegisterSimpleJsonApplayerLogger(ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log);

OutputPacketLoggerRegister();
OutputFiledataLoggerRegister();
OutputFileLoggerRegister();
Expand All @@ -917,7 +972,7 @@ static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p,
}

const char *name;
switch (al->proto) {
switch (f->proto) {
Copy link
Member

@victorjulien victorjulien Jul 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

instead of doing this at runtime, can we do it at init time and store the result in EveJsonSimpleAppLayerLogger?

Copy link
Contributor Author

@catenacyber catenacyber Jul 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice idea :-)

case ALPROTO_HTTP2:
// special case
name = "http";
Expand All @@ -931,7 +986,7 @@ static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p,
name = "bittorrent_dht";
break;
default:
name = AppProtoToString(al->proto);
name = AppProtoToString(f->proto);
}
JsonBuilder *js = CreateEveHeader(p, dir, name, NULL, thread->ctx);
if (unlikely(js == NULL)) {
Expand Down Expand Up @@ -1115,55 +1170,3 @@ void OutputRegisterLoggers(void)
/* ARP JSON logger */
JsonArpLogRegister();
}

static EveJsonSimpleAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = {
{ ALPROTO_UNKNOWN, NULL },
{ ALPROTO_HTTP1, NULL }, // special: uses some options flags
{ ALPROTO_FTP, EveFTPLogCommand },
{ ALPROTO_SMTP, NULL }, // special: uses state
{ ALPROTO_TLS, JsonTlsLogJSONExtended },
{ ALPROTO_SSH, rs_ssh_log_json },
{ ALPROTO_IMAP, NULL }, // protocol detection only
{ ALPROTO_JABBER, NULL }, // no parser, no logging
{ ALPROTO_SMB, NULL }, // special: uses state
{ ALPROTO_DCERPC, NULL }, // special: uses state
{ ALPROTO_IRC, NULL }, // no parser, no logging
{ ALPROTO_DNS, AlertJsonDns },
{ ALPROTO_MODBUS, (EveJsonSimpleTxLogFunc)rs_modbus_to_json },
{ ALPROTO_ENIP, SCEnipLoggerLog },
{ ALPROTO_DNP3, AlertJsonDnp3 },
{ ALPROTO_NFS, NULL }, // special: uses state
{ ALPROTO_NTP, NULL }, // no logging
{ ALPROTO_FTPDATA, EveFTPDataAddMetadata },
{ ALPROTO_TFTP, (EveJsonSimpleTxLogFunc)rs_tftp_log_json_request },
{ ALPROTO_IKE, NULL }, // special: uses state
{ ALPROTO_KRB5, (EveJsonSimpleTxLogFunc)rs_krb5_log_json_response },
{ ALPROTO_QUIC, rs_quic_to_json },
{ ALPROTO_DHCP, NULL }, // TODO missing
{ ALPROTO_SNMP, (EveJsonSimpleTxLogFunc)rs_snmp_log_json_response },
{ ALPROTO_SIP, (EveJsonSimpleTxLogFunc)rs_sip_log_json },
{ ALPROTO_RFB, rs_rfb_logger_log },
{ ALPROTO_MQTT, JsonMQTTAddMetadata },
{ ALPROTO_PGSQL, JsonPgsqlAddMetadata },
{ ALPROTO_TELNET, NULL }, // no logging
{ ALPROTO_WEBSOCKET, rs_websocket_logger_log },
{ ALPROTO_TEMPLATE, rs_template_logger_log },
{ ALPROTO_RDP, (EveJsonSimpleTxLogFunc)rs_rdp_to_json },
{ ALPROTO_HTTP2, rs_http2_log_json },
{ ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log },
{ ALPROTO_POP3, NULL }, // protocol detection only
{ ALPROTO_HTTP, NULL }, // signature protocol, not for app-layer logging
{ ALPROTO_FAILED, NULL },
#ifdef UNITTESTS
{ ALPROTO_TEST, NULL },
#endif /* UNITESTS */
};

EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto)
{
if (alproto < ALPROTO_MAX) {
BUG_ON(simple_json_applayer_loggers[alproto].proto != alproto);
return &simple_json_applayer_loggers[alproto];
}
return NULL;
}
1 change: 0 additions & 1 deletion src/output.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,6 @@ void OutputClearActiveLoggers(void);
typedef bool (*EveJsonSimpleTxLogFunc)(void *, struct JsonBuilder *);

typedef struct EveJsonSimpleAppLayerLogger {
AppProto proto;
EveJsonSimpleTxLogFunc LogTx;
} EveJsonSimpleAppLayerLogger;

Expand Down
10 changes: 7 additions & 3 deletions src/runmodes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ static void RunOutputFreeList(void)

static int file_logger_count = 0;
static int filedata_logger_count = 0;
static LoggerId logger_bits[ALPROTO_MAX];
static LoggerId *logger_bits = NULL;

int RunModeOutputFiledataEnabled(void)
{
Expand Down Expand Up @@ -592,6 +592,7 @@ void RunModeShutDown(void)

OutputClearActiveLoggers();

SCFree(logger_bits);
/* Reset logger counts. */
file_logger_count = 0;
filedata_logger_count = 0;
Expand Down Expand Up @@ -790,8 +791,11 @@ void RunModeInitializeOutputs(void)
char tls_log_enabled = 0;
char tls_store_present = 0;

memset(&logger_bits, 0, sizeof(logger_bits));

// ALPROTO_MAX is set to its final value
logger_bits = SCCalloc(ALPROTO_MAX, sizeof(LoggerId));
if (unlikely(logger_bits == NULL)) {
FatalError("Failed to allocate logger_bits");
}
TAILQ_FOREACH(output, &outputs->head, next) {

output_config = ConfNodeLookupChild(output, output->val);
Expand Down
2 changes: 1 addition & 1 deletion src/util-thash.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,8 +378,8 @@ void THashShutdown(THashTableContext *ctx)
}
SCFreeAligned(ctx->array);
ctx->array = NULL;
(void)SC_ATOMIC_SUB(ctx->memuse, ctx->config.hash_size * sizeof(THashHashRow));
}
(void) SC_ATOMIC_SUB(ctx->memuse, ctx->config.hash_size * sizeof(THashHashRow));
THashDataQueueDestroy(&ctx->spare_q);
DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(ctx->memuse) != 0);
SCFree(ctx);
Expand Down