Skip to content

Applayer plugin 5053 v2.10 #11572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
catenacyber wants to merge 3 commits into from
Closed

Applayer plugin 5053 v2.10 #11572

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
35ad540
runmodes: use dynamic number of app-layer protos
catenacyber Jul 4, 2024
4a42138
output: use dynamic number of app-layer protos
catenacyber Jul 4, 2024
721b442
output/tx: use dynamic number of app-layer protos
catenacyber Jul 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion src/output-tx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ typedef struct OutputTxLogger_ {
void (*ThreadExitPrintStats)(ThreadVars *, void *);
} OutputTxLogger;

static OutputTxLogger *list[ALPROTO_MAX] = { NULL };
static OutputTxLogger **list = NULL;

int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto,
TxLogger LogFunc,
Expand All @@ -71,6 +71,14 @@ int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto,
ThreadDeinitFunc ThreadDeinit,
void (*ThreadExitPrintStats)(ThreadVars *, void *))
{
if (list == NULL) {
list = SCCalloc(ALPROTO_MAX, sizeof(OutputTxLogger *));
if (unlikely(list == NULL)) {
SCLogError("Failed to allocate OutputTx list");
return -1;
}
}

if (alproto != ALPROTO_UNKNOWN && !(AppLayerParserIsEnabled(alproto))) {
SCLogDebug(
"%s logger not enabled: protocol %s is disabled", name, AppProtoToString(alproto));
Expand Down Expand Up @@ -666,12 +674,21 @@ static uint32_t OutputTxLoggerGetActiveCount(void)

void OutputTxLoggerRegister (void)
{
BUG_ON(list);
list = SCCalloc(ALPROTO_MAX, sizeof(OutputTxLogger *));
if (unlikely(list == NULL)) {
FatalError("Failed to allocate OutputTx list");
}
OutputRegisterRootLogger(OutputTxLogThreadInit, OutputTxLogThreadDeinit,
OutputTxLogExitPrintStats, OutputTxLog, OutputTxLoggerGetActiveCount);
}

void OutputTxShutdown(void)
{
// called in different places because of unix socket mode, and engine-analysis mode
if (list == NULL) {
return;
}
for (AppProto alproto = 0; alproto < ALPROTO_MAX; alproto++) {
OutputTxLogger *logger = list[alproto];
while (logger) {
Expand All @@ -681,4 +698,6 @@ void OutputTxShutdown(void)
}
list[alproto] = NULL;
}
SCFree(list);
list = NULL;
}
141 changes: 69 additions & 72 deletions src/output.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -669,6 +669,8 @@ OutputModule *OutputGetModuleByConfName(const char *conf_name)
return NULL;
}

static EveJsonSimpleAppLayerLogger *simple_json_applayer_loggers;

/**
* \brief Deregister all modules. Useful for a memory clean exit.
*/
Expand All @@ -680,6 +682,8 @@ void OutputDeregisterAll(void)
TAILQ_REMOVE(&output_modules, module, entries);
SCFree(module);
}
SCFree(simple_json_applayer_loggers);
simple_json_applayer_loggers = NULL;
}

static int drop_loggers = 0;
Expand Down Expand Up @@ -895,11 +899,75 @@ void TmModuleLoggerRegister(void)
OutputRegisterLoggers();
}

EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto)
{
if (alproto < ALPROTO_MAX) {
return &simple_json_applayer_loggers[alproto];
}
return NULL;
}

static void RegisterSimpleJsonApplayerLogger(
AppProto alproto, EveJsonSimpleTxLogFunc LogTx, const char *name)
{
simple_json_applayer_loggers[alproto].LogTx = LogTx;
if (name) {
simple_json_applayer_loggers[alproto].name = name;
} else {
simple_json_applayer_loggers[alproto].name = AppProtoToString(alproto);
}
}

/**
* \brief Register all root loggers.
*/
void OutputRegisterRootLoggers(void)
{
simple_json_applayer_loggers = SCCalloc(ALPROTO_MAX, sizeof(EveJsonSimpleAppLayerLogger));
if (unlikely(simple_json_applayer_loggers == NULL)) {
FatalError("Failed to allocate simple_json_applayer_loggers");
}
// ALPROTO_HTTP1 special: uses some options flags
RegisterSimpleJsonApplayerLogger(ALPROTO_FTP, EveFTPLogCommand, NULL);
// ALPROTO_SMTP special: uses state
RegisterSimpleJsonApplayerLogger(ALPROTO_TLS, JsonTlsLogJSONExtended, NULL);
// no cast here but done in rust for SSHTransaction
RegisterSimpleJsonApplayerLogger(ALPROTO_SSH, rs_ssh_log_json, NULL);
// ALPROTO_SMB special: uses state
// ALPROTO_DCERPC special: uses state
RegisterSimpleJsonApplayerLogger(ALPROTO_DNS, AlertJsonDns, NULL);
// either need a cast here or in rust for ModbusTransaction, done here
RegisterSimpleJsonApplayerLogger(
ALPROTO_MODBUS, (EveJsonSimpleTxLogFunc)rs_modbus_to_json, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_ENIP, SCEnipLoggerLog, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_DNP3, AlertJsonDnp3, NULL);
// ALPROTO_NFS special: uses state
// underscore instead of dash for ftp_data
RegisterSimpleJsonApplayerLogger(ALPROTO_FTPDATA, EveFTPDataAddMetadata, "ftp_data");
RegisterSimpleJsonApplayerLogger(
ALPROTO_TFTP, (EveJsonSimpleTxLogFunc)rs_tftp_log_json_request, NULL);
// ALPROTO_IKE special: uses state
RegisterSimpleJsonApplayerLogger(
ALPROTO_KRB5, (EveJsonSimpleTxLogFunc)rs_krb5_log_json_response, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_QUIC, rs_quic_to_json, NULL);
// ALPROTO_DHCP TODO missing
RegisterSimpleJsonApplayerLogger(
ALPROTO_SNMP, (EveJsonSimpleTxLogFunc)rs_snmp_log_json_response, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_SIP, (EveJsonSimpleTxLogFunc)rs_sip_log_json, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_RFB, rs_rfb_logger_log, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_MQTT, JsonMQTTAddMetadata, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_PGSQL, JsonPgsqlAddMetadata, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_WEBSOCKET, rs_websocket_logger_log, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_LDAP, rs_ldap_logger_log, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_DOH2, AlertJsonDoh2, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_TEMPLATE, rs_template_logger_log, NULL);
RegisterSimpleJsonApplayerLogger(ALPROTO_RDP, (EveJsonSimpleTxLogFunc)rs_rdp_to_json, NULL);
// special case : http2 is logged in http object
RegisterSimpleJsonApplayerLogger(ALPROTO_HTTP2, rs_http2_log_json, "http");
// underscore instead of dash for bittorrent_dht
RegisterSimpleJsonApplayerLogger(
ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log, "bittorrent_dht");

OutputPacketLoggerRegister();
OutputFiledataLoggerRegister();
OutputFileLoggerRegister();
Expand All @@ -916,24 +984,7 @@ static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p,
return TM_ECODE_FAILED;
}

const char *name;
switch (al->proto) {
case ALPROTO_HTTP2:
// special case
name = "http";
break;
case ALPROTO_FTPDATA:
// underscore instead of dash
name = "ftp_data";
break;
case ALPROTO_BITTORRENT_DHT:
// underscore instead of dash
name = "bittorrent_dht";
break;
default:
name = AppProtoToString(al->proto);
}
JsonBuilder *js = CreateEveHeader(p, dir, name, NULL, thread->ctx);
JsonBuilder *js = CreateEveHeader(p, dir, al->name, NULL, thread->ctx);
if (unlikely(js == NULL)) {
return TM_ECODE_FAILED;
}
Expand Down Expand Up @@ -1121,57 +1172,3 @@ void OutputRegisterLoggers(void)
/* ARP JSON logger */
JsonArpLogRegister();
}

static EveJsonSimpleAppLayerLogger simple_json_applayer_loggers[ALPROTO_MAX] = {
{ ALPROTO_UNKNOWN, NULL },
{ ALPROTO_HTTP1, NULL }, // special: uses some options flags
{ ALPROTO_FTP, EveFTPLogCommand },
{ ALPROTO_SMTP, NULL }, // special: uses state
{ ALPROTO_TLS, JsonTlsLogJSONExtended },
{ ALPROTO_SSH, rs_ssh_log_json },
{ ALPROTO_IMAP, NULL }, // protocol detection only
{ ALPROTO_JABBER, NULL }, // no parser, no logging
{ ALPROTO_SMB, NULL }, // special: uses state
{ ALPROTO_DCERPC, NULL }, // special: uses state
{ ALPROTO_IRC, NULL }, // no parser, no logging
{ ALPROTO_DNS, AlertJsonDns },
{ ALPROTO_MODBUS, (EveJsonSimpleTxLogFunc)rs_modbus_to_json },
{ ALPROTO_ENIP, SCEnipLoggerLog },
{ ALPROTO_DNP3, AlertJsonDnp3 },
{ ALPROTO_NFS, NULL }, // special: uses state
{ ALPROTO_NTP, NULL }, // no logging
{ ALPROTO_FTPDATA, EveFTPDataAddMetadata },
{ ALPROTO_TFTP, (EveJsonSimpleTxLogFunc)rs_tftp_log_json_request },
{ ALPROTO_IKE, NULL }, // special: uses state
{ ALPROTO_KRB5, (EveJsonSimpleTxLogFunc)rs_krb5_log_json_response },
{ ALPROTO_QUIC, rs_quic_to_json },
{ ALPROTO_DHCP, NULL }, // TODO missing
{ ALPROTO_SNMP, (EveJsonSimpleTxLogFunc)rs_snmp_log_json_response },
{ ALPROTO_SIP, (EveJsonSimpleTxLogFunc)rs_sip_log_json },
{ ALPROTO_RFB, rs_rfb_logger_log },
{ ALPROTO_MQTT, JsonMQTTAddMetadata },
{ ALPROTO_PGSQL, JsonPgsqlAddMetadata },
{ ALPROTO_TELNET, NULL }, // no logging
{ ALPROTO_WEBSOCKET, rs_websocket_logger_log },
{ ALPROTO_LDAP, rs_ldap_logger_log },
{ ALPROTO_DOH2, AlertJsonDoh2 },
{ ALPROTO_TEMPLATE, rs_template_logger_log },
{ ALPROTO_RDP, (EveJsonSimpleTxLogFunc)rs_rdp_to_json },
{ ALPROTO_HTTP2, rs_http2_log_json },
{ ALPROTO_BITTORRENT_DHT, rs_bittorrent_dht_logger_log },
{ ALPROTO_POP3, NULL }, // protocol detection only
{ ALPROTO_HTTP, NULL }, // signature protocol, not for app-layer logging
{ ALPROTO_FAILED, NULL },
#ifdef UNITTESTS
{ ALPROTO_TEST, NULL },
#endif /* UNITESTS */
};

EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto)
{
if (alproto < ALPROTO_MAX) {
BUG_ON(simple_json_applayer_loggers[alproto].proto != alproto);
return &simple_json_applayer_loggers[alproto];
}
return NULL;
}
2 changes: 1 addition & 1 deletion src/output.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,8 @@ void OutputClearActiveLoggers(void);
typedef bool (*EveJsonSimpleTxLogFunc)(void *, struct JsonBuilder *);

typedef struct EveJsonSimpleAppLayerLogger {
AppProto proto;
EveJsonSimpleTxLogFunc LogTx;
const char *name;
} EveJsonSimpleAppLayerLogger;

EveJsonSimpleAppLayerLogger *SCEveJsonSimpleGetLogger(AppProto alproto);
Expand Down
25 changes: 12 additions & 13 deletions src/runmodes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -525,7 +525,6 @@ static void RunOutputFreeList(void)

static int file_logger_count = 0;
static int filedata_logger_count = 0;
static LoggerId logger_bits[ALPROTO_MAX];

int RunModeOutputFiledataEnabled(void)
{
Expand Down Expand Up @@ -596,7 +595,8 @@ static void AddOutputToFreeList(OutputModule *module, OutputCtx *output_ctx)
}

/** \brief Turn output into thread module */
static void SetupOutput(const char *name, OutputModule *module, OutputCtx *output_ctx)
static void SetupOutput(
const char *name, OutputModule *module, OutputCtx *output_ctx, LoggerId *logger_bits)
{
/* flow logger doesn't run in the packet path */
if (module->FlowLogFunc) {
Expand Down Expand Up @@ -657,7 +657,7 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu
}
}

static void RunModeInitializeEveOutput(ConfNode *conf, OutputCtx *parent_ctx)
static void RunModeInitializeEveOutput(ConfNode *conf, OutputCtx *parent_ctx, LoggerId *logger_bits)
{
ConfNode *types = ConfNodeLookupChild(conf, "types");
SCLogDebug("types %p", types);
Expand Down Expand Up @@ -710,8 +710,7 @@ static void RunModeInitializeEveOutput(ConfNode *conf, OutputCtx *parent_ctx)
}

AddOutputToFreeList(sub_module, result.ctx);
SetupOutput(sub_module->name, sub_module,
result.ctx);
SetupOutput(sub_module->name, sub_module, result.ctx, logger_bits);
}
}

Expand All @@ -724,7 +723,7 @@ static void RunModeInitializeEveOutput(ConfNode *conf, OutputCtx *parent_ctx)
}
}

static void RunModeInitializeLuaOutput(ConfNode *conf, OutputCtx *parent_ctx)
static void RunModeInitializeLuaOutput(ConfNode *conf, OutputCtx *parent_ctx, LoggerId *logger_bits)
{
OutputModule *lua_module = OutputGetModuleByConfName("lua");
BUG_ON(lua_module == NULL);
Expand Down Expand Up @@ -752,7 +751,7 @@ static void RunModeInitializeLuaOutput(ConfNode *conf, OutputCtx *parent_ctx)
}

AddOutputToFreeList(m, result.ctx);
SetupOutput(m->name, m, result.ctx);
SetupOutput(m->name, m, result.ctx, logger_bits);
}
}

Expand All @@ -775,8 +774,8 @@ void RunModeInitializeOutputs(void)
char tls_log_enabled = 0;
char tls_store_present = 0;

memset(&logger_bits, 0, sizeof(logger_bits));

// ALPROTO_MAX is set to its final value
LoggerId logger_bits[ALPROTO_MAX] = { 0 };
TAILQ_FOREACH(output, &outputs->head, next) {

output_config = ConfNodeLookupChild(output, output->val);
Expand Down Expand Up @@ -842,7 +841,7 @@ void RunModeInitializeOutputs(void)

// TODO if module == parent, find it's children
if (strcmp(output->val, "eve-log") == 0) {
RunModeInitializeEveOutput(output_config, output_ctx);
RunModeInitializeEveOutput(output_config, output_ctx, logger_bits);

/* add 'eve-log' to free list as it's the owner of the
* main output ctx from which the sub-modules share the
Expand All @@ -852,11 +851,11 @@ void RunModeInitializeOutputs(void)
SCLogDebug("handle lua");
if (output_ctx == NULL)
continue;
RunModeInitializeLuaOutput(output_config, output_ctx);
RunModeInitializeLuaOutput(output_config, output_ctx, logger_bits);
AddOutputToFreeList(module, output_ctx);
} else {
AddOutputToFreeList(module, output_ctx);
SetupOutput(module->name, module, output_ctx);
SetupOutput(module->name, module, output_ctx, logger_bits);
}
}
if (count == 0) {
Expand Down Expand Up @@ -895,7 +894,7 @@ void RunModeInitializeOutputs(void)
}

AddOutputToFreeList(module, output_ctx);
SetupOutput(module->name, module, output_ctx);
SetupOutput(module->name, module, output_ctx, logger_bits);
}
}
}
Expand Down
1 change: 1 addition & 0 deletions src/suricata.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,7 @@ void GlobalsDestroy(void)
AppLayerDeSetup();
DatasetsSave();
DatasetsDestroy();
OutputTxShutdown();
TagDestroyCtx();

LiveDeviceListClean();
Expand Down