Output alert applayer v13.3

rust/src/applayertemplate/logger.rs

@@ -20,12 +20,14 @@ use crate::jsonbuilder::{JsonBuilder, JsonError};
 use std;
 
 fn log_template(tx: &TemplateTransaction, js: &mut JsonBuilder) -> Result<(), JsonError> {
+    js.open_object("template")?;
     if let Some(ref request) = tx.request {
         js.set_string("request", request)?;
     }
     if let Some(ref response) = tx.response {
         js.set_string("response", response)?;
     }
+    js.close()?;
     Ok(())
 }
 

rust/src/bittorrent_dht/logger.rs

@@ -48,6 +48,7 @@ fn print_ip_addr(addr: &[u8]) -> std::string::String {
 fn log_bittorrent_dht(
     tx: &BitTorrentDHTTransaction, js: &mut JsonBuilder,
 ) -> Result<(), JsonError> {
+    js.open_object("bittorrent_dht")?;
     js.set_hex("transaction_id", &tx.transaction_id)?;
     if let Some(client_version) = &tx.client_version {
         js.set_hex("client_version", client_version)?;
@@ -125,6 +126,7 @@ fn log_bittorrent_dht(
         }
         js.close()?;
     };
+    js.close()?;
     Ok(())
 }
 

rust/src/core.rs

@@ -253,6 +253,13 @@ pub extern "C" fn rs_init(context: &'static SuricataContext)
     init_ffi(context);
 }
 
+#[no_mangle]
+pub extern "C" fn rs_update_alproto_failed(alproto: AppProto) {
+    unsafe {
+        ALPROTO_FAILED = alproto;
+    }
+}
+
 /// DetectEngineStateFree wrapper.
 pub fn sc_detect_engine_state_free(state: *mut DetectEngineState)
 {

rust/src/http2/logger.rs

@@ -192,6 +192,7 @@ fn log_http2_frames(frames: &[HTTP2Frame], js: &mut JsonBuilder) -> Result<bool,
 }
 
 fn log_http2(tx: &HTTP2Transaction, js: &mut JsonBuilder) -> Result<bool, JsonError> {
+    js.open_object("http")?;
     js.set_string("version", "2")?;
 
     let mut common: HashMap<HeaderName, &Vec<u8>> = HashMap::new();
@@ -261,8 +262,8 @@ fn log_http2(tx: &HTTP2Transaction, js: &mut JsonBuilder) -> Result<bool, JsonEr
     let has_response = log_http2_frames(&tx.frames_tc, js)?;
     js.close()?;
 
-    // Close http2.
-    js.close()?;
+    js.close()?; // http2
+    js.close()?; // http
 
     return Ok(has_request || has_response || has_headers);
 }

rust/src/krb/log.rs

@@ -22,6 +22,7 @@ use crate::krb::krb5::{KRB5Transaction,test_weak_encryption};
 
 fn krb5_log_response(jsb: &mut JsonBuilder, tx: &mut KRB5Transaction) -> Result<(), JsonError>
 {
+    jsb.open_object("krb5")?;
     match tx.error_code {
         Some(c) => {
             jsb.set_string("msg_type", &format!("{:?}", tx.msg_type))?;
@@ -63,12 +64,13 @@ fn krb5_log_response(jsb: &mut JsonBuilder, tx: &mut KRB5Transaction) -> Result<
         jsb.set_string("ticket_encryption", &refs)?;
         jsb.set_bool("ticket_weak_encryption", test_weak_encryption(x))?;
     }
+    jsb.close()?;
 
     return Ok(());
 }
 
 #[no_mangle]
-pub extern "C" fn rs_krb5_log_json_response(jsb: &mut JsonBuilder, tx: &mut KRB5Transaction) -> bool
+pub extern "C" fn rs_krb5_log_json_response(tx: &mut KRB5Transaction, jsb: &mut JsonBuilder) -> bool
 {
     krb5_log_response(jsb, tx).is_ok()
 }

rust/src/mqtt/detect.rs

@@ -168,7 +168,7 @@ pub unsafe extern "C" fn rs_mqtt_tx_get_connect_username(
 
 #[no_mangle]
 pub unsafe extern "C" fn rs_mqtt_tx_get_connect_password(
-    tx: &MQTTTransaction, buffer: *mut *const u8, buffer_len: *mut u32,
+    tx: &MQTTTransaction, _flow_flags: u8, buffer: *mut *const u8, buffer_len: *mut u32,
 ) -> u8 {
     for msg in tx.msg.iter() {
         if let MQTTOperation::CONNECT(ref cv) = msg.op {

rust/src/quic/logger.rs

@@ -88,7 +88,7 @@ fn quic_tls_extension_name(e: u16) -> Option<String> {
     }
 }
 
-fn log_template(tx: &QuicTransaction, js: &mut JsonBuilder) -> Result<(), JsonError> {
+fn log_quic(tx: &QuicTransaction, js: &mut JsonBuilder) -> Result<(), JsonError> {
     js.open_object("quic")?;
     if tx.header.ty != QuicType::Short {
         js.set_string("version", String::from(tx.header.version).as_str())?;
@@ -153,5 +153,5 @@ pub unsafe extern "C" fn rs_quic_to_json(
     tx: *mut std::os::raw::c_void, js: &mut JsonBuilder,
 ) -> bool {
     let tx = cast_pointer!(tx, QuicTransaction);
-    log_template(tx, js).is_ok()
+    log_quic(tx, js).is_ok()
 }

rust/src/snmp/log.rs

@@ -39,6 +39,7 @@ fn str_of_pdu_type(t:&PduType) -> Cow<str> {
 
 fn snmp_log_response(jsb: &mut JsonBuilder, tx: &mut SNMPTransaction) -> Result<(), JsonError>
 {
+    jsb.open_object("snmp")?;
     jsb.set_uint("version", tx.version as u64)?;
     if tx.encrypted {
         jsb.set_string("pdu_type", "encrypted")?;
@@ -71,11 +72,12 @@ fn snmp_log_response(jsb: &mut JsonBuilder, tx: &mut SNMPTransaction) -> Result<
         }
     }
 
+    jsb.close()?;
     return Ok(());
 }
 
 #[no_mangle]
-pub extern "C" fn rs_snmp_log_json_response(jsb: &mut JsonBuilder, tx: &mut SNMPTransaction) -> bool
+pub extern "C" fn rs_snmp_log_json_response(tx: &mut SNMPTransaction, jsb: &mut JsonBuilder) -> bool
 {
     snmp_log_response(jsb, tx).is_ok()
 }

rust/src/ssh/logger.rs

@@ -19,6 +19,7 @@ use super::ssh::SSHTransaction;
 use crate::jsonbuilder::{JsonBuilder, JsonError};
 
 fn log_ssh(tx: &SSHTransaction, js: &mut JsonBuilder) -> Result<bool, JsonError> {
+    js.open_object("ssh")?;
     if tx.cli_hdr.protover.is_empty() && tx.srv_hdr.protover.is_empty() {
         return Ok(false);
     }
@@ -58,6 +59,7 @@ fn log_ssh(tx: &SSHTransaction, js: &mut JsonBuilder) -> Result<bool, JsonError>
         }
         js.close()?;
     }
+    js.close()?;
     return Ok(true);
 }
 

rust/src/tftp/log.rs

@@ -24,13 +24,15 @@ fn tftp_log_request(tx: &mut TFTPTransaction,
                     jb: &mut JsonBuilder)
                     -> Result<(), JsonError>
 {
+    jb.open_object("tftp")?;
     match tx.opcode {
         1 => jb.set_string("packet", "read")?,
         2 => jb.set_string("packet", "write")?,
         _ => jb.set_string("packet", "error")?
     };
     jb.set_string("file", tx.filename.as_str())?;
     jb.set_string("mode", tx.mode.as_str())?;
+    jb.close()?;
     Ok(())
 }
 

scripts/setup-app-layer.py

@@ -200,39 +200,34 @@ def logger_patch_output_c(proto):
     output = io.StringIO()
     inlines = open(filename).readlines()
     for i, line in enumerate(inlines):
-        if line.find("output-json-template.h") > -1:
-            output.write(line.replace("template", proto.lower()))
         if line.find("/* Template JSON logger.") > -1:
             output.write(inlines[i].replace("Template", proto))
             output.write(inlines[i+1].replace("Template", proto))
+            output.write(inlines[i+2].replace("TEMPLATE", proto.upper()).replace(
+                    "template", proto.lower()).replace("Template", proto))
+            output.write(inlines[i+3])
+        if line.find("rs_template_logger_log") > -1:
+            output.write(inlines[i].replace("TEMPLATE", proto.upper()).replace(
+                    "template", proto.lower()))
+        if line.find("OutputTemplateLogInitSub(") > -1:
+            output.write(inlines[i].replace("Template", proto))
+            output.write(inlines[i+1])
+            output.write(inlines[i+2].replace("TEMPLATE", proto.upper()))
+            output.write(inlines[i+3])
+            output.write(inlines[i+4])
         output.write(line)
     open(filename, "w").write(output.getvalue())
 
 def logger_copy_templates(proto):
     lower = proto.lower()
 
     pairs = (
-        ("src/output-json-template.h",
-         "src/output-json-%s.h" % (lower)),
-        ("src/output-json-template.c",
-         "src/output-json-%s.c" % (lower)),
         ("rust/src/applayertemplate/logger.rs",
          "rust/src/applayer%s/logger.rs" % (lower)),
     )
 
     common_copy_templates(proto, pairs)
 
-def logger_patch_makefile_am(protoname):
-    filename = "src/Makefile.am"
-    print("Patching %s." % (filename))
-    output = io.StringIO()
-    with open(filename) as infile:
-        for line in infile:
-            if line.lstrip().startswith("output-json-template."):
-                output.write(line.replace("template", protoname.lower()))
-            output.write(line)
-    open(filename, "w").write(output.getvalue())
-
 
 def detect_copy_templates(proto, buffername):
     lower = proto.lower()
@@ -394,7 +389,6 @@ def main():
             raise SetupError("no app-layer parser exists for %s" % (proto))
         logger_copy_templates(proto)
         patch_rust_applayer_mod_rs(proto)
-        logger_patch_makefile_am(proto)
         logger_patch_output_c(proto)
         logger_patch_suricata_yaml_in(proto)
 

src/Makefile.am

@@ -137,6 +137,7 @@ noinst_HEADERS = \
 	detect-engine-file.h \
 	detect-engine-frame.h \
 	detect-engine.h \
+	detect-engine-helper.h \
 	detect-engine-iponly.h \
 	detect-engine-loader.h \
 	detect-engine-mpm.h \
@@ -392,7 +393,6 @@ noinst_HEADERS = \
 	output.h \
 	output-json-alert.h \
 	output-json-anomaly.h \
-	output-json-bittorrent-dht.h \
 	output-json-dcerpc.h \
 	output-json-dhcp.h \
 	output-json-dnp3.h \
@@ -405,27 +405,16 @@ noinst_HEADERS = \
 	output-json-frame.h \
 	output-json-ftp.h \
 	output-json.h \
-	output-json-http2.h \
 	output-json-http.h \
 	output-json-ike.h \
-	output-json-krb5.h \
 	output-json-metadata.h \
-	output-json-modbus.h \
-	output-json-quic.h \
 	output-json-mqtt.h \
 	output-json-netflow.h \
 	output-json-nfs.h \
 	output-json-pgsql.h \
-	output-json-rdp.h \
-	output-json-rfb.h \
-	output-json-sip.h \
 	output-json-smb.h \
 	output-json-smtp.h \
-	output-json-snmp.h \
-	output-json-ssh.h \
 	output-json-stats.h \
-	output-json-template.h \
-	output-json-tftp.h \
 	output-json-tls.h \
 	output-eve-syslog.h \
 	output-lua.h \
@@ -748,6 +737,7 @@ libsuricata_c_a_SOURCES = \
 	detect-engine-event.c \
 	detect-engine-file.c \
 	detect-engine-frame.c \
+	detect-engine-helper.c \
 	detect-engine-iponly.c \
 	detect-engine-loader.c \
 	detect-engine-mpm.c \
@@ -1000,7 +990,6 @@ libsuricata_c_a_SOURCES = \
 	output-flow.c \
 	output-json-alert.c \
 	output-json-anomaly.c \
-	output-json-bittorrent-dht.c \
 	output-json.c \
 	output-json-common.c \
 	output-json-dcerpc.c \
@@ -1014,27 +1003,16 @@ libsuricata_c_a_SOURCES = \
 	output-json-flow.c \
 	output-json-frame.c \
 	output-json-ftp.c \
-	output-json-http2.c \
 	output-json-http.c \
 	output-json-ike.c \
-	output-json-krb5.c \
 	output-json-metadata.c \
-	output-json-modbus.c \
-	output-json-quic.c \
 	output-json-mqtt.c \
 	output-json-netflow.c \
 	output-json-nfs.c \
 	output-json-pgsql.c \
-	output-json-rdp.c \
-	output-json-rfb.c \
-	output-json-sip.c \
 	output-json-smb.c \
 	output-json-smtp.c \
-	output-json-snmp.c \
-	output-json-ssh.c \
 	output-json-stats.c \
-	output-json-template.c \
-	output-json-tftp.c \
 	output-json-tls.c \
 	output-eve-syslog.c \
 	output-lua.c \

src/app-layer-detect-proto.c

@@ -489,8 +489,8 @@ static inline AppProto PPGetProto(const AppLayerProtoDetectProbingParserElement
         if (AppProtoIsValid(alproto)) {
             SCReturnUInt(alproto);
         }
-        if (alproto == ALPROTO_FAILED ||
-            (pe->max_depth != 0 && buflen > pe->max_depth)) {
+        if (alproto == ALPROTO_FAILED || alproto == ALPROTO_INVALID ||
+                (pe->max_depth != 0 && buflen > pe->max_depth)) {
             alproto_masks[0] |= pe->alproto_mask;
         }
         pe = pe->next;

src/app-layer-ftp.c

@@ -1405,13 +1405,10 @@ uint16_t JsonGetNextLineFromBuffer(const char *buffer, const uint16_t len)
     return c == NULL ? len : (uint16_t)(c - buffer + 1);
 }
 
-void EveFTPDataAddMetadata(const Flow *f, JsonBuilder *jb)
+bool EveFTPDataAddMetadata(void *vtx, JsonBuilder *jb)
 {
-    const FtpDataState *ftp_state = NULL;
-    if (f->alstate == NULL)
-        return;
-
-    ftp_state = (FtpDataState *)f->alstate;
+    const FtpDataState *ftp_state = (FtpDataState *)vtx;
+    jb_open_object(jb, "ftp_data");
 
     if (ftp_state->file_name) {
         jb_set_string_from_bytes(jb, "filename", ftp_state->file_name, ftp_state->file_len);
@@ -1426,6 +1423,8 @@ void EveFTPDataAddMetadata(const Flow *f, JsonBuilder *jb)
         default:
             break;
     }
+    jb_close(jb);
+    return true;
 }
 
 /**

src/app-layer-ftp.h

@@ -190,7 +190,7 @@ uint64_t FTPMemuseGlobalCounter(void);
 uint64_t FTPMemcapGlobalCounter(void);
 
 uint16_t JsonGetNextLineFromBuffer(const char *buffer, const uint16_t len);
-void EveFTPDataAddMetadata(const Flow *f, JsonBuilder *jb);
+bool EveFTPDataAddMetadata(void *vtx, JsonBuilder *jb);
 
 #endif /* __APP_LAYER_FTP_H__ */
 

src/app-layer-parser.c

@@ -63,6 +63,10 @@
 #include "app-layer-rdp.h"
 #include "app-layer-http2.h"
 
+#ifdef ALPROTO_DYNAMIC_NB
+#include "util-plugin.h"
+#endif
+
 struct AppLayerParserThreadCtx_ {
     void *alproto_local_storage[FLOW_PROTO_MAX][ALPROTO_MAX];
 };
@@ -1789,6 +1793,16 @@ void AppLayerParserRegisterProtocolParsers(void)
                   "imap");
     }
 
+#ifdef ALPROTO_DYNAMIC_NB
+    for (size_t i = 0; i < ALPROTO_DYNAMIC_NB; i++) {
+        SCAppLayerPlugin *app_layer_plugin = SCPluginFindAppLayerByIndex(i);
+        if (app_layer_plugin == NULL) {
+            break;
+        }
+        app_layer_plugin->Register();
+    }
+#endif
+
     ValidateParsers();
     return;
 }