next/1335/80x/20260508/v1

configure.ac

@@ -439,33 +439,6 @@
         CFLAGS="${CFLAGS} -pg"
     ])
 
-    #enable gcc march=native gcc 4.2 or later
-    AC_ARG_ENABLE(gccmarch_native,
-           AS_HELP_STRING([--enable-gccmarch-native], [Enable gcc march=native gcc 4.2 and later only]),[enable_gccmarch_native=$enableval],[enable_gccmarch_native=yes])
-    AS_IF([test "x$enable_gccmarch_native" = "xyes"], [
-        case "$host" in
-            *powerpc*)
-                ;;
-            *)
-            OFLAGS="$CFLAGS"
-            CFLAGS="$CFLAGS -march=native"
-            AC_MSG_CHECKING([checking if $CC supports -march=native])
-            AC_COMPILE_IFELSE(  [AC_LANG_PROGRAM([[#include <stdlib.h>]])],
-                        [
-                          AC_MSG_RESULT([yes])
-                          OPTIMIZATION_CFLAGS="-march=native"
-                          AC_SUBST(OPTIMIZATION_CFLAGS)
-                        ],
-                        [
-                          AC_MSG_RESULT([no])
-                          CFLAGS="$OFLAGS"
-                          enable_gccmarch_native=no
-                        ]
-                     )
-                ;;
-        esac
-    ])
-
 # options
 
 
@@ -2426,6 +2399,34 @@ return 0;
 
     AM_CONDITIONAL([HAS_FUZZLDFLAGS], [test "x$has_sanitizefuzzer" = "xyes"])
 
+#enable gcc march=native gcc 4.2 or later
+    AC_ARG_ENABLE(gccmarch_native,
+           AS_HELP_STRING([--enable-gccmarch-native], [Enable gcc march=native gcc 4.2 and later only]),[enable_gccmarch_native=$enableval],[enable_gccmarch_native=yes])
+    AS_IF([test "x$enable_gccmarch_native" = "xyes"], [
+        case "$host" in
+            *powerpc*)
+                ;;
+            *)
+            OFLAGS="$CFLAGS"
+            CFLAGS="$CFLAGS -march=native"
+            AC_MSG_CHECKING([checking if $CC supports -march=native])
+            AC_COMPILE_IFELSE(  [AC_LANG_PROGRAM([[#include <stdlib.h>]])],
+                        [
+                          AC_MSG_RESULT([yes])
+                          OPTIMIZATION_CFLAGS="-march=native"
+                          AC_SUBST(OPTIMIZATION_CFLAGS)
+                        ],
+                        [
+                          AC_MSG_RESULT([no])
+                          CFLAGS="$OFLAGS"
+                          enable_gccmarch_native=no
+                        ]
+                     )
+                ;;
+        esac
+    ])
+
+
 # get git revision and last commit date
     AC_PATH_PROG(HAVE_GIT_CMD, git, "no")
     if test "$HAVE_GIT_CMD" != "no"; then

doc/userguide/output/eve/eve-json-output.rst

@@ -32,9 +32,9 @@ may be held in memory and written a short time later opening the possibility --
 loss.
 
 Hence, a heartbeat mechanism is introduced to limit the amount of time buffered data may exist before being
-flushed.  Control is provided to instruct Suricata's detection threads to flush their EVE output. With default
+flushed.  A heartbeat thread periodically flushes all active EVE log files directly. With default
 values, there is no change in output buffering and flushing behavior. ``output-flush-interval`` controls
-how often Suricata's detect threads will flush output in a heartbeat fashion. A value of ``0`` means
+how often Suricata will flush EVE output in a heartbeat fashion. A value of ``0`` means
 "never"; non-zero values must be in ``[1-60]`` seconds.
 
 Flushing should be considered when ``outputs.buffer-size`` is greater than 0 to limit the amount and

doc/userguide/partials/eve-log.yaml

@@ -286,12 +286,12 @@ outputs:
         #   spurious-retransmission: false  # log spurious retransmission packets
         #
 heartbeat:
-  # The output-flush-interval value governs how often Suricata will instruct the
-  # detection threads to flush their EVE output. Specify the value in seconds [1-60]
-  # and Suricata will initiate EVE log output flushes at that interval. A value
-  # of 0 means no EVE log output flushes are initiated. When the EVE output
+  # The output-flush-interval value governs how often Suricata will flush
+  # EVE log file output. Specify the value in seconds [1-60] and Suricata will
+  # flush all active EVE log files at that interval. A value of 0 means
+  # no EVE log output flushes are performed. When the EVE output
   # buffer-size value is non-zero, some EVE output that was written may remain
   # buffered. The output-flush-interval governs how much buffered data exists.
   #
-  # The default value is: 0 (never instruct detection threads to flush output)
+  # The default value is: 0 (no periodic flushing)
   #output-flush-interval: 0

doc/userguide/upgrade.rst

@@ -34,6 +34,25 @@ also check all the new features that have been added but are not covered by
 this guide. Those features are either not enabled by default or require
 dedicated new configuration.
 
+Upgrading to 8.0.5
+------------------
+
+Other Changes
+~~~~~~~~~~~~~
+
+- We've made a change in the way that background EVE flushing driven by the
+  heartbeat mechanism operates. The heartbeat mechanism provides a way to
+  periodically flush EVE outputs when ``eve-log.buffer-size`` is non-zero and
+  ``heartbeat.output-flush-interval`` is non-zero.
+
+  There is no change to the functionality enabled by the heartbeat mechanism; we
+  overhauled and simplified the implementation. Developers maintaining
+  out-of-tree output plugins may need to update their code in order for the
+  plugin to compile and load, due to the removal of the packet-logger
+  ``FlushFunc`` registration field and related helpers (``OutputJsonFlush``,
+  ``OutputJsonLogFlush``, ``OutputLoggerFlush``). Periodic heartbeat flushing is
+  specific to EVE output types and handled entirely within the EVE logic.
+
 Upgrading to 8.0.3
 ------------------
 

rules/Makefile.am

@@ -2,6 +2,7 @@ ruledir = $(datadir)/suricata/rules
 
 dist_rule_DATA = \
 app-layer-events.rules \
+bittorrent-events.rules \
 decoder-events.rules \
 dhcp-events.rules \
 dnp3-events.rules \
@@ -25,6 +26,7 @@ quic-events.rules \
 rfb-events.rules \
 smb-events.rules \
 smtp-events.rules \
+snmp-events.rules \
 ssh-events.rules \
 stream-events.rules \
 tls-events.rules \

rules/README.md

@@ -30,9 +30,11 @@ signature IDs.
 | FTP      | 2232000 | 2232999 |
 | POP3     | 2236000 | 2236999 |
 | LDAP     | 2237000 | 2237999 |
+| SNMP     | 2238000 | 2238999 |
 | DNS      | 2240000 | 2240999 |
 | PGSQL    | 2241000 | 2241999 |
 | mDNS     | 2242000 | 2242999 |
+| Bittorent| 2243000 | 2243999 |
 | MODBUS   | 2250000 | 2250999 |
 | DNP3     | 2270000 | 2270999 |
 | HTTP2    | 2290000 | 2290999 |

rules/bittorrent-events.rules

@@ -0,0 +1,7 @@
+# BitTorrent DHT app layer event rules
+#
+# SID's fall in the 2243000+ range. See https://redmine.openinfosecfoundation.org/projects/suricata/wiki/AppLayer
+#
+# These sigs fire at most once per connection.
+#
+alert bittorrent-dht any any -> any any (msg:"SURICATA BitTorrent DHT malformed packet"; app-layer-event:bittorrent-dht.malformed_packet; classtype:protocol-command-decode; sid:2243000; rev:1;)

rules/snmp-events.rules

@@ -0,0 +1,9 @@
+# SNMP app layer event rules
+#
+# SID's fall in the 2238000+ range. See https://redmine.openinfosecfoundation.org/projects/suricata/wiki/AppLayer
+#
+# These sigs fire at most once per connection.
+#
+alert snmp any any -> any any (msg:"SURICATA SNMP malformed data"; app-layer-event:snmp.malformed_data; classtype:protocol-command-decode; sid:2238000; rev:1;)
+alert snmp any any -> any any (msg:"SURICATA SNMP unknown security model"; app-layer-event:snmp.unknown_security_model; classtype:protocol-command-decode; sid:2238001; rev:1;)
+alert snmp any any -> any any (msg:"SURICATA SNMP version mismatch"; app-layer-event:snmp.version_mismatch; classtype:protocol-command-decode; sid:2238002; rev:1;)

src/alert-debuglog.c

@@ -482,7 +482,6 @@ void AlertDebugLogRegister(void)
 {
     OutputPacketLoggerFunctions output_logger_functions = {
         .LogFunc = AlertDebugLogLogger,
-        .FlushFunc = NULL,
         .ConditionFunc = AlertDebugLogCondition,
         .ThreadInitFunc = AlertDebugLogThreadInit,
         .ThreadDeinitFunc = AlertDebugLogThreadDeinit,

src/alert-fastlog.c

@@ -78,7 +78,6 @@ void AlertFastLogRegister(void)
 {
     OutputPacketLoggerFunctions output_logger_functions = {
         .LogFunc = AlertFastLogger,
-        .FlushFunc = NULL,
         .ConditionFunc = AlertFastLogCondition,
         .ThreadInitFunc = AlertFastLogThreadInit,
         .ThreadDeinitFunc = AlertFastLogThreadDeinit,

src/alert-syslog.c

@@ -386,7 +386,6 @@ void AlertSyslogRegister (void)
 #ifndef OS_WIN32
     OutputPacketLoggerFunctions output_logger_functions = {
         .LogFunc = AlertSyslogLogger,
-        .FlushFunc = NULL,
         .ConditionFunc = AlertSyslogCondition,
         .ThreadInitFunc = AlertSyslogThreadInit,
         .ThreadDeinitFunc = AlertSyslogThreadDeinit,

src/app-layer-ftp.c

@@ -274,7 +274,7 @@ static AppLayerResult FTPGetLineForDirection(
     if (input->len <= 0)
         return APP_LAYER_ERROR;
 
-    uint8_t *lf_idx = memchr(input->buf + input->consumed, 0x0a, input->len);
+    const uint8_t *lf_idx = memchr(input->buf + input->consumed, 0x0a, input->len);
 
     if (lf_idx == NULL) {
         if (!(*current_line_truncated) && (uint32_t)input->len >= ftp_max_line_len) {
@@ -1375,7 +1375,7 @@ uint16_t JsonGetNextLineFromBuffer(const char *buffer, const uint16_t len)
         return UINT16_MAX;
     }
 
-    char *c = strchr(buffer, '\n');
+    const char *c = strchr(buffer, '\n');
     return c == NULL ? len : (uint16_t)(c - buffer + 1);
 }
 

src/app-layer-smtp.c

@@ -555,7 +555,7 @@ static AppLayerResult SMTPGetLine(Flow *f, StreamSlice *slice, SMTPState *state,
     }
     SCLogDebug("frame %p", frame);
 
-    uint8_t *lf_idx = memchr(input->buf + input->consumed, 0x0a, input->len);
+    const uint8_t *lf_idx = memchr(input->buf + input->consumed, 0x0a, input->len);
     bool discard_till_lf = (direction == 0) ? state->discard_till_lf_ts : state->discard_till_lf_tc;
 
     if (lf_idx == NULL) {

src/conf-yaml-loader.c

@@ -82,7 +82,7 @@ Mangle(char *string)
 static void
 ConfYamlSetConfDirname(const char *filename)
 {
-    char *ep;
+    const char *ep;
 
     ep = strrchr(filename, '\\');
     if (ep == NULL)

src/datasets.c

@@ -156,7 +156,7 @@ static int DatasetLoadIPv4(Dataset *set)
 int DatasetParseIpv6String(Dataset *set, const char *line, struct in6_addr *in6)
 {
     /* Checking IPv6 case */
-    char *got_colon = strchr(line, ':');
+    const char *got_colon = strchr(line, ':');
     if (got_colon) {
         uint32_t ip6addr[4];
         if (inet_pton(AF_INET6, line, in6) != 1) {

src/decode.c

@@ -743,7 +743,7 @@ void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
             }
 
             char name[256];
-            char *dot = strchr(DEvents[i].event_name, '.');
+            const char *dot = strchr(DEvents[i].event_name, '.');
             BUG_ON(!dot);
             snprintf(name, sizeof(name), "%s.%s",
                     stats_decoder_events_prefix, dot+1);

src/decode.h

@@ -1315,12 +1315,8 @@ void DecodeUnregisterCounters(void);
 #define PKT_FIRST_ALERTS BIT_U32(29)
 #define PKT_FIRST_TAG    BIT_U32(30)
 
-#define PKT_PSEUDO_LOG_FLUSH BIT_U32(31) /**< Detect/log flush for protocol upgrade */
-
 /** \brief return 1 if the packet is a pseudo packet */
-#define PKT_IS_PSEUDOPKT(p) \
-    ((p)->flags & (PKT_PSEUDO_STREAM_END|PKT_PSEUDO_DETECTLOG_FLUSH))
-#define PKT_IS_FLUSHPKT(p) ((p)->flags & (PKT_PSEUDO_LOG_FLUSH))
+#define PKT_IS_PSEUDOPKT(p) ((p)->flags & (PKT_PSEUDO_STREAM_END | PKT_PSEUDO_DETECTLOG_FLUSH))
 
 #define PKT_SET_SRC(p, src_val) ((p)->pkt_src = src_val)
 

src/detect-app-layer-protocol.c

@@ -157,7 +157,7 @@ static DetectAppLayerProtocolData *DetectAppLayerProtocolParse(const char *arg,
     AppProto alproto = ALPROTO_UNKNOWN;
 
     char alproto_copy[MAX_ALPROTO_NAME];
-    char *sep = strchr(arg, ',');
+    const char *sep = strchr(arg, ',');
     char *alproto_name;
     if (sep && sep - arg < MAX_ALPROTO_NAME) {
         strlcpy(alproto_copy, arg, sep - arg + 1);

src/detect-engine.c

@@ -2241,38 +2241,6 @@ int DetectEngineInspectPktBufferGeneric(
     }
 }
 
-/** \internal
- *  \brief inject a pseudo packet into each detect thread
- *         if the thread should flush its output logs.
- */
-void InjectPacketsForFlush(ThreadVars **detect_tvs, int no_of_detect_tvs)
-{
-    /* inject a fake packet if the detect thread that needs it. This function
-     * is called when a heartbeat log-flush request has been made
-     * and it should process a pseudo packet and flush its output logs
-     * to speed the process. */
-#if DEBUG
-    int count = 0;
-#endif
-    for (int i = 0; i < no_of_detect_tvs; i++) {
-        if (detect_tvs[i]) { // && detect_tvs[i]->inq != NULL) {
-            Packet *p = PacketGetFromAlloc();
-            if (p != NULL) {
-                SCLogDebug("Injecting pkt for tv %s[i=%d] %d", detect_tvs[i]->name, i, count++);
-                p->flags |= PKT_PSEUDO_STREAM_END;
-                p->flags |= PKT_PSEUDO_LOG_FLUSH;
-                PKT_SET_SRC(p, PKT_SRC_DETECT_RELOAD_FLUSH);
-                PacketQueue *q = detect_tvs[i]->stream_pq;
-                SCMutexLock(&q->mutex_q);
-                PacketEnqueue(q, p);
-                SCCondSignal(&q->cond_q);
-                SCMutexUnlock(&q->mutex_q);
-            }
-        }
-    }
-    SCLogDebug("leaving: thread notification count = %d", count);
-}
-
 /** \internal
  *  \brief inject a pseudo packet into each detect thread
  *      -that doesn't use the new det_ctx yet

src/detect-engine.h

@@ -209,6 +209,4 @@ bool DetectMd5ValidateCallback(
 
 void DeStateRegisterTests(void);
 
-/* packet injection */
-void InjectPacketsForFlush(ThreadVars **detect_tvs, int no_of_detect_tvs);
 #endif /* SURICATA_DETECT_ENGINE_H */

src/detect-pcre.c

@@ -416,9 +416,9 @@ static DetectPcreData *DetectPcreParse (DetectEngineCtx *de_ctx,
     bool apply_match_limit = false;
 
     int cut_capture = 0;
-    char *fcap = strstr(regexstr, "flow:");
-    char *pcap = strstr(regexstr, "pkt:");
-    char *acap = strstr(regexstr, "alert:");
+    const char *fcap = strstr(regexstr, "flow:");
+    const char *pcap = strstr(regexstr, "pkt:");
+    const char *acap = strstr(regexstr, "alert:");
     /* take the size of the whole input as buffer size for the regex we will
      * extract below. Add 1 to please Coverity's alloc_strlen test. */
     size_t slen = strlen(regexstr) + 1;

src/flow-worker.c

@@ -73,8 +73,6 @@ typedef struct FlowWorkerThreadData_ {
 
     SC_ATOMIC_DECLARE(DetectEngineThreadCtxPtr, detect_thread);
 
-    SC_ATOMIC_DECLARE(bool, flush_ack);
-
     void *output_thread; /* Output thread data. */
     void *output_thread_flow; /* Output thread data. */
 
@@ -564,15 +562,6 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
 
     SCLogDebug("packet %"PRIu64, p->pcap_cnt);
 
-    if ((PKT_IS_FLUSHPKT(p))) {
-        SCLogDebug("thread %s flushing", tv->printable_name);
-        OutputLoggerFlush(tv, p, fw->output_thread);
-        /* Ack if a flush was requested */
-        bool notset = false;
-        SC_ATOMIC_CAS(&fw->flush_ack, notset, true);
-        return TM_ECODE_OK;
-    }
-
     /* handle Flow */
     if (det_ctx != NULL && det_ctx->de_ctx->PreFlowHook != NULL) {
         const uint8_t action = det_ctx->de_ctx->PreFlowHook(tv, det_ctx, p);
@@ -756,18 +745,6 @@ void *FlowWorkerGetThreadData(void *flow_worker)
     return (FlowWorkerThreadData *)flow_worker;
 }
 
-bool FlowWorkerGetFlushAck(void *flow_worker)
-{
-    FlowWorkerThreadData *fw = flow_worker;
-    return SC_ATOMIC_GET(fw->flush_ack) == true;
-}
-
-void FlowWorkerSetFlushAck(void *flow_worker)
-{
-    FlowWorkerThreadData *fw = flow_worker;
-    SC_ATOMIC_SET(fw->flush_ack, false);
-}
-
 const char *ProfileFlowWorkerIdToString(enum ProfileFlowWorkerId fwi)
 {
     switch (fwi) {

src/flow-worker.h

@@ -33,8 +33,6 @@ const char *ProfileFlowWorkerIdToString(enum ProfileFlowWorkerId fwi);
 void FlowWorkerReplaceDetectCtx(void *flow_worker, void *detect_ctx);
 void *FlowWorkerGetDetectCtxPtr(void *flow_worker);
 void *FlowWorkerGetThreadData(void *flow_worker);
-bool FlowWorkerGetFlushAck(void *flow_worker);
-void FlowWorkerSetFlushAck(void *flow_worker);
 
 void TmModuleFlowWorkerRegister (void);