Next/20230608/v10

configure.ac

@@ -273,7 +273,7 @@
             CFLAGS="${CFLAGS} -DOS_WIN32"
             WINDOWS_PATH="yes"
             AC_DEFINE([HAVE_NON_POSIX_MKDIR], [1], [mkdir is not POSIX compliant: single arg])
-            RUST_LDADD=" -lws2_32 -liphlpapi -lwbemuuid -lOle32 -lOleAut32 -lUuid -luserenv -lshell32 -ladvapi32 -lgcc_eh -lbcrypt"
+            RUST_LDADD=" -lws2_32 -liphlpapi -lwbemuuid -lOle32 -lOleAut32 -lUuid -luserenv -lshell32 -ladvapi32 -lgcc_eh -lbcrypt -lntdll"
             TRY_WPCAP="yes"
             ;;
         *-*-cygwin)

doc/userguide/configuration/suricata-yaml.rst

@@ -2474,6 +2474,8 @@ Example:
   [10703] 26/11/2010 -- 11:41:15 - (detect.c:560) <Info> (SigLoadSignatures)
   -- Engine-Analysis for fast_pattern printed to file - /var/log/suricata/rules_fast_pattern.txt
 
+  alert tcp any any -> any any (content:"Volume Serial Number"; sid:1292;)
+
   == Sid: 1292 ==
   Fast pattern matcher: content
   Fast pattern set: no

doc/userguide/devguide/codebase/contributing/github-pr-workflow.rst

@@ -0,0 +1,46 @@
+GitHub Pull Request Workflow
+============================
+
+Draft Pull Requests
+~~~~~~~~~~~~~~~~~~~
+
+A Pull Request (PR) should be marked as `draft` if it is not intended to be merged as is,
+but is waiting for some sort of feedback.
+The author of the PR should be explicit with what kind of feedback is expected
+(CI/QA run, discussion on the code, etc...)
+
+GitHub filter is ``is:pr is:open draft:true sort:updated-asc``
+
+A draft may be closed if it has not been updated in two months.
+
+Mergeable Pull Requests
+~~~~~~~~~~~~~~~~~~~~~~~
+
+When a Pull Request is intended to be merged as is, the workflow is the following:
+ 1. get reviewed, and either request changes or get approved
+ 2. if approved, get staged in a next branch (with other PRs), wait for CI validation
+    (and eventually request changes if CI finds anything)
+ 3. get merged and closed
+
+A newly created PR should match the filter
+``is:pr is:open draft:false review:none sort:updated-asc no:assignee``
+The whole team is responsible to assign a PR to someone precise within 2 weeks.
+
+When someone gets assigned a PR, the PR should get a review status within 2 weeks:
+either changes requested, approved, or assigned to someone else if more
+expertise is needed.
+
+GitHub filter for changes-requested PRs is ``is:pr is:open draft:false sort:
+updated-asc review:changes-requested``
+
+Such a PR may be closed if it has not been updated in two months.
+It is expected that the author creates a new PR with a new version of the patch
+as described in :ref:`Pull Requests Criteria <pull-requests-criteria>`.
+
+Command to get approved PRs is ``gh pr list --json number,reviewDecision --search
+"state:open type:pr -review:none" | jq '.[] | select(.reviewDecision=="")'``
+
+Web UI filter does not work cf https://github.com/orgs/community/discussions/55826
+
+Once in approved state, the PRs are in the responsibility of the merger, along
+with the next branches/PRs.

doc/userguide/devguide/codebase/contributing/index.rst

@@ -6,3 +6,4 @@ Contributing
 
     contribution-process
     code-submission-process
+    github-pr-workflow

doc/userguide/install.rst

@@ -227,6 +227,18 @@ To reload rules::
 
 .. _install-advanced:
 
+Arch Based
+^^^^^^^^^^
+
+The ArchLinux AUR contains Suricata and suricata-nfqueue packages, with commonly
+used configurations for compilation (may also be edited to your liking). You may
+use makepkg, yay (sample below), or other AUR helpers to compile and build
+Suricata packages.
+
+::
+
+    yay -S suricata
+
 Advanced Installation
 ---------------------
 

doc/userguide/upgrade.rst

@@ -60,6 +60,9 @@ Logging changes
 - Protocol values and their names are built into Suricata instead of using the system's ``/etc/protocols`` file. Some names and casing may have changed
   in the values ``proto`` in ``eve.json`` log entries and other logs containing protocol names and values.
   See https://redmine.openinfosecfoundation.org/issues/4267 for more information.
+- Custom logging of HTTP headers via suricata.yaml ``outputs.eve-log.types.http.custom``
+  is now done in subobjects ``response_headers`` or ``request_headers`` (as for option ``dump-all-headers``)
+  instead of at the root of the ``http`` json object (to avoid some collisions).
 
 Deprecations
 ~~~~~~~~~~~~

rust/src/snmp/log.rs

@@ -18,7 +18,7 @@
 // written by Pierre Chifflier  <chifflier@wzdftpd.net>
 
 use crate::jsonbuilder::{JsonBuilder, JsonError};
-use crate::snmp::snmp::{SNMPState,SNMPTransaction};
+use crate::snmp::snmp::SNMPTransaction;
 use crate::snmp::snmp_parser::{NetworkAddress,PduType};
 use std::borrow::Cow;
 
@@ -37,9 +37,9 @@ fn str_of_pdu_type(t:&PduType) -> Cow<str> {
     }
 }
 
-fn snmp_log_response(jsb: &mut JsonBuilder, state: &mut SNMPState, tx: &mut SNMPTransaction) -> Result<(), JsonError>
+fn snmp_log_response(jsb: &mut JsonBuilder, tx: &mut SNMPTransaction) -> Result<(), JsonError>
 {
-    jsb.set_uint("version", state.version as u64)?;
+    jsb.set_uint("version", tx.version as u64)?;
     if tx.encrypted {
         jsb.set_string("pdu_type", "encrypted")?;
     } else {
@@ -75,7 +75,7 @@ fn snmp_log_response(jsb: &mut JsonBuilder, state: &mut SNMPState, tx: &mut SNMP
 }
 
 #[no_mangle]
-pub extern "C" fn rs_snmp_log_json_response(jsb: &mut JsonBuilder, state: &mut SNMPState, tx: &mut SNMPTransaction) -> bool
+pub extern "C" fn rs_snmp_log_json_response(jsb: &mut JsonBuilder, tx: &mut SNMPTransaction) -> bool
 {
-    snmp_log_response(jsb, state, tx).is_ok()
+    snmp_log_response(jsb, tx).is_ok()
 }

src/app-layer-ftp.c

@@ -340,7 +340,8 @@ typedef struct FtpInput_ {
     int32_t orig_len;
 } FtpInput;
 
-static AppLayerResult FTPGetLineForDirection(FtpState *state, FtpLineState *line, FtpInput *input)
+static AppLayerResult FTPGetLineForDirection(
+        FtpState *state, FtpLineState *line, FtpInput *input, bool *current_line_truncated)
 {
     SCEnter();
 
@@ -351,18 +352,18 @@ static AppLayerResult FTPGetLineForDirection(FtpState *state, FtpLineState *line
     uint8_t *lf_idx = memchr(input->buf + input->consumed, 0x0a, input->len);
 
     if (lf_idx == NULL) {
-        if (!state->current_line_truncated && (uint32_t)input->len >= ftp_max_line_len) {
-            state->current_line_truncated = true;
+        if (!(*current_line_truncated) && (uint32_t)input->len >= ftp_max_line_len) {
+            *current_line_truncated = true;
             line->buf = input->buf;
             line->len = ftp_max_line_len;
             line->delim_len = 0;
             input->len = 0;
             SCReturnStruct(APP_LAYER_OK);
         }
         SCReturnStruct(APP_LAYER_INCOMPLETE(input->consumed, input->len + 1));
-    } else if (state->current_line_truncated) {
+    } else if (*current_line_truncated) {
         // Whatever came in with first LF should also get discarded
-        state->current_line_truncated = false;
+        *current_line_truncated = false;
         line->len = 0;
         line->delim_len = 0;
         input->len = 0;
@@ -372,26 +373,18 @@ static AppLayerResult FTPGetLineForDirection(FtpState *state, FtpLineState *line
         // e.g. input_len = 5077
         //      lf_idx = 5010
         //      max_line_len = 4096
-        if (!state->current_line_truncated && (uint32_t)input->len >= ftp_max_line_len) {
-            state->current_line_truncated = true;
-            line->buf = input->buf;
-            line->len = ftp_max_line_len;
-            if (input->consumed >= 2 && input->buf[input->consumed - 2] == 0x0D) {
-                line->delim_len = 2;
-                line->len -= 2;
-            } else {
-                line->delim_len = 1;
-                line->len -= 1;
-            }
-            input->len = 0;
-            SCReturnStruct(APP_LAYER_OK);
-        }
         uint32_t o_consumed = input->consumed;
         input->consumed = lf_idx - input->buf + 1;
         line->len = input->consumed - o_consumed;
         input->len -= line->len;
+        line->lf_found = true;
         DEBUG_VALIDATE_BUG_ON((input->consumed + input->len) != input->orig_len);
         line->buf = input->buf + o_consumed;
+        if (line->len >= ftp_max_line_len) {
+            *current_line_truncated = true;
+            line->len = ftp_max_line_len;
+            SCReturnStruct(APP_LAYER_OK);
+        }
         if (input->consumed >= 2 && input->buf[input->consumed - 2] == 0x0D) {
             line->delim_len = 2;
             line->len -= 2;
@@ -505,12 +498,12 @@ static AppLayerResult FTPParseRequest(Flow *f, void *ftp_state, AppLayerParserSt
     }
 
     FtpInput ftpi = { .buf = input, .len = input_len, .orig_len = input_len, .consumed = 0 };
-    FtpLineState line = { .buf = NULL, .len = 0, .delim_len = 0 };
+    FtpLineState line = { .buf = NULL, .len = 0, .delim_len = 0, .lf_found = false };
 
     uint8_t direction = STREAM_TOSERVER;
     AppLayerResult res;
     while (1) {
-        res = FTPGetLineForDirection(state, &line, &ftpi);
+        res = FTPGetLineForDirection(state, &line, &ftpi, &state->current_line_truncated_ts);
         if (res.status == 1) {
             return res;
         } else if (res.status == -1) {
@@ -531,8 +524,11 @@ static AppLayerResult FTPParseRequest(Flow *f, void *ftp_state, AppLayerParserSt
 
         tx->command_descriptor = cmd_descriptor;
         tx->request_length = CopyCommandLine(&tx->request, &line);
-        tx->request_truncated = state->current_line_truncated;
+        tx->request_truncated = state->current_line_truncated_ts;
 
+        if (line.lf_found) {
+            state->current_line_truncated_ts = false;
+        }
         if (tx->request_truncated) {
             AppLayerDecoderEventsSetEventRaw(&tx->tx_data.events, FtpEventRequestCommandTooLong);
         }
@@ -695,12 +691,12 @@ static AppLayerResult FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserS
         SCReturnStruct(APP_LAYER_OK);
     }
     FtpInput ftpi = { .buf = input, .len = input_len, .orig_len = input_len, .consumed = 0 };
-    FtpLineState line = { .buf = NULL, .len = 0, .delim_len = 0 };
+    FtpLineState line = { .buf = NULL, .len = 0, .delim_len = 0, .lf_found = false };
 
     FTPTransaction *lasttx = TAILQ_FIRST(&state->tx_list);
     AppLayerResult res;
     while (1) {
-        res = FTPGetLineForDirection(state, &line, &ftpi);
+        res = FTPGetLineForDirection(state, &line, &ftpi, &state->current_line_truncated_tc);
         if (res.status == 1) {
             return res;
         } else if (res.status == -1) {
@@ -771,11 +767,14 @@ static AppLayerResult FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserS
             FTPString *response = FTPStringAlloc();
             if (likely(response)) {
                 response->len = CopyCommandLine(&response->str, &line);
-                response->truncated = state->current_line_truncated;
+                response->truncated = state->current_line_truncated_tc;
                 if (response->truncated) {
                     AppLayerDecoderEventsSetEventRaw(
                             &tx->tx_data.events, FtpEventResponseCommandTooLong);
                 }
+                if (line.lf_found) {
+                    state->current_line_truncated_tc = false;
+                }
                 TAILQ_INSERT_TAIL(&tx->response_list, response, next);
             }
         }

src/app-layer-ftp.h

@@ -105,6 +105,7 @@ typedef struct FtpLineState_ {
     const uint8_t *buf;
     uint32_t len;
     uint8_t delim_len;
+    bool lf_found;
 } FtpLineState;
 
 typedef struct FTPString_ {
@@ -148,7 +149,8 @@ typedef struct FtpState_ {
     TAILQ_HEAD(, FTPTransaction_) tx_list;  /**< transaction list */
     uint64_t tx_cnt;
 
-    bool current_line_truncated;
+    bool current_line_truncated_ts;
+    bool current_line_truncated_tc;
 
     FtpRequestCommand command;
     FtpRequestCommandArgOfs arg_offset;

src/decode-ipv6.c

@@ -574,6 +574,7 @@ int DecodeIPV6(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *
         CLEAR_IPV6_PACKET(p);
         return TM_ECODE_FAILED;
     }
+    p->proto = IPV6_GET_NH(p);
 
 #ifdef DEBUG
     if (SCLogDebugEnabled()) { /* only convert the addresses if debug is really enabled */

src/output-json-alert.c

@@ -253,7 +253,7 @@ static void AlertJsonSNMP(const Flow *f, const uint64_t tx_id, JsonBuilder *js)
                 tx_id);
         if (tx != NULL) {
             jb_open_object(js, "snmp");
-            rs_snmp_log_json_response(js, snmp_state, tx);
+            rs_snmp_log_json_response(js, tx);
             jb_close(js);
         }
     }

src/output-json-snmp.c

@@ -60,7 +60,7 @@ static int JsonSNMPLogger(ThreadVars *tv, void *thread_data,
     }
 
     jb_open_object(jb, "snmp");
-    if (!rs_snmp_log_json_response(jb, state, snmptx)) {
+    if (!rs_snmp_log_json_response(jb, snmptx)) {
         goto error;
     }
     jb_close(jb);

src/source-pcap-file-helper.c

@@ -47,7 +47,7 @@ void CleanupPcapFileFileVars(PcapFileFileVars *pfv)
             if (pfv->shared != NULL && pfv->shared->should_delete) {
                 SCLogDebug("Deleting pcap file %s", pfv->filename);
                 if (unlink(pfv->filename) != 0) {
-                    SCLogWarning("Failed to delete %s", pfv->filename);
+                    SCLogWarning("Failed to delete %s: %s", pfv->filename, strerror(errno));
                 }
             }
             SCFree(pfv->filename);

src/util-exception-policy.c

@@ -72,6 +72,9 @@ void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDro
         case EXCEPTION_POLICY_REJECT:
             SCLogDebug("EXCEPTION_POLICY_REJECT");
             PacketDrop(p, ACTION_REJECT, drop_reason);
+            if (!EngineModeIsIPS()) {
+                break;
+            }
             /* fall through */
         case EXCEPTION_POLICY_DROP_FLOW:
             SCLogDebug("EXCEPTION_POLICY_DROP_FLOW");