Next/60x/20221125/v7

.readthedocs.yaml

@@ -1,6 +1,8 @@
 # Required by Read The Docs
 version: 2
 
+formats: all
+
 python:
   version: "3.8"
 

qa/coccinelle/Makefile.am

@@ -1,5 +1,4 @@
 EXTRA_DIST= access-pkt-packet.cocci \
-    action-pkt.cocci \
     banned-functions.cocci \
     direct-packet.cocci \
     malloc-error-check.cocci \

qa/coccinelle/access-pkt-packet.cocci

@@ -30,7 +30,7 @@ p->pkt@p1
 p1 << pktfield.p1;
 @@
 
-print "Invalid Packet->pkt usage, GET_PKT_DATA macro must be used at %s:%s" % (p1[0].file, p1[0].line)
+print("Invalid Packet->pkt usage, GET_PKT_DATA macro must be used at %s:%s" % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)
 
@@ -50,6 +50,6 @@ p->pktlen@p1
 p1 << pktlenfield.p1;
 @@
 
-print "Invalid Packet->pktlen usage, GET_PKT_LEN macro must be used at %s:%s" % (p1[0].file, p1[0].line)
+print("Invalid Packet->pktlen usage, GET_PKT_LEN macro must be used at %s:%s" % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

qa/coccinelle/direct-packet.cocci

@@ -10,6 +10,6 @@ Packet p@p1;
 p1 << directpacket.p1;
 @@
 
-print "Invalid Packet definition, explicit allocation must be used at %s:%s" % (p1[0].file, p1[0].line)
+print("Invalid Packet definition, explicit allocation must be used at %s:%s" % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

qa/coccinelle/malloc-error-check.cocci

@@ -58,6 +58,6 @@ BUG_ON(x == NULL)
 @script:python depends on !realloc && !istested && !inlinetested@
 p1 << malloced.p1;
 @@
-print "Structure malloced at %s:%s but error is not checked." % (p1[0].file, p1[0].line)
+print("Structure malloced at %s:%s but error is not checked." % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

qa/coccinelle/pktnotset-packet.cocci

@@ -24,6 +24,6 @@ PACKET_INITIALIZE(p)
 p1 << zeroed.p1;
 @@
 
-print "Packet zeroed at %s:%s but pkt field is not set afterward." % (p1[0].file, p1[0].line)
+print("Packet zeroed at %s:%s but pkt field is not set afterward." % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

qa/coccinelle/realloc.cocci

@@ -13,6 +13,6 @@ x@p1 = (ty *) SCRealloc(x, E)
 @script:python@
 p1 << realloc.p1;
 @@
-print "Structure reallocated at %s:%s but original pointer is lost and not freed in case of error." % (p1[0].file, p1[0].line)
+print("Structure reallocated at %s:%s but original pointer is lost and not freed in case of error." % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

qa/coccinelle/siginit.cocci

@@ -21,6 +21,6 @@ goto error;
 @script:python@
 p1 << siginit.p1;
 @@
-print "SigMatch added at %s:%s but error handling can cause it to be freed later." % (p1[0].file, p1[0].line)
+print("SigMatch added at %s:%s but error handling can cause it to be freed later." % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

qa/coccinelle/size_t.cocci

@@ -39,6 +39,6 @@ funcn(E1, E2,..., (uint64_t) p, ...)@p1;
 p1 << sizet.p1;
 @@
 
-print "Invalid printf with size_t (not casted to uintmax_t) at %s:%s" % (p1[0].file, p1[0].line)
+print("Invalid printf with size_t (not casted to uintmax_t) at %s:%s" % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

qa/coccinelle/sz3.cocci

@@ -24,7 +24,7 @@ sizeof(e@p1)
 p1 << preuse.p1;
 @@
 
-print "Size of pointed value not pointer used at %s:%s" % (p1[0].file, p1[0].line)
+print("Size of pointed value not pointer used at %s:%s" % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)
 
@@ -43,6 +43,6 @@ sizeof(e@p1)
 p1 << postuse.p1;
 @@
 
-print "Size of pointed value not pointer used at %s:%s" % (p1[0].file, p1[0].line)
+print("Size of pointed value not pointer used at %s:%s" % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

qa/coccinelle/uint.cocci

@@ -8,6 +8,6 @@ i@p1
 @script:python@
 p1 << uint.p1;
 @@
-print "banned type uint used at at %s:%s, please use a explicit length." % (p1[0].file, p1[0].line)
+print("banned type uint used at at %s:%s, please use a explicit length." % (p1[0].file, p1[0].line))
 import sys
 sys.exit(1)

rust/src/filecontainer.rs

@@ -26,8 +26,6 @@ extern {
 }
 pub const FILE_USE_DETECT:    u16 = BIT_U16!(13);
 
-
-pub struct File;
 #[repr(C)]
 #[derive(Debug)]
 pub struct FileContainer {
@@ -49,13 +47,13 @@ impl FileContainer {
         }
     }
 
-    pub fn file_open(&mut self, cfg: &'static SuricataFileContext, track_id: &u32, name: &[u8], flags: u16) -> i32 {
+    pub fn file_open(&mut self, cfg: &'static SuricataFileContext, track_id: u32, name: &[u8], flags: u16) -> i32 {
         match unsafe {SC} {
             None => panic!("BUG no suricata_config"),
             Some(c) => {
                 SCLogDebug!("FILE {:p} OPEN flags {:04X}", &self, flags);
 
-                let res = (c.FileOpenFile)(&self, cfg.files_sbcfg, *track_id,
+                let res = (c.FileOpenFile)(&self, cfg.files_sbcfg, track_id,
                         name.as_ptr(), name.len() as u16,
                         ptr::null(), 0u32, flags);
                 res

rust/src/filetracker.rs

@@ -95,7 +95,7 @@ impl FileTransferTracker {
     fn open(&mut self, config: &'static SuricataFileContext,
             files: &mut FileContainer, flags: u16, name: &[u8]) -> i32
     {
-        let r = files.file_open(config, &self.track_id, name, flags);
+        let r = files.file_open(config, self.track_id, name, flags);
         if r == 0 {
             files.file_set_txid_on_last_file(self.tx_id);
             self.file_open = true;

rust/src/http2/decompression.rs

@@ -50,6 +50,11 @@ impl HTTP2cursor {
     pub fn set_position(&mut self, pos: u64) {
         return self.cursor.set_position(pos);
     }
+
+    pub fn clear(&mut self) {
+        self.cursor.get_mut().clear();
+        self.cursor.set_position(0);
+    }
 }
 
 // we need to implement this as flate2 and brotli crates
@@ -152,8 +157,7 @@ fn http2_decompress<'a>(
         }
     }
     //brotli does not consume all input if it reaches some end
-
-    decoder.get_mut().set_position(0);
+    decoder.get_mut().clear();
     return Ok(&output[..offset]);
 }
 

rust/src/http2/http2.rs

@@ -832,6 +832,7 @@ impl HTTP2State {
                     let over = head.flags & parser::HTTP2_FLAG_HEADER_EOS != 0;
                     let ftype = head.ftype;
                     let sid = head.stream_id;
+                    let padded = head.flags & parser::HTTP2_FLAG_HEADER_PADDED != 0;
                     if dir == STREAM_TOSERVER {
                         tx.frames_ts.push(HTTP2Frame {
                             header: head,
@@ -851,8 +852,12 @@ impl HTTP2State {
                                 if index > 0 {
                                     let tx_same = &mut self.transactions[index - 1];
                                     let (files, flags) = self.files.get(dir);
+                                    let mut dinput = &rem[..hlsafe];
+                                    if padded && rem.len() > 0 && usize::from(rem[0]) < hlsafe{
+                                        dinput = &rem[1..hlsafe - usize::from(rem[0])];
+                                    }
                                     match tx_same.decompress(
-                                        &rem[..hlsafe],
+                                        dinput,
                                         dir,
                                         sfcm,
                                         over,

rust/src/http2/parser.rs

@@ -610,7 +610,7 @@ pub struct HTTP2FrameHeaders {
 //end stream
 pub const HTTP2_FLAG_HEADER_EOS: u8 = 0x1;
 pub const HTTP2_FLAG_HEADER_END_HEADERS: u8 = 0x4;
-const HTTP2_FLAG_HEADER_PADDED: u8 = 0x8;
+pub const HTTP2_FLAG_HEADER_PADDED: u8 = 0x8;
 const HTTP2_FLAG_HEADER_PRIORITY: u8 = 0x20;
 
 pub fn http2_parse_frame_headers<'a>(

rust/src/smb/smb.rs

@@ -690,6 +690,24 @@ impl SMBCommonHdr {
         }
 
     }
+    pub fn from2_notree(r: &Smb2Record, rec_type: u32) -> SMBCommonHdr {
+        // async responses do not have a tree id (even if the request has it)
+        // making thus the match between the two impossible.
+        // Per spec, MessageId should be enough to identifiy a message request and response uniquely
+        // across all messages that are sent on the same SMB2 Protocol transport connection.
+        // cf https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-smb2/ea4560b7-90da-4803-82b5-344754b92a79
+        let msg_id = match rec_type {
+            SMBHDR_TYPE_TRANS_FRAG | SMBHDR_TYPE_SHARE => { 0 },
+            _ => { r.message_id as u64 },
+        };
+
+        SMBCommonHdr {
+            rec_type : rec_type,
+            ssn_id : r.session_id,
+            tree_id : 0,
+            msg_id : msg_id,
+        }
+    }
     pub fn from1(r: &SmbRecord, rec_type: u32) -> SMBCommonHdr {
         let tree_id = match rec_type {
             SMBHDR_TYPE_TREE => { 0 },

rust/src/smb/smb2.rs

@@ -143,7 +143,7 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
 
             // get the request info. If we don't have it, there is nothing
             // we can do except skip this record.
-            let guid_key = SMBCommonHdr::from2(r, SMBHDR_TYPE_OFFSET);
+            let guid_key = SMBCommonHdr::from2_notree(r, SMBHDR_TYPE_OFFSET);
             let (offset, file_guid) = match state.ssn2vecoffset_map.remove(&guid_key) {
                 Some(o) => (o.offset, o.guid),
                 None => {
@@ -523,7 +523,7 @@ pub fn smb2_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
                                 rd.guid, rd.rd_len, rd.rd_offset);
 
                         // store read guid,offset in map
-                        let guid_key = SMBCommonHdr::from2(r, SMBHDR_TYPE_OFFSET);
+                        let guid_key = SMBCommonHdr::from2_notree(r, SMBHDR_TYPE_OFFSET);
                         let guidoff = SMBFileGUIDOffset::new(rd.guid.to_vec(), rd.rd_offset);
                         state.ssn2vecoffset_map.insert(guid_key, guidoff);
                     }
@@ -542,7 +542,7 @@ pub fn smb2_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
 
                     SCLogDebug!("create_options {:08x}", cr.create_options);
 
-                    let name_key = SMBCommonHdr::from2(r, SMBHDR_TYPE_FILENAME);
+                    let name_key = SMBCommonHdr::from2_notree(r, SMBHDR_TYPE_FILENAME);
                     state.ssn2vec_map.insert(name_key, cr.data.to_vec());
 
                     let tx_hdr = SMBCommonHdr::from2(r, SMBHDR_TYPE_GENERICTX);
@@ -671,7 +671,7 @@ pub fn smb2_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
             } else if r.nt_status == SMB_NTSTATUS_END_OF_FILE {
                 SCLogDebug!("SMBv2: read response => EOF");
 
-                let guid_key = SMBCommonHdr::from2(r, SMBHDR_TYPE_OFFSET);
+                let guid_key = SMBCommonHdr::from2_notree(r, SMBHDR_TYPE_OFFSET);
                 let file_guid = match state.ssn2vecoffset_map.remove(&guid_key) {
                     Some(o) => o.guid,
                     _ => {
@@ -707,7 +707,7 @@ pub fn smb2_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
                     Ok((_, cr)) => {
                         SCLogDebug!("SMBv2: Create response => {:?}", cr);
 
-                        let guid_key = SMBCommonHdr::from2(r, SMBHDR_TYPE_FILENAME);
+                        let guid_key = SMBCommonHdr::from2_notree(r, SMBHDR_TYPE_FILENAME);
                         if let Some(mut p) = state.ssn2vec_map.remove(&guid_key) {
                             p.retain(|&i|i != 0x00);
                             state.guid2name_map.insert(cr.guid.to_vec(), p);

src/app-layer-smtp.c

@@ -446,8 +446,9 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
                         (uint8_t *) chunk, len, flags) != 0) {
                 ret = MIME_DEC_ERR_DATA;
                 SCLogDebug("FileOpenFile() failed");
+            } else {
+                SMTPNewFile(smtp_state->curr_tx, files->tail);
             }
-            SMTPNewFile(smtp_state->curr_tx, files->tail);
 
             /* If close in the same chunk, then pass in empty bytes */
             if (state->body_end) {

src/decode.h

@@ -887,27 +887,17 @@ void CaptureStatsSetup(ThreadVars *tv, CaptureStats *s);
  * handle the case of a root packet
  * for tunnels */
 
-#define PACKET_SET_ACTION(p, a) do { \
-    ((p)->root ? \
-     ((p)->root->action = a) : \
-     ((p)->action = a)); \
-} while (0)
-
-#define PACKET_ALERT(p) PACKET_SET_ACTION(p, ACTION_ALERT)
-
-#define PACKET_ACCEPT(p) PACKET_SET_ACTION(p, ACTION_ACCEPT)
-
-#define PACKET_PASS(p) PACKET_SET_ACTION(p, ACTION_PASS)
-
-#define PACKET_TEST_ACTION_DO(p, a) (p)->action &(a)
+static inline void PacketSetActionOnCurrentPkt(Packet *p, const uint8_t action)
+{
+    p->action |= action;
+}
 
-#define PACKET_UPDATE_ACTION(p, a) (p)->action |= (a)
-static inline void PacketUpdateAction(Packet *p, const uint8_t a)
+static inline void PacketSetActionOnRealPkt(Packet *p, const uint8_t action)
 {
     if (likely(p->root == NULL)) {
-        PACKET_UPDATE_ACTION(p, a);
+        p->action |= action;
     } else {
-        PACKET_UPDATE_ACTION(p->root, a);
+        p->root->action |= action;
     }
 }
 
@@ -916,19 +906,29 @@ static inline void PacketDrop(Packet *p, const uint8_t action, enum PacketDropRe
     if (p->drop_reason == PKT_DROP_REASON_NOT_SET)
         p->drop_reason = (uint8_t)r;
 
-    PacketUpdateAction(p, action);
+    if (likely(p->root == NULL)) {
+        p->action |= action;
+    } else {
+        p->root->action |= action;
+    }
+}
+
+static inline uint8_t PacketTestActionOnCurrentPkt(const Packet *p, const uint8_t a)
+{
+    return (p->action & a);
 }
-#define PACKET_DROP(p) PacketDrop((p), PKT_DROP_REASON_NOT_SET)
 
-static inline uint8_t PacketTestAction(const Packet *p, const uint8_t a)
+static inline uint8_t PacketTestActionOnRealPkt(const Packet *p, const uint8_t a)
 {
     if (likely(p->root == NULL)) {
-        return PACKET_TEST_ACTION_DO(p, a);
+        return (p->action & a);
     } else {
-        return PACKET_TEST_ACTION_DO(p->root, a);
+        return (p->root->action & a);
     }
 }
-#define PACKET_TEST_ACTION(p, a) PacketTestAction((p), (a))
+
+// Tests on "real" packets. Should only be used to check for ACTION_DROP|ACTION_REJECT*
+#define PACKET_TEST_ACTION(p, a) PacketTestActionOnRealPkt((p), (a))
 
 #define TUNNEL_INCR_PKT_RTV_NOLOCK(p) do {                                          \
         ((p)->root ? (p)->root->tunnel_rtv_cnt++ : (p)->tunnel_rtv_cnt++);          \