From b4e1f53e9f823b5aa5e7e13daa309bfdb50b75ff Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sun, 26 May 2024 08:34:35 +0200 Subject: [PATCH 01/12] app-layer: minor code clarification 'dir' was too generic, so indicate it's about the app-layer update direction. --- src/app-layer.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/app-layer.c b/src/app-layer.c index a7e50d66d024..0cf8405eab0e 100644 --- a/src/app-layer.c +++ b/src/app-layer.c @@ -389,7 +389,7 @@ extern enum ExceptionPolicy g_applayerparser_error_policy; */ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, AppLayerThreadCtx *app_tctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, - uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir dir) + uint8_t *data, uint32_t data_len, uint8_t flags, enum StreamUpdateDir app_update_dir) { AppProto *alproto; AppProto *alproto_otherdir; @@ -555,7 +555,7 @@ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, int r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto, flags, data, data_len); PACKET_PROFILING_APP_END(app_tctx, f->alproto); - p->app_update_direction = (uint8_t)dir; + p->app_update_direction = (uint8_t)app_update_dir; if (r != 1) { StreamTcpUpdateAppLayerProgress(ssn, direction, data_len); } @@ -643,7 +643,7 @@ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, f->alproto, flags, data, data_len); PACKET_PROFILING_APP_END(app_tctx, f->alproto); - p->app_update_direction = (uint8_t)dir; + p->app_update_direction = (uint8_t)app_update_dir; if (r != 1) { StreamTcpUpdateAppLayerProgress(ssn, direction, data_len); } @@ -706,7 +706,7 @@ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, */ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet *p, Flow *f, TcpSession *ssn, TcpStream **stream, uint8_t *data, uint32_t data_len, uint8_t flags, - enum StreamUpdateDir dir) + enum StreamUpdateDir app_update_dir) { SCEnter(); @@ -752,7 +752,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto, flags, data, data_len); PACKET_PROFILING_APP_END(app_tctx, f->alproto); - p->app_update_direction = (uint8_t)dir; + p->app_update_direction = (uint8_t)app_update_dir; /* ignore parser result for gap */ StreamTcpUpdateAppLayerProgress(ssn, direction, data_len); if (r < 0) { @@ -771,8 +771,8 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet if (alproto == ALPROTO_UNKNOWN && (flags & STREAM_START)) { DEBUG_VALIDATE_BUG_ON(FlowChangeProto(f)); /* run protocol detection */ - if (TCPProtoDetect(tv, ra_ctx, app_tctx, p, f, ssn, stream, data, data_len, flags, dir) != - 0) { + if (TCPProtoDetect(tv, ra_ctx, app_tctx, p, f, ssn, stream, data, data_len, flags, + app_update_dir) != 0) { goto failure; } } else if (alproto != ALPROTO_UNKNOWN && FlowChangeProto(f)) { @@ -784,8 +784,8 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet StreamTcpResetStreamFlagAppProtoDetectionCompleted(&ssn->client); StreamTcpResetStreamFlagAppProtoDetectionCompleted(&ssn->server); /* rerun protocol detection */ - int rd = - TCPProtoDetect(tv, ra_ctx, app_tctx, p, f, ssn, stream, data, data_len, flags, dir); + int rd = TCPProtoDetect( + tv, ra_ctx, app_tctx, p, f, ssn, stream, data, data_len, flags, app_update_dir); if (f->alproto == ALPROTO_UNKNOWN) { DEBUG_VALIDATE_BUG_ON(alstate_orig != f->alstate); // not enough data, revert AppLayerProtoDetectReset to rerun detection @@ -838,7 +838,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto, flags, data, data_len); PACKET_PROFILING_APP_END(app_tctx, f->alproto); - p->app_update_direction = (uint8_t)dir; + p->app_update_direction = (uint8_t)app_update_dir; if (r != 1) { StreamTcpUpdateAppLayerProgress(ssn, direction, data_len); if (r < 0) { From c712cae1af381d92ca2404ae2c91656236d0b161 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sun, 26 May 2024 08:35:05 +0200 Subject: [PATCH 02/12] stream: minor code clarification 'dir' was too generic, so indicate it's about the app-layer update direction. --- src/stream-tcp-reassemble.c | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index 97c2408912ee..c310cd0e3df3 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -1214,10 +1214,8 @@ static inline uint32_t AdjustToAcked(const Packet *p, * \param stream pointer to pointer as app-layer can switch flow dir * \retval 0 success */ -static int ReassembleUpdateAppLayer (ThreadVars *tv, - TcpReassemblyThreadCtx *ra_ctx, - TcpSession *ssn, TcpStream **stream, - Packet *p, enum StreamUpdateDir dir) +static int ReassembleUpdateAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, + TcpStream **stream, Packet *p, enum StreamUpdateDir app_update_dir) { uint64_t app_progress = STREAM_APP_PROGRESS(*stream); @@ -1249,7 +1247,7 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv, SCLogDebug("sending GAP to app-layer (size: %u)", mydata_len); int r = AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, NULL, mydata_len, - StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, dir); + StreamGetAppLayerFlags(ssn, *stream, p) | STREAM_GAP, app_update_dir); AppLayerProfilingStore(ra_ctx->app_tctx, p); StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP); @@ -1321,8 +1319,8 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv, SCLogDebug("parser"); /* update the app-layer */ - (void)AppLayerHandleTCPData( - tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata, mydata_len, flags, dir); + (void)AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, (uint8_t *)mydata, + mydata_len, flags, app_update_dir); AppLayerProfilingStore(ra_ctx->app_tctx, p); AppLayerFrameDump(p->flow); uint64_t new_app_progress = STREAM_APP_PROGRESS(*stream); @@ -1348,9 +1346,8 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv, * any issues, since processing of each stream is independent of the * other stream. */ -int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, - TcpSession *ssn, TcpStream *stream, - Packet *p, enum StreamUpdateDir dir) +int StreamTcpReassembleAppLayer(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, TcpSession *ssn, + TcpStream *stream, Packet *p, enum StreamUpdateDir app_update_dir) { SCEnter(); @@ -1376,7 +1373,7 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, SCLogDebug("sending empty eof message"); /* send EOF to app layer */ AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, &stream, NULL, 0, - StreamGetAppLayerFlags(ssn, stream, p), dir); + StreamGetAppLayerFlags(ssn, stream, p), app_update_dir); AppLayerProfilingStore(ra_ctx->app_tctx, p); SCReturnInt(0); @@ -1384,7 +1381,7 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, } /* with all that out of the way, lets update the app-layer */ - return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, dir); + return ReassembleUpdateAppLayer(tv, ra_ctx, ssn, &stream, p, app_update_dir); } /** \internal From 492b885b32fd90ffe91029f14b3d9f914539daec Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 3 Jun 2024 10:28:44 +0200 Subject: [PATCH 03/12] frames: fix bounds check --- src/app-layer-frames.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c index 0aa0e19b82ac..b8554f59a982 100644 --- a/src/app-layer-frames.c +++ b/src/app-layer-frames.c @@ -427,7 +427,7 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice, if (f->proto == IPPROTO_TCP && f->protoctx == NULL) return NULL; if (frame_start < stream_slice->input || - frame_start >= stream_slice->input + stream_slice->input_len) + frame_start > stream_slice->input + stream_slice->input_len) return NULL; #endif BUG_ON(frame_start < stream_slice->input); From 584d18d020dfb3ea09ebc39780e8e77d62aaa6cd Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sun, 12 Nov 2023 09:41:13 +0100 Subject: [PATCH 04/12] frames: add FrameGetLastOpenByType Getter for the most recent frame with unknown length (-1). --- src/app-layer-frames.c | 28 +++++++++++++++++++++++++++- src/app-layer-frames.h | 1 + 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c index b8554f59a982..6953d26c9a45 100644 --- a/src/app-layer-frames.c +++ b/src/app-layer-frames.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2022 Open Information Security Foundation +/* Copyright (C) 2007-2024 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -83,6 +83,32 @@ static void FrameDebug(const char *prefix, const Frames *frames, const Frame *fr #endif } +/** + * \note "open" means a frame that has no length set (len == -1) + * \todo perhaps we can search backwards */ +Frame *FrameGetLastOpenByType(Frames *frames, const uint8_t frame_type) +{ + Frame *candidate = NULL; + + SCLogDebug( + "frames %p cnt %u, looking for last of type %" PRIu8, frames, frames->cnt, frame_type); + for (uint16_t i = 0; i < frames->cnt; i++) { + if (i < FRAMES_STATIC_CNT) { + Frame *frame = &frames->sframes[i]; + FrameDebug("get_by_id(static)", frames, frame); + if (frame->type == frame_type && frame->len == -1) + candidate = frame; + } else { + const uint16_t o = i - FRAMES_STATIC_CNT; + Frame *frame = &frames->dframes[o]; + FrameDebug("get_by_id(dynamic)", frames, frame); + if (frame->type == frame_type && frame->len == -1) + candidate = frame; + } + } + return candidate; +} + Frame *FrameGetById(Frames *frames, const int64_t id) { SCLogDebug("frames %p cnt %u, looking for %" PRIi64, frames, frames->cnt, id); diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h index 1904917b42c2..b78077fd2b1d 100644 --- a/src/app-layer-frames.h +++ b/src/app-layer-frames.h @@ -88,6 +88,7 @@ void AppLayerFrameDump(Flow *f); Frame *FrameGetByIndex(Frames *frames, const uint32_t idx); Frame *FrameGetById(Frames *frames, const int64_t id); +Frame *FrameGetLastOpenByType(Frames *frames, const uint8_t frame_type); Frame *AppLayerFrameGetById(Flow *f, const int direction, const FrameId frame_id); FrameId AppLayerFrameGetId(Frame *r); From 02d1c1349f0fd6a82656c8e2244a66f77053d672 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sun, 12 Nov 2023 09:41:45 +0100 Subject: [PATCH 05/12] app-layer/frames: add by type getter AppLayerFrameGetLastOpenByType: Returns the most recent frame with a type with unknown length (-1). Check if type is globally enabled first. --- src/app-layer-frames.c | 21 +++++++++++++++++++++ src/app-layer-frames.h | 3 +++ 2 files changed, 24 insertions(+) diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c index 6953d26c9a45..ff263b210f95 100644 --- a/src/app-layer-frames.c +++ b/src/app-layer-frames.c @@ -692,6 +692,27 @@ Frame *AppLayerFrameGetById(Flow *f, const int dir, const FrameId frame_id) return FrameGetById(frames, frame_id); } +Frame *AppLayerFrameGetLastOpenByType(Flow *f, const int dir, const uint8_t frame_type) +{ + if (!(FrameConfigTypeIsEnabled(f->alproto, frame_type))) + return NULL; + + FramesContainer *frames_container = AppLayerFramesGetContainer(f); + SCLogDebug("get frame_type %" PRIu8 " direction %u/%s frames_container %p", frame_type, dir, + dir == 0 ? "toserver" : "toclient", frames_container); + if (frames_container == NULL) + return NULL; + + Frames *frames; + if (dir == 0) { + frames = &frames_container->toserver; + } else { + frames = &frames_container->toclient; + } + SCLogDebug("frames %p", frames); + return FrameGetLastOpenByType(frames, frame_type); +} + static inline bool FrameIsDone(const Frame *frame, const uint64_t abs_right_edge) { /* frame with negative length means we don't know the size yet. */ diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h index b78077fd2b1d..2eb314331674 100644 --- a/src/app-layer-frames.h +++ b/src/app-layer-frames.h @@ -91,7 +91,10 @@ Frame *FrameGetById(Frames *frames, const int64_t id); Frame *FrameGetLastOpenByType(Frames *frames, const uint8_t frame_type); Frame *AppLayerFrameGetById(Flow *f, const int direction, const FrameId frame_id); +Frame *AppLayerFrameGetLastOpenByType(Flow *f, const int direction, const uint8_t frame_type); + FrameId AppLayerFrameGetId(Frame *r); + void AppLayerFrameAddEvent(Frame *frame, uint8_t e); void AppLayerFrameAddEventById(Flow *f, const int dir, const FrameId id, uint8_t e); void AppLayerFrameSetLength(Frame *frame, int64_t len); From f8cab57fc60044da60a3944bfc49da5d7c28abba Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 30 Nov 2023 11:59:45 +0100 Subject: [PATCH 06/12] detect/frames: avoid IPS rescanning Make sure to only scan the data when the app layer has been updated as well. Ticket: #6718. --- src/detect.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/detect.c b/src/detect.c index 989f1133daeb..7c7536a22709 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1627,6 +1627,15 @@ static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngin const SigGroupHead *const sgh = scratch->sgh; const AppProto alproto = f->alproto; + /* for TCP, limit inspection to pseudo packets or real packet that did + * an app-layer update. */ + if (p->proto == IPPROTO_TCP && !PKT_IS_PSEUDOPKT(p) && + ((PKT_IS_TOSERVER(p) && (f->flags & FLOW_TS_APP_UPDATED) == 0) || + (PKT_IS_TOCLIENT(p) && (f->flags & FLOW_TC_APP_UPDATED) == 0))) { + SCLogDebug("pcap_cnt %" PRIu64 ": %s: skip frame inspection for TCP w/o APP UPDATE", + p->pcap_cnt, PKT_IS_TOSERVER(p) ? "toserver" : "toclient"); + return; + } FramesContainer *frames_container = AppLayerFramesGetContainer(f); if (frames_container == NULL) { return; From 5510255a9d89f0575d438c1f4391e522b166d948 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 26 Jan 2024 14:36:16 +0100 Subject: [PATCH 07/12] app-layer: flag flow for next packet in other dir Add new flags to trigger FLOW_TS_APP_UPDATED/FLOW_TC_APP_UPDATED flags to be set for the next packet in the relevant direction. This allows for app relevant work to be done in the next packet in our direction. --- src/flow-worker.c | 16 ++++++++++++---- src/flow.h | 6 +++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/flow-worker.c b/src/flow-worker.c index fdc584df5d55..1f219c83ad5a 100644 --- a/src/flow-worker.c +++ b/src/flow-worker.c @@ -523,19 +523,19 @@ static void PacketAppUpdate2FlowFlags(Packet *p) break; case UPDATE_DIR_BOTH: if (PKT_IS_TOSERVER(p)) { - p->flow->flags |= FLOW_TS_APP_UPDATED; + p->flow->flags |= FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATE_NEXT; SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED set", p->pcap_cnt); } else { - p->flow->flags |= FLOW_TC_APP_UPDATED; + p->flow->flags |= FLOW_TC_APP_UPDATED | FLOW_TS_APP_UPDATE_NEXT; SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED set", p->pcap_cnt); } /* fall through */ case UPDATE_DIR_OPPOSING: if (PKT_IS_TOSERVER(p)) { - p->flow->flags |= FLOW_TC_APP_UPDATED; + p->flow->flags |= FLOW_TC_APP_UPDATED | FLOW_TS_APP_UPDATE_NEXT; SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED set", p->pcap_cnt); } else { - p->flow->flags |= FLOW_TS_APP_UPDATED; + p->flow->flags |= FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATE_NEXT; SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED set", p->pcap_cnt); } break; @@ -583,6 +583,14 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data) /* handle TCP and app layer */ if (p->flow) { + if (PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATE_NEXT)) { + p->flow->flags |= FLOW_TS_APP_UPDATED; + p->flow->flags &= ~FLOW_TS_APP_UPDATE_NEXT; + } else if (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATE_NEXT)) { + p->flow->flags |= FLOW_TC_APP_UPDATED; + p->flow->flags &= ~FLOW_TC_APP_UPDATE_NEXT; + } + if (PacketIsTCP(p)) { SCLogDebug("packet %" PRIu64 " is TCP. Direction %s", p->pcap_cnt, PKT_IS_TOSERVER(p) ? "TOSERVER" : "TOCLIENT"); diff --git a/src/flow.h b/src/flow.h index d633554243c0..bf28d02a5812 100644 --- a/src/flow.h +++ b/src/flow.h @@ -52,7 +52,8 @@ typedef struct AppLayerParserState_ AppLayerParserState; /** At least one packet from the destination address was seen */ #define FLOW_TO_DST_SEEN BIT_U32(1) -// vacancy +/** next packet in toclient direction will act on updated app-layer state */ +#define FLOW_TC_APP_UPDATE_NEXT BIT_U32(2) /** Flow was inspected against IP-Only sigs in the toserver direction */ #define FLOW_TOSERVER_IPONLY_SET BIT_U32(3) @@ -117,6 +118,9 @@ typedef struct AppLayerParserState_ AppLayerParserState; #define FLOW_TS_APP_UPDATED BIT_U32(29) #define FLOW_TC_APP_UPDATED BIT_U32(30) +/** next packet in toserver direction will act on updated app-layer state */ +#define FLOW_TS_APP_UPDATE_NEXT BIT_U32(31) + /* File flags */ #define FLOWFILE_INIT 0 From 8ddd6335a4d12232f1db8d8cf61318c9750235b2 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 26 Jan 2024 15:11:30 +0100 Subject: [PATCH 08/12] detect/frames: inspect frames only in correct direction Inspect frames in the correct direction after they have been created. --- src/flow-worker.c | 21 ++++++++++++++++----- src/output-json-frame.c | 10 ++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/flow-worker.c b/src/flow-worker.c index 1f219c83ad5a..9af47ac7c818 100644 --- a/src/flow-worker.c +++ b/src/flow-worker.c @@ -524,19 +524,23 @@ static void PacketAppUpdate2FlowFlags(Packet *p) case UPDATE_DIR_BOTH: if (PKT_IS_TOSERVER(p)) { p->flow->flags |= FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATE_NEXT; - SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED set", p->pcap_cnt); + SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED|FLOW_TC_APP_UPDATE_NEXT set", + p->pcap_cnt); } else { p->flow->flags |= FLOW_TC_APP_UPDATED | FLOW_TS_APP_UPDATE_NEXT; - SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED set", p->pcap_cnt); + SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED|FLOW_TS_APP_UPDATE_NEXT set", + p->pcap_cnt); } /* fall through */ case UPDATE_DIR_OPPOSING: if (PKT_IS_TOSERVER(p)) { p->flow->flags |= FLOW_TC_APP_UPDATED | FLOW_TS_APP_UPDATE_NEXT; - SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED set", p->pcap_cnt); + SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TC_APP_UPDATED|FLOW_TS_APP_UPDATE_NEXT set", + p->pcap_cnt); } else { p->flow->flags |= FLOW_TS_APP_UPDATED | FLOW_TC_APP_UPDATE_NEXT; - SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED set", p->pcap_cnt); + SCLogDebug("pcap_cnt %" PRIu64 ", FLOW_TS_APP_UPDATED|FLOW_TC_APP_UPDATE_NEXT set", + p->pcap_cnt); } break; } @@ -583,12 +587,15 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data) /* handle TCP and app layer */ if (p->flow) { + /* see if need to consider flags set by prev packets */ if (PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATE_NEXT)) { p->flow->flags |= FLOW_TS_APP_UPDATED; p->flow->flags &= ~FLOW_TS_APP_UPDATE_NEXT; + SCLogDebug("FLOW_TS_APP_UPDATED"); } else if (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATE_NEXT)) { p->flow->flags |= FLOW_TC_APP_UPDATED; p->flow->flags &= ~FLOW_TC_APP_UPDATE_NEXT; + SCLogDebug("FLOW_TC_APP_UPDATED"); } if (PacketIsTCP(p)) { @@ -640,7 +647,11 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data) StreamTcpSessionCleanup(p->flow->protoctx); } } else if (p->proto == IPPROTO_TCP && p->flow->protoctx && p->flags & PKT_STREAM_EST) { - FramesPrune(p->flow, p); + if ((p->flow->flags & FLOW_TS_APP_UPDATED) && PKT_IS_TOSERVER(p)) { + FramesPrune(p->flow, p); + } else if ((p->flow->flags & FLOW_TC_APP_UPDATED) && PKT_IS_TOCLIENT(p)) { + FramesPrune(p->flow, p); + } FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE); StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ? STREAM_TOSERVER : STREAM_TOCLIENT); diff --git a/src/output-json-frame.c b/src/output-json-frame.c index 4e0ec5b2b24c..4f761e7ca173 100644 --- a/src/output-json-frame.c +++ b/src/output-json-frame.c @@ -409,6 +409,16 @@ static bool JsonFrameLogCondition(ThreadVars *tv, void *thread_data, const Packe return false; if ((p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) && p->flow->alparser != NULL) { + if (p->proto == IPPROTO_TCP) { + if ((p->flow->flags & FLOW_TS_APP_UPDATED) && PKT_IS_TOSERVER(p)) { + // fallthrough + } else if ((p->flow->flags & FLOW_TC_APP_UPDATED) && PKT_IS_TOCLIENT(p)) { + // fallthrough + } else { + return false; + } + } + FramesContainer *frames_container = AppLayerFramesGetContainer(p->flow); if (frames_container == NULL) return false; From d7ca456343cfbd3e35026e632167c346375fabb2 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sun, 26 May 2024 08:38:13 +0200 Subject: [PATCH 09/12] stream: process ASYNC in packet dir There will generally not be an opposing direction to handle the app update. --- src/stream-tcp-reassemble.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index c310cd0e3df3..b9e9e6797dd6 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -1983,6 +1983,9 @@ int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ } } else if (ssn->state == TCP_CLOSED) { dir = UPDATE_DIR_BOTH; + } else if ((ssn->flags & STREAMTCP_FLAG_ASYNC) != 0) { + dir = UPDATE_DIR_PACKET; + SCLogDebug("%" PRIu64 ": ASYNC: UPDATE_DIR_PACKET", p->pcap_cnt); } /* handle ack received */ From 87353dfd05c3497aa85a1f274020c2b0369bbedf Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sun, 26 May 2024 08:40:11 +0200 Subject: [PATCH 10/12] flow-worker: debug output about updates --- src/flow-worker.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/flow-worker.c b/src/flow-worker.c index 9af47ac7c818..3efb290da44c 100644 --- a/src/flow-worker.c +++ b/src/flow-worker.c @@ -511,6 +511,7 @@ static void PacketAppUpdate2FlowFlags(Packet *p) { switch ((enum StreamUpdateDir)p->app_update_direction) { case UPDATE_DIR_NONE: // NONE implies pseudo packet + SCLogDebug("pcap_cnt %" PRIu64 ", UPDATE_DIR_NONE", p->pcap_cnt); break; case UPDATE_DIR_PACKET: if (PKT_IS_TOSERVER(p)) { @@ -587,6 +588,11 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data) /* handle TCP and app layer */ if (p->flow) { + SCLogDebug("packet %" PRIu64 + ": direction %s FLOW_TS_APP_UPDATE_NEXT %s FLOW_TC_APP_UPDATE_NEXT %s", + p->pcap_cnt, PKT_IS_TOSERVER(p) ? "toserver" : "toclient", + BOOL2STR((p->flow->flags & FLOW_TS_APP_UPDATE_NEXT) != 0), + BOOL2STR((p->flow->flags & FLOW_TC_APP_UPDATE_NEXT) != 0)); /* see if need to consider flags set by prev packets */ if (PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATE_NEXT)) { p->flow->flags |= FLOW_TS_APP_UPDATED; @@ -667,11 +673,13 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data) if (PKT_IS_PSEUDOPKT(p) || (p->flow->flags & (FLOW_TS_APP_UPDATED))) { AppLayerParserTransactionsCleanup(p->flow, STREAM_TOSERVER); p->flow->flags &= ~FLOW_TS_APP_UPDATED; + SCLogDebug("~FLOW_TS_APP_UPDATED"); } } else { if (PKT_IS_PSEUDOPKT(p) || (p->flow->flags & (FLOW_TC_APP_UPDATED))) { AppLayerParserTransactionsCleanup(p->flow, STREAM_TOCLIENT); p->flow->flags &= ~FLOW_TC_APP_UPDATED; + SCLogDebug("~FLOW_TC_APP_UPDATED"); } } } From f454c37574e3bae65fa5a54eee8c44a851f41653 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 13 Nov 2023 06:43:32 +0100 Subject: [PATCH 11/12] smtp/frames: initial frame support Adds the following frames: command_line data response_line The *_line frames are per line, so in multi-line responses each line will have it's own frame. Ticket: #4905. --- src/app-layer-smtp.c | 122 ++++++++++++++++++++++++++++++++++++++----- src/app-layer-smtp.h | 1 + 2 files changed, 109 insertions(+), 14 deletions(-) diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 3558ff0009c5..981f08266512 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -35,6 +35,7 @@ #include "app-layer-detect-proto.h" #include "app-layer-protos.h" #include "app-layer-parser.h" +#include "app-layer-frames.h" #include "app-layer-smtp.h" #include "util-enum.h" @@ -154,6 +155,43 @@ SCEnumCharMap smtp_decoder_event_table[] = { { NULL, -1 }, }; +enum SMTPFrameTypes { + SMTP_FRAME_COMMAND_LINE, + SMTP_FRAME_DATA, + SMTP_FRAME_RESPONSE_LINE, +}; + +SCEnumCharMap smtp_frame_table[] = { + { + "command_line", + SMTP_FRAME_COMMAND_LINE, + }, + { + "data", + SMTP_FRAME_DATA, + }, + { + "response_line", + SMTP_FRAME_RESPONSE_LINE, + }, + { NULL, -1 }, +}; + +static int SMTPGetFrameIdByName(const char *frame_name) +{ + int id = SCMapEnumNameToValue(frame_name, smtp_frame_table); + if (id < 0) { + return -1; + } + return id; +} + +static const char *SMTPGetFrameNameById(const uint8_t frame_id) +{ + const char *name = SCMapEnumValueToName(frame_id, smtp_frame_table); + return name; +} + typedef struct SMTPThreadCtx_ { MpmThreadCtx *smtp_mpm_thread_ctx; PrefilterRuleStore *pmq; @@ -646,8 +684,8 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, * \retval -1 Either when we don't have any new lines to supply anymore or * on failure. */ -static AppLayerResult SMTPGetLine( - SMTPState *state, SMTPInput *input, SMTPLine *line, uint16_t direction) +static AppLayerResult SMTPGetLine(Flow *f, StreamSlice *slice, SMTPState *state, SMTPInput *input, + SMTPLine *line, uint16_t direction) { SCEnter(); @@ -655,6 +693,26 @@ static AppLayerResult SMTPGetLine( if (input->len <= 0) return APP_LAYER_ERROR; + const uint8_t type = direction == 0 ? SMTP_FRAME_COMMAND_LINE : SMTP_FRAME_RESPONSE_LINE; + Frame *frame = AppLayerFrameGetLastOpenByType(f, direction, type); + if (frame == NULL) { + if (direction == 0 && + !(state->current_command == SMTP_COMMAND_DATA && + (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE))) { + frame = AppLayerFrameNewByPointer( + f, slice, input->buf + input->consumed, -1, 0, SMTP_FRAME_COMMAND_LINE); + /* can't set tx id before (possibly) creating it */ + + } else if (direction == 1) { + frame = AppLayerFrameNewByPointer( + f, slice, input->buf + input->consumed, -1, 1, SMTP_FRAME_RESPONSE_LINE); + if (frame != NULL && state->curr_tx) { + AppLayerFrameSetTxId(frame, state->curr_tx->tx_id); + } + } + } + SCLogDebug("frame %p", frame); + uint8_t *lf_idx = memchr(input->buf + input->consumed, 0x0a, input->len); bool discard_till_lf = (direction == 0) ? state->discard_till_lf_ts : state->discard_till_lf_tc; @@ -681,6 +739,11 @@ static AppLayerResult SMTPGetLine( input->len -= line->len; DEBUG_VALIDATE_BUG_ON((input->consumed + input->len) != input->orig_len); line->buf = input->buf + o_consumed; + + if (frame != NULL) { + frame->len = (int64_t)line->len; + } + if (line->len >= SMTP_LINE_BUFFER_LIMIT) { line->len = SMTP_LINE_BUFFER_LIMIT; line->delim_len = 0; @@ -1170,11 +1233,22 @@ static int NoNewTx(SMTPState *state, const SMTPLine *line) * -2 if MIME state could not be allocated * */ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *pstate, - SMTPInput *input, const SMTPLine *line) + SMTPInput *input, const SMTPLine *line, const StreamSlice *slice) { SCEnter(); SMTPTransaction *tx = state->curr_tx; + Frame *frame = AppLayerFrameGetLastOpenByType(f, 0, SMTP_FRAME_COMMAND_LINE); + if (frame) { + frame->len = (int64_t)line->len; + } else { + if (!(state->current_command == SMTP_COMMAND_DATA && + (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE))) { + frame = AppLayerFrameNewByPointer( + f, slice, line->buf, line->len, 0, SMTP_FRAME_COMMAND_LINE); + } + } + /* If current input is to be discarded because it completes a long line, * line's length and delimiter len are reset to 0. Skip processing this line. * This line is only to get us out of the state where we should discard any @@ -1195,6 +1269,9 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, smtp_config.content_inspect_min_size); } + if (frame != NULL && state->curr_tx) { + AppLayerFrameSetTxId(frame, state->curr_tx->tx_id); + } state->toserver_data_count += (line->len + line->delim_len); @@ -1240,6 +1317,15 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps } } state->curr_tx->is_data = true; + + Frame *data_frame = AppLayerFrameNewByPointer( + f, slice, input->buf + input->consumed, -1, 0, SMTP_FRAME_DATA); + if (data_frame == NULL) { + SCLogDebug("data_frame %p - no data frame set up", data_frame); + } else { + AppLayerFrameSetTxId(data_frame, state->curr_tx->tx_id); + } + /* Enter immediately data mode without waiting for server reply */ if (state->parser_state & SMTP_PARSER_STATE_PIPELINING_SERVER) { state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE; @@ -1334,8 +1420,8 @@ static inline void ResetLine(SMTPLine *line) * 1 for handing control over to GetLine * -1 for errors and inconsistent states * */ -static int SMTPPreProcessCommands( - SMTPState *state, Flow *f, AppLayerParserState *pstate, SMTPInput *input, SMTPLine *line) +static int SMTPPreProcessCommands(SMTPState *state, Flow *f, AppLayerParserState *pstate, + StreamSlice *slice, SMTPInput *input, SMTPLine *line) { DEBUG_VALIDATE_BUG_ON((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0); DEBUG_VALIDATE_BUG_ON(line->len != 0); @@ -1384,10 +1470,11 @@ static int SMTPPreProcessCommands( if (line->len < 0) { return -1; } + input->consumed = total_consumed; input->len -= current_line_consumed; DEBUG_VALIDATE_BUG_ON(input->consumed + input->len != input->orig_len); - if (SMTPProcessRequest(state, f, pstate, input, line) == -1) { + if (SMTPProcessRequest(state, f, pstate, input, line, slice) == -1) { return -1; } line_complete = false; @@ -1396,8 +1483,13 @@ static int SMTPPreProcessCommands( line->delim_len = 0; /* bail if `SMTPProcessRequest` ended the data mode */ - if ((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0) + if ((state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE) == 0) { + Frame *data_frame = AppLayerFrameGetLastOpenByType(f, 0, SMTP_FRAME_DATA); + if (data_frame) { + data_frame->len = (slice->offset + input->consumed) - data_frame->offset; + } break; + } } } return 0; @@ -1428,7 +1520,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state, if (((state->current_command == SMTP_COMMAND_DATA) || (state->current_command == SMTP_COMMAND_BDAT)) && (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { - int ret = SMTPPreProcessCommands(state, f, pstate, &input, &line); + int ret = SMTPPreProcessCommands(state, f, pstate, &stream_slice, &input, &line); DEBUG_VALIDATE_BUG_ON(ret != 0 && ret != -1 && ret != 1); if (ret == 0 && input.consumed == input.orig_len) { SCReturnStruct(APP_LAYER_OK); @@ -1436,9 +1528,9 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state, SCReturnStruct(APP_LAYER_ERROR); } } - AppLayerResult res = SMTPGetLine(state, &input, &line, direction); + AppLayerResult res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction); while (res.status == 0) { - int retval = SMTPProcessRequest(state, f, pstate, &input, &line); + int retval = SMTPProcessRequest(state, f, pstate, &input, &line, &stream_slice); if (retval != 0) SCReturnStruct(APP_LAYER_ERROR); if (line.delim_len == 0 && line.len == SMTP_LINE_BUFFER_LIMIT) { @@ -1459,7 +1551,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state, * In case of another boundary, the control should be passed to SMTPGetLine */ if ((input.len > 0) && (state->current_command == SMTP_COMMAND_DATA) && (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { - int ret = SMTPPreProcessCommands(state, f, pstate, &input, &line); + int ret = SMTPPreProcessCommands(state, f, pstate, &stream_slice, &input, &line); DEBUG_VALIDATE_BUG_ON(ret != 0 && ret != -1 && ret != 1); if (ret == 0 && input.consumed == input.orig_len) { SCReturnStruct(APP_LAYER_OK); @@ -1467,13 +1559,13 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state, SCReturnStruct(APP_LAYER_ERROR); } } - res = SMTPGetLine(state, &input, &line, direction); + res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction); } if (res.status == 1) return res; /* toclient */ } else { - AppLayerResult res = SMTPGetLine(state, &input, &line, direction); + AppLayerResult res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction); while (res.status == 0) { if (SMTPProcessReply(state, f, pstate, thread_data, &input, &line) != 0) SCReturnStruct(APP_LAYER_ERROR); @@ -1485,7 +1577,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state, SMTPSetEvent(state, SMTP_DECODER_EVENT_TRUNCATED_LINE); break; } - res = SMTPGetLine(state, &input, &line, direction); + res = SMTPGetLine(f, &stream_slice, state, &input, &line, direction); } if (res.status == 1) return res; @@ -1887,6 +1979,8 @@ void RegisterSMTPParsers(void) AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxData); AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetStateData); AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_SMTP, 1, 1); + AppLayerParserRegisterGetFrameFuncs( + IPPROTO_TCP, ALPROTO_SMTP, SMTPGetFrameIdByName, SMTPGetFrameNameById); } else { SCLogInfo("Parser disabled for %s protocol. Protocol detection still on.", proto_name); } diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h index 37369af034f3..ad0334f86ae6 100644 --- a/src/app-layer-smtp.h +++ b/src/app-layer-smtp.h @@ -24,6 +24,7 @@ #ifndef SURICATA_APP_LAYER_SMTP_H #define SURICATA_APP_LAYER_SMTP_H +#include "app-layer-frames.h" #include "util-decode-mime.h" #include "util-streaming-buffer.h" #include "rust.h" From ec743f01ef92eb7722ad062d0d0029c596386732 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 3 Jun 2024 18:24:57 +0200 Subject: [PATCH 12/12] github-action: remove end of life CentOS 8 stream --- .github/workflows/builds.yml | 88 ------------------------------------ 1 file changed, 88 deletions(-) diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index cf296200dc73..b9a0ed64cb88 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -613,94 +613,6 @@ jobs: - run: make clean - run: make -j ${{ env.CPUS }} - centos-stream8: - name: CentOS Stream 8 - runs-on: ubuntu-latest - container: quay.io/centos/centos:stream8 - needs: [prepare-deps, debian-12-dist] - steps: - # Cache Rust stuff. - - name: Cache cargo registry - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 - with: - path: ~/.cargo - key: ${{ github.job }}-cargo - - - name: Cache RPMs - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 - with: - path: /var/cache/dnf - key: ${{ github.job }}-dnf - - run: echo "keepcache=1" >> /etc/dnf/dnf.conf - - - name: Determine number of CPUs - run: echo CPUS=$(nproc --all) >> $GITHUB_ENV - - - name: Install system packages - run: | - dnf -y install dnf-plugins-core epel-release - dnf config-manager --set-enabled powertools - dnf -y install \ - autoconf \ - automake \ - diffutils \ - numactl-devel \ - dpdk-devel \ - file-devel \ - gcc \ - gcc-c++ \ - git \ - jansson-devel \ - jq \ - libtool \ - libyaml-devel \ - libnfnetlink-devel \ - libnetfilter_queue-devel \ - libnet-devel \ - libcap-ng-devel \ - libevent-devel \ - libmaxminddb-devel \ - libpcap-devel \ - libtool \ - lz4-devel \ - make \ - pcre2-devel \ - pkgconfig \ - python3-devel \ - python3-yaml \ - rust-toolset \ - sudo \ - which \ - zlib-devel - - name: Download suricata.tar.gz - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e - with: - name: dist - - run: tar zxvf suricata-*.tar.gz --strip-components=1 - - name: ./configure - run: CFLAGS="${DEFAULT_CFLAGS}" ./configure - - run: make -j ${{ env.CPUS }} - - run: make install - - run: make install-conf - - run: suricatasc -h - - run: suricata-update -V - - name: Check if Suricata-Update example configuration files are installed - run: | - test -e /usr/local/lib/suricata/python/suricata/update/configs/disable.conf - test -e /usr/local/lib/suricata/python/suricata/update/configs/drop.conf - test -e /usr/local/lib/suricata/python/suricata/update/configs/enable.conf - test -e /usr/local/lib/suricata/python/suricata/update/configs/modify.conf - test -e /usr/local/lib/suricata/python/suricata/update/configs/threshold.in - test -e /usr/local/lib/suricata/python/suricata/update/configs/update.yaml - - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e - with: - name: prep - path: prep - - run: tar xf prep/suricata-verify.tar.gz - - run: python3 ./suricata-verify/run.py -q --debug-failed - - run: suricata-update -V - - run: suricatasc -h - fedora-39-sv-codecov: name: Fedora 39 (Suricata Verify codecov) runs-on: ubuntu-latest