diff --git a/doc/userguide/upgrade.rst b/doc/userguide/upgrade.rst index 6da52adb1f88..2cc195fd260b 100644 --- a/doc/userguide/upgrade.rst +++ b/doc/userguide/upgrade.rst @@ -39,7 +39,7 @@ Upgrading to 7.0.8 - Unknown requirements in the ``requires`` keyword will now be treated as unsatisfied requirements, causing the rule to not be loaded. See :ref:`keyword_requires`. To opt out of this change and to ignore - uknown requirements, effectively treating them as satified the + unknown requirements, effectively treating them as satisfied the ``ignore-unknown-requirements`` configuration option can be used. Command line example:: @@ -66,6 +66,13 @@ Upgrading to 7.0.8 the engine will NOT log any transaction metadata if there is more than one live transaction, to reduce the chances of logging unrelated data.** This may lead to what looks like a regression in behavior, but it is a considered choice. +- The configuration setting controlling stream checksum checks no longer affects + checksum keyword validation. In previous Suricata versions, when ``stream.checksum-validation`` + was set to ``no``, the checksum keywords (e.g., ``ipv4-csum``, ``tcpv4-csum``, etc) + will always consider it valid; e.g., ``tcpv4-csum: invalid`` will never match. Now, + ``stream.checksum-validation`` no longer affects the checksum rule keywords. + E.g., ``ipv4-csum: valid`` will only match if the check sum is valid, even when engine + checksum validations are disabled. Upgrading 6.0 to 7.0 -------------------- diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs index 551a125d9794..023d169585ca 100644 --- a/rust/src/applayer.rs +++ b/rust/src/applayer.rs @@ -98,6 +98,12 @@ pub struct AppLayerTxData { /// config: log flags pub config: AppLayerTxConfig, + /// The tx has been updated and needs to be processed : detection, logging, cleaning + /// It can then be skipped until new data arrives. + /// There is a boolean for both directions : to server and to client + pub updated_tc: bool, + pub updated_ts: bool, + /// logger flags for tx logging api logged: LoggerFlags, @@ -152,6 +158,8 @@ impl AppLayerTxData { files_stored: 0, file_flags: 0, file_tx: 0, + updated_tc: true, + updated_ts: true, detect_flags_ts: 0, detect_flags_tc: 0, de_state: std::ptr::null_mut(), @@ -162,9 +170,9 @@ impl AppLayerTxData { /// Create new AppLayerTxData for a transaction in a single /// direction. pub fn for_direction(direction: Direction) -> Self { - let (detect_flags_ts, detect_flags_tc) = match direction { - Direction::ToServer => (0, APP_LAYER_TX_SKIP_INSPECT_FLAG), - Direction::ToClient => (APP_LAYER_TX_SKIP_INSPECT_FLAG, 0), + let (detect_flags_ts, detect_flags_tc, updated_ts, updated_tc) = match direction { + Direction::ToServer => (0, APP_LAYER_TX_SKIP_INSPECT_FLAG, true, false), + Direction::ToClient => (APP_LAYER_TX_SKIP_INSPECT_FLAG, 0, false, true), }; Self { config: AppLayerTxConfig::new(), @@ -174,6 +182,8 @@ impl AppLayerTxData { files_stored: 0, file_flags: 0, file_tx: 0, + updated_tc, + updated_ts, detect_flags_ts, detect_flags_tc, de_state: std::ptr::null_mut(), diff --git a/rust/src/applayertemplate/template.rs b/rust/src/applayertemplate/template.rs index dbbc7841fad5..0fc6730b4e24 100644 --- a/rust/src/applayertemplate/template.rs +++ b/rust/src/applayertemplate/template.rs @@ -200,6 +200,7 @@ impl TemplateState { start = rem; if let Some(tx) = self.find_request() { + tx.tx_data.updated_tc = true; tx.response = Some(response); SCLogNotice!("Found response for request:"); SCLogNotice!("- Request: {:?}", tx.request); diff --git a/rust/src/dcerpc/dcerpc.rs b/rust/src/dcerpc/dcerpc.rs index 345d46656624..b99b23d3f562 100644 --- a/rust/src/dcerpc/dcerpc.rs +++ b/rust/src/dcerpc/dcerpc.rs @@ -361,6 +361,8 @@ impl DCERPCState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.req_done || !tx_old.resp_done { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.req_done = true; tx_old.resp_done = true; break; @@ -537,6 +539,8 @@ impl DCERPCState { } } } + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/dcerpc/dcerpc_udp.rs b/rust/src/dcerpc/dcerpc_udp.rs index 05d49973c864..b4ac255b266e 100644 --- a/rust/src/dcerpc/dcerpc_udp.rs +++ b/rust/src/dcerpc/dcerpc_udp.rs @@ -88,6 +88,8 @@ impl DCERPCUDPState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.req_done || !tx_old.resp_done { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.req_done = true; tx_old.resp_done = true; break; @@ -165,6 +167,8 @@ impl DCERPCUDPState { } if let Some(tx) = otx { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; let done = (hdr.flags1 & PFCL1_FRAG) == 0 || (hdr.flags1 & PFCL1_LASTFRAG) != 0; match hdr.pkt_type { diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index d679c99e3e82..eace93ccc36f 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -655,6 +655,8 @@ impl HTTP2State { let tx = &mut self.transactions[index - 1]; tx.tx_data.update_file_flags(self.state_data.file_flags); tx.update_file_flags(tx.tx_data.file_flags); + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } else { // do not use SETTINGS_MAX_CONCURRENT_STREAMS as it can grow too much @@ -667,6 +669,8 @@ impl HTTP2State { tx_old.set_event(HTTP2Event::TooManyStreams); // use a distinct state, even if we do not log it tx_old.state = HTTP2TransactionState::HTTP2StateTodrop; + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; } return None; } diff --git a/rust/src/modbus/modbus.rs b/rust/src/modbus/modbus.rs index b09c26b11894..80a3f16f0b03 100644 --- a/rust/src/modbus/modbus.rs +++ b/rust/src/modbus/modbus.rs @@ -124,6 +124,8 @@ impl ModbusState { for tx in &mut self.transactions { if let Some(req) = &tx.request { if tx.response.is_none() && resp.matches(req) { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -139,6 +141,8 @@ impl ModbusState { for tx in &mut self.transactions { if let Some(resp) = &tx.response { if tx.request.is_none() && req.matches(resp) { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -184,6 +188,8 @@ impl ModbusState { match self.find_response_and_validate(&mut msg) { Some(tx) => { tx.set_events_from_flags(&msg.error_flags); + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; tx.request = Some(msg); } None => { @@ -210,6 +216,8 @@ impl ModbusState { } else { tx.set_events_from_flags(&msg.error_flags); } + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; tx.response = Some(msg); } None => { diff --git a/rust/src/mqtt/mqtt.rs b/rust/src/mqtt/mqtt.rs index b31b6eacea5d..3fd35b30a960 100644 --- a/rust/src/mqtt/mqtt.rs +++ b/rust/src/mqtt/mqtt.rs @@ -174,6 +174,8 @@ impl MQTTState { if !tx.complete { if let Some(mpktid) = tx.pkt_id { if mpktid == pkt_id { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -196,6 +198,8 @@ impl MQTTState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.complete { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.complete = true; MQTTState::set_event(tx_old, MQTTEvent::TooManyTransactions); break; diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs index b472689a5104..98e6f5905787 100644 --- a/rust/src/nfs/nfs.rs +++ b/rust/src/nfs/nfs.rs @@ -431,6 +431,8 @@ impl NFSState { // set at least one another transaction to the drop state for tx_old in &mut self.transactions { if !tx_old.request_done || !tx_old.response_done { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.request_done = true; tx_old.response_done = true; tx_old.is_file_closed = true; @@ -500,6 +502,8 @@ impl NFSState { pub fn mark_response_tx_done(&mut self, xid: u32, rpc_status: u32, nfs_status: u32, resp_handle: &[u8]) { if let Some(mytx) = self.get_tx_by_xid(xid) { + mytx.tx_data.updated_tc = true; + mytx.tx_data.updated_ts = true; mytx.response_done = true; mytx.rpc_response_status = rpc_status; mytx.nfs_response_status = nfs_status; @@ -756,6 +760,8 @@ impl NFSState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid); + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/pgsql/pgsql.rs b/rust/src/pgsql/pgsql.rs index 4b2b9213f1a8..9bea2b877d67 100644 --- a/rust/src/pgsql/pgsql.rs +++ b/rust/src/pgsql/pgsql.rs @@ -209,6 +209,8 @@ impl PgsqlState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if tx_old.tx_res_state < PgsqlTxProgress::TxDone { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; // we don't check for TxReqDone for the majority of requests are basically completed // when they're parsed, as of now tx_old.tx_req_state = PgsqlTxProgress::TxFlushedOut; @@ -361,6 +363,7 @@ impl PgsqlState { // A simplified finite state machine for PostgreSQL v3 can be found at: // https://samadhiweb.com/blog/2013.04.28.graphviz.postgresv3.html if let Some(tx) = self.find_or_create_tx() { + tx.tx_data.updated_ts = true; tx.request = Some(request); if let Some(state) = new_state { if Self::request_is_complete(state) { @@ -519,6 +522,7 @@ impl PgsqlState { self.state_progress = state; } if let Some(tx) = self.find_or_create_tx() { + tx.tx_data.updated_tc = true; if tx.tx_res_state == PgsqlTxProgress::TxInit { tx.tx_res_state = PgsqlTxProgress::TxReceived; } diff --git a/rust/src/rfb/rfb.rs b/rust/src/rfb/rfb.rs index 8c3381345012..4d16c2261926 100644 --- a/rust/src/rfb/rfb.rs +++ b/rust/src/rfb/rfb.rs @@ -165,7 +165,13 @@ impl RFBState { fn get_current_tx(&mut self) -> Option<&mut RFBTransaction> { let tx_id = self.tx_id; - self.transactions.iter_mut().find(|tx| tx.tx_id == tx_id) + let r = self.transactions.iter_mut().find(|tx| tx.tx_id == tx_id); + if let Some(tx) = r { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; + return Some(tx); + } + return None; } fn parse_request(&mut self, flow: *const Flow, stream_slice: StreamSlice) -> AppLayerResult { diff --git a/rust/src/sip/detect.rs b/rust/src/sip/detect.rs index 63f636529a34..91df4fb29932 100644 --- a/rust/src/sip/detect.rs +++ b/rust/src/sip/detect.rs @@ -23,9 +23,7 @@ use std::ptr; #[no_mangle] pub unsafe extern "C" fn rs_sip_tx_get_method( - tx: &mut SIPTransaction, - buffer: *mut *const u8, - buffer_len: *mut u32, + tx: &mut SIPTransaction, buffer: *mut *const u8, buffer_len: *mut u32, ) -> u8 { if let Some(ref r) = tx.request { let m = &r.method; @@ -44,9 +42,7 @@ pub unsafe extern "C" fn rs_sip_tx_get_method( #[no_mangle] pub unsafe extern "C" fn rs_sip_tx_get_uri( - tx: &mut SIPTransaction, - buffer: *mut *const u8, - buffer_len: *mut u32, + tx: &mut SIPTransaction, buffer: *mut *const u8, buffer_len: *mut u32, ) -> u8 { if let Some(ref r) = tx.request { let p = &r.path; @@ -65,10 +61,7 @@ pub unsafe extern "C" fn rs_sip_tx_get_uri( #[no_mangle] pub unsafe extern "C" fn rs_sip_tx_get_protocol( - tx: &mut SIPTransaction, - buffer: *mut *const u8, - buffer_len: *mut u32, - direction: u8, + tx: &mut SIPTransaction, buffer: *mut *const u8, buffer_len: *mut u32, direction: u8, ) -> u8 { match direction.into() { Direction::ToServer => { @@ -101,9 +94,7 @@ pub unsafe extern "C" fn rs_sip_tx_get_protocol( #[no_mangle] pub unsafe extern "C" fn rs_sip_tx_get_stat_code( - tx: &mut SIPTransaction, - buffer: *mut *const u8, - buffer_len: *mut u32, + tx: &mut SIPTransaction, buffer: *mut *const u8, buffer_len: *mut u32, ) -> u8 { if let Some(ref r) = tx.response { let c = &r.code; @@ -122,9 +113,7 @@ pub unsafe extern "C" fn rs_sip_tx_get_stat_code( #[no_mangle] pub unsafe extern "C" fn rs_sip_tx_get_stat_msg( - tx: &mut SIPTransaction, - buffer: *mut *const u8, - buffer_len: *mut u32, + tx: &mut SIPTransaction, buffer: *mut *const u8, buffer_len: *mut u32, ) -> u8 { if let Some(ref r) = tx.response { let re = &r.reason; @@ -143,9 +132,7 @@ pub unsafe extern "C" fn rs_sip_tx_get_stat_msg( #[no_mangle] pub unsafe extern "C" fn rs_sip_tx_get_request_line( - tx: &mut SIPTransaction, - buffer: *mut *const u8, - buffer_len: *mut u32, + tx: &mut SIPTransaction, buffer: *mut *const u8, buffer_len: *mut u32, ) -> u8 { if let Some(ref r) = tx.request_line { if !r.is_empty() { @@ -163,9 +150,7 @@ pub unsafe extern "C" fn rs_sip_tx_get_request_line( #[no_mangle] pub unsafe extern "C" fn rs_sip_tx_get_response_line( - tx: &mut SIPTransaction, - buffer: *mut *const u8, - buffer_len: *mut u32, + tx: &mut SIPTransaction, buffer: *mut *const u8, buffer_len: *mut u32, ) -> u8 { if let Some(ref r) = tx.response_line { if !r.is_empty() { diff --git a/rust/src/sip/log.rs b/rust/src/sip/log.rs index 792acfa49021..a7a98d5eb1b3 100644 --- a/rust/src/sip/log.rs +++ b/rust/src/sip/log.rs @@ -51,4 +51,4 @@ fn log(tx: &SIPTransaction, js: &mut JsonBuilder) -> Result<(), JsonError> { #[no_mangle] pub extern "C" fn rs_sip_log_json(tx: &mut SIPTransaction, js: &mut JsonBuilder) -> bool { log(tx, js).is_ok() -} \ No newline at end of file +} diff --git a/rust/src/sip/parser.rs b/rust/src/sip/parser.rs index a34bc2615e53..cdf5fb136cfe 100644 --- a/rust/src/sip/parser.rs +++ b/rust/src/sip/parser.rs @@ -15,11 +15,11 @@ * 02110-1301, USA. */ -// written by Giuseppe Longo +// written by Giuseppe Longo -use nom7::bytes::streaming::{take, take_while, take_while1}; +use nom7::bytes::streaming::{tag, take, take_while, take_while1}; use nom7::character::streaming::{char, crlf}; -use nom7::character::{is_alphabetic, is_alphanumeric, is_space}; +use nom7::character::{is_alphabetic, is_alphanumeric, is_digit, is_space}; use nom7::combinator::map_res; use nom7::sequence::delimited; use nom7::{Err, IResult, Needed}; @@ -57,9 +57,13 @@ pub struct Response { pub body_len: u16, } +/** + * Valid tokens and chars are defined in RFC3261: + * https://www.rfc-editor.org/rfc/rfc3261#section-25.1 + */ #[inline] fn is_token_char(b: u8) -> bool { - is_alphanumeric(b) || b"!%'*+-._`".contains(&b) + is_alphanumeric(b) || b"!%'*+-._`~".contains(&b) } #[inline] @@ -69,12 +73,12 @@ fn is_method_char(b: u8) -> bool { #[inline] fn is_request_uri_char(b: u8) -> bool { - is_alphanumeric(b) || is_token_char(b) || b"~#@:".contains(&b) + is_alphanumeric(b) || is_token_char(b) || b"~#@:;=?+&$,/".contains(&b) } #[inline] fn is_version_char(b: u8) -> bool { - is_alphanumeric(b) || b"./".contains(&b) + is_digit(b) || b".".contains(&b) } #[inline] @@ -107,7 +111,7 @@ pub fn sip_parse_request(oi: &[u8]) -> IResult<&[u8], Request> { Request { method: method.into(), path: path.into(), - version: version.into(), + version, headers, request_line_len: request_line_len as u16, @@ -133,7 +137,7 @@ pub fn sip_parse_response(oi: &[u8]) -> IResult<&[u8], Response> { Ok(( bi, Response { - version: version.into(), + version, code: code.into(), reason: reason.into(), @@ -156,8 +160,10 @@ fn parse_request_uri(i: &[u8]) -> IResult<&[u8], &str> { } #[inline] -fn parse_version(i: &[u8]) -> IResult<&[u8], &str> { - map_res(take_while1(is_version_char), std::str::from_utf8)(i) +fn parse_version(i: &[u8]) -> IResult<&[u8], String> { + let (i, prefix) = map_res(tag("SIP/"), std::str::from_utf8)(i)?; + let (i, version) = map_res(take_while1(is_version_char), std::str::from_utf8)(i)?; + Ok((i, format!("{}{}", prefix, version))) } #[inline] @@ -322,4 +328,20 @@ mod tests { } } } + + #[test] + fn test_parse_invalid_version() { + let buf: &[u8] = "HTTP/1.1\r\n".as_bytes(); + + // This test must fail if 'HTTP/1.1' is accepted + assert!(parse_version(buf).is_err()); + } + + #[test] + fn test_parse_valid_version() { + let buf: &[u8] = "SIP/2.0\r\n".as_bytes(); + + let (_rem, result) = parse_version(buf).unwrap(); + assert_eq!(result, "SIP/2.0"); + } } diff --git a/rust/src/sip/sip.rs b/rust/src/sip/sip.rs index 4e86f5ea476d..d1aefde548be 100755 --- a/rust/src/sip/sip.rs +++ b/rust/src/sip/sip.rs @@ -17,10 +17,10 @@ // written by Giuseppe Longo -use crate::frames::*; use crate::applayer::{self, *}; use crate::core; use crate::core::{AppProto, Flow, ALPROTO_UNKNOWN}; +use crate::frames::*; use crate::sip::parser::*; use nom7::Err; use std; @@ -96,10 +96,7 @@ impl SIPState { } fn free_tx(&mut self, tx_id: u64) { - let tx = self - .transactions - .iter() - .position(|tx| tx.id == tx_id + 1); + let tx = self.transactions.iter().position(|tx| tx.id == tx_id + 1); debug_assert!(tx.is_some()); if let Some(idx) = tx { let _ = self.transactions.remove(idx); @@ -149,7 +146,13 @@ impl SIPState { fn parse_response(&mut self, flow: *const core::Flow, stream_slice: StreamSlice) -> bool { let input = stream_slice.as_slice(); - let _pdu = Frame::new(flow, &stream_slice, input, input.len() as i64, SIPFrameType::Pdu as u8); + let _pdu = Frame::new( + flow, + &stream_slice, + input, + input.len() as i64, + SIPFrameType::Pdu as u8, + ); SCLogDebug!("tc: pdu {:?}", _pdu); match sip_parse_response(input) { @@ -224,20 +227,40 @@ fn sip_frames_ts(flow: *const core::Flow, stream_slice: &StreamSlice, r: &Reques fn sip_frames_tc(flow: *const core::Flow, stream_slice: &StreamSlice, r: &Response) { let oi = stream_slice.as_slice(); - let _f = Frame::new(flow, stream_slice, oi, r.response_line_len as i64, SIPFrameType::ResponseLine as u8); - let hi = &oi[r.response_line_len as usize ..]; + let _f = Frame::new( + flow, + stream_slice, + oi, + r.response_line_len as i64, + SIPFrameType::ResponseLine as u8, + ); + let hi = &oi[r.response_line_len as usize..]; SCLogDebug!("tc: response_line {:?}", _f); - let _f = Frame::new(flow, stream_slice, hi, r.headers_len as i64, SIPFrameType::ResponseHeaders as u8); + let _f = Frame::new( + flow, + stream_slice, + hi, + r.headers_len as i64, + SIPFrameType::ResponseHeaders as u8, + ); SCLogDebug!("tc: response_headers {:?}", _f); if r.body_len > 0 { - let bi = &oi[r.body_offset as usize ..]; - let _f = Frame::new(flow, stream_slice, bi, r.body_len as i64, SIPFrameType::ResponseBody as u8); + let bi = &oi[r.body_offset as usize..]; + let _f = Frame::new( + flow, + stream_slice, + bi, + r.body_len as i64, + SIPFrameType::ResponseBody as u8, + ); SCLogDebug!("tc: response_body {:?}", _f); } } #[no_mangle] -pub extern "C" fn rs_sip_state_new(_orig_state: *mut std::os::raw::c_void, _orig_proto: AppProto) -> *mut std::os::raw::c_void { +pub extern "C" fn rs_sip_state_new( + _orig_state: *mut std::os::raw::c_void, _orig_proto: AppProto, +) -> *mut std::os::raw::c_void { let state = SIPState::new(); let boxed = Box::new(state); return Box::into_raw(boxed) as *mut _; @@ -251,8 +274,7 @@ pub extern "C" fn rs_sip_state_free(state: *mut std::os::raw::c_void) { #[no_mangle] pub unsafe extern "C" fn rs_sip_state_get_tx( - state: *mut std::os::raw::c_void, - tx_id: u64, + state: *mut std::os::raw::c_void, tx_id: u64, ) -> *mut std::os::raw::c_void { let state = cast_pointer!(state, SIPState); match state.get_tx_by_id(tx_id) { @@ -275,8 +297,7 @@ pub unsafe extern "C" fn rs_sip_state_tx_free(state: *mut std::os::raw::c_void, #[no_mangle] pub extern "C" fn rs_sip_tx_get_alstate_progress( - _tx: *mut std::os::raw::c_void, - _direction: u8, + _tx: *mut std::os::raw::c_void, _direction: u8, ) -> std::os::raw::c_int { 1 } @@ -285,11 +306,7 @@ static mut ALPROTO_SIP: AppProto = ALPROTO_UNKNOWN; #[no_mangle] pub unsafe extern "C" fn rs_sip_probing_parser_ts( - _flow: *const Flow, - _direction: u8, - input: *const u8, - input_len: u32, - _rdir: *mut u8, + _flow: *const Flow, _direction: u8, input: *const u8, input_len: u32, _rdir: *mut u8, ) -> AppProto { let buf = build_slice!(input, input_len as usize); if sip_parse_request(buf).is_ok() { @@ -300,11 +317,7 @@ pub unsafe extern "C" fn rs_sip_probing_parser_ts( #[no_mangle] pub unsafe extern "C" fn rs_sip_probing_parser_tc( - _flow: *const Flow, - _direction: u8, - input: *const u8, - input_len: u32, - _rdir: *mut u8, + _flow: *const Flow, _direction: u8, input: *const u8, input_len: u32, _rdir: *mut u8, ) -> AppProto { let buf = build_slice!(input, input_len as usize); if sip_parse_response(buf).is_ok() { @@ -315,11 +328,8 @@ pub unsafe extern "C" fn rs_sip_probing_parser_tc( #[no_mangle] pub unsafe extern "C" fn rs_sip_parse_request( - flow: *const core::Flow, - state: *mut std::os::raw::c_void, - _pstate: *mut std::os::raw::c_void, - stream_slice: StreamSlice, - _data: *const std::os::raw::c_void, + flow: *const core::Flow, state: *mut std::os::raw::c_void, _pstate: *mut std::os::raw::c_void, + stream_slice: StreamSlice, _data: *const std::os::raw::c_void, ) -> AppLayerResult { let state = cast_pointer!(state, SIPState); state.parse_request(flow, stream_slice).into() @@ -327,11 +337,8 @@ pub unsafe extern "C" fn rs_sip_parse_request( #[no_mangle] pub unsafe extern "C" fn rs_sip_parse_response( - flow: *const core::Flow, - state: *mut std::os::raw::c_void, - _pstate: *mut std::os::raw::c_void, - stream_slice: StreamSlice, - _data: *const std::os::raw::c_void, + flow: *const core::Flow, state: *mut std::os::raw::c_void, _pstate: *mut std::os::raw::c_void, + stream_slice: StreamSlice, _data: *const std::os::raw::c_void, ) -> AppLayerResult { let state = cast_pointer!(state, SIPState); state.parse_response(flow, stream_slice).into() diff --git a/rust/src/smb/dcerpc.rs b/rust/src/smb/dcerpc.rs index de6b8def73b3..6c2a2f9345df 100644 --- a/rust/src/smb/dcerpc.rs +++ b/rust/src/smb/dcerpc.rs @@ -168,6 +168,8 @@ impl SMBState { _ => { false }, }; if found { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/smb/files.rs b/rust/src/smb/files.rs index b290357428ee..ef9e620bb8f3 100644 --- a/rust/src/smb/files.rs +++ b/rust/src/smb/files.rs @@ -126,6 +126,8 @@ impl SMBState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); } + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -152,6 +154,8 @@ impl SMBState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); } + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/smb/session.rs b/rust/src/smb/session.rs index be7866976dc9..241973fc2b4d 100644 --- a/rust/src/smb/session.rs +++ b/rust/src/smb/session.rs @@ -61,6 +61,8 @@ impl SMBState { _ => { false }, }; if hit { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/smb/smb.rs b/rust/src/smb/smb.rs index b936f89211ac..97d4bf4f2272 100644 --- a/rust/src/smb/smb.rs +++ b/rust/src/smb/smb.rs @@ -806,6 +806,8 @@ impl SMBState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.request_done || !tx_old.response_done { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.request_done = true; tx_old.response_done = true; tx_old.set_event(SMBEvent::TooManyTransactions); @@ -924,6 +926,8 @@ impl SMBState { false }; if found { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -948,6 +952,8 @@ impl SMBState { false }; if found { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -986,6 +992,8 @@ impl SMBState { _ => { false }, }; if found { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -1019,6 +1027,8 @@ impl SMBState { _ => { false }, }; if hit { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/ssh/ssh.rs b/rust/src/ssh/ssh.rs index a0586894f9fb..32526132ec10 100644 --- a/rust/src/ssh/ssh.rs +++ b/rust/src/ssh/ssh.rs @@ -359,6 +359,7 @@ pub unsafe extern "C" fn rs_ssh_parse_request( let state = &mut cast_pointer!(state, SSHState); let buf = stream_slice.as_slice(); let hdr = &mut state.transaction.cli_hdr; + state.transaction.tx_data.updated_ts = true; if hdr.flags < SSHConnectionState::SshStateBannerDone { return state.parse_banner(buf, false, pstate); } else { @@ -375,6 +376,7 @@ pub unsafe extern "C" fn rs_ssh_parse_response( let state = &mut cast_pointer!(state, SSHState); let buf = stream_slice.as_slice(); let hdr = &mut state.transaction.srv_hdr; + state.transaction.tx_data.updated_tc = true; if hdr.flags < SSHConnectionState::SshStateBannerDone { return state.parse_banner(buf, true, pstate); } else { diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c index c7f902edc22f..2341ab6fe140 100644 --- a/src/app-layer-detect-proto.c +++ b/src/app-layer-detect-proto.c @@ -614,7 +614,7 @@ static AppProto AppLayerProtoDetectPPGetProto(Flow *f, const uint8_t *buf, uint3 else if (pp_port_sp) mask = pp_port_sp->alproto_mask; - if (alproto_masks[0] == mask) { + if ((alproto_masks[0] & mask) == mask) { FLOW_SET_PP_DONE(f, dir); SCLogDebug("%s, mask is now %08x, needed %08x, so done", (dir == STREAM_TOSERVER) ? "toserver":"toclient", diff --git a/src/app-layer-dnp3.c b/src/app-layer-dnp3.c index 9501b9f5ea57..26422cb18021 100644 --- a/src/app-layer-dnp3.c +++ b/src/app-layer-dnp3.c @@ -894,6 +894,7 @@ static void DNP3HandleUserDataRequest(DNP3State *dnp3, const uint8_t *input, /* Update the saved transport header so subsequent segments * will be matched to this sequence number. */ tx->th = th; + tx->tx_data.updated_ts = true; } else { ah = (DNP3ApplicationHeader *)(input + sizeof(DNP3LinkHeader) + @@ -971,6 +972,7 @@ static void DNP3HandleUserDataResponse(DNP3State *dnp3, const uint8_t *input, /* Replace the transport header in the transaction with this * one in case there are more frames. */ tx->th = th; + tx->tx_data.updated_tc = true; } else { ah = (DNP3ApplicationHeader *)(input + offset); diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index d1804c61e3ba..9ee93989d29e 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -718,6 +718,7 @@ static AppLayerResult FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserS SCReturnStruct(APP_LAYER_ERROR); } lasttx = tx; + tx->tx_data.updated_tc = true; if (state->command == FTP_COMMAND_UNKNOWN || tx->command_descriptor == NULL) { /* unknown */ tx->command_descriptor = &FtpCommands[FTP_COMMAND_MAX - 1]; @@ -1008,7 +1009,11 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state, SCTxDataUpdateFileFlags(&ftpdata_state->tx_data, ftpdata_state->state_data.file_flags); if (ftpdata_state->tx_data.file_tx == 0) ftpdata_state->tx_data.file_tx = direction & (STREAM_TOSERVER | STREAM_TOCLIENT); - + if (direction & STREAM_TOSERVER) { + ftpdata_state->tx_data.updated_ts = true; + } else { + ftpdata_state->tx_data.updated_tc = true; + } /* we depend on detection engine for file pruning */ const uint16_t flags = FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction); int ret = 0; diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 84cfd2148cf8..16f832fd5d8d 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1877,6 +1877,7 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d) if (tx_ud == NULL) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.updated_ts = true; SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags); if (!tx_ud->response_body_init) { @@ -2007,6 +2008,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d) if (tx_ud == NULL) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.updated_tc = true; SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags); if (!tx_ud->request_body_init) { tx_ud->request_body_init = 1; @@ -2113,6 +2115,7 @@ static int HTPCallbackRequestHasTrailer(htp_tx_t *tx) { HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated_ts = true; htud->request_has_trailers = 1; } return HTP_OK; @@ -2122,6 +2125,7 @@ static int HTPCallbackResponseHasTrailer(htp_tx_t *tx) { HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated_tc = true; htud->response_has_trailers = 1; } return HTP_OK; @@ -2164,6 +2168,8 @@ static int HTPCallbackRequestStart(htp_tx_t *tx) } tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files htp_tx_set_user_data(tx, tx_ud); + } else { + tx_ud->tx_data.updated_ts = true; } SCReturnInt(HTP_OK); } @@ -2204,6 +2210,8 @@ static int HTPCallbackResponseStart(htp_tx_t *tx) tx_ud->tx_data.file_tx = STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed. htp_tx_set_user_data(tx, tx_ud); + } else { + tx_ud->tx_data.updated_tc = true; } SCReturnInt(HTP_OK); } @@ -2255,6 +2263,7 @@ static int HTPCallbackRequestComplete(htp_tx_t *tx) HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated_ts = true; if (htud->tsflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER); @@ -2310,6 +2319,7 @@ static int HTPCallbackResponseComplete(htp_tx_t *tx) HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated_tc = true; if (htud->tcflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT); @@ -2428,6 +2438,7 @@ static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data) return HTP_OK; } tx_ud->request_headers_raw = ptmp; + tx_ud->tx_data.updated_ts = true; memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len, tx_data->data, tx_data->len); @@ -2450,6 +2461,7 @@ static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data) if (tx_ud == NULL) { return HTP_OK; } + tx_ud->tx_data.updated_tc = true; ptmp = HTPRealloc(tx_ud->response_headers_raw, tx_ud->response_headers_raw_len, tx_ud->response_headers_raw_len + tx_data->len); diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index a051616dd3b9..5d33dca71415 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -975,7 +975,14 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) AppLayerParserFileTxHousekeeping(f, tx, pkt_dir, (bool)pkt_dir_trunc); } - + if (txd) { + // should be reset by parser next time it updates the tx + if (pkt_dir & STREAM_TOSERVER) { + txd->updated_ts = false; + } else { + txd->updated_tc = false; + } + } const int tx_progress_tc = AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); if (tx_progress_tc < tx_end_state_tc) { diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 5a08c02ce46a..475d0abe12a3 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -912,6 +912,9 @@ static int SMTPProcessReply(SMTPState *state, Flow *f, AppLayerParserState *psta return 0; // to continue processing further } + if (state->curr_tx) { + state->curr_tx->tx_data.updated_tc = true; + } /* the reply code has to contain at least 3 bytes, to hold the 3 digit * reply code */ if (line->len < 3) { @@ -1204,6 +1207,7 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, smtp_config.content_inspect_min_size); } + tx->tx_data.updated_ts = true; state->toserver_data_count += (line->len + line->delim_len); diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c index 7dff89dac6c3..27de2b1791a3 100644 --- a/src/app-layer-ssl.c +++ b/src/app-layer-ssl.c @@ -2665,6 +2665,8 @@ static AppLayerResult SSLDecode(Flow *f, uint8_t direction, void *alstate, AppLayerParserState *pstate, StreamSlice stream_slice) { SSLState *ssl_state = (SSLState *)alstate; + ssl_state->tx_data.updated_tc = true; + ssl_state->tx_data.updated_ts = true; uint32_t counter = 0; ssl_state->f = f; const uint8_t *input = StreamSliceGetData(&stream_slice); diff --git a/src/detect.c b/src/detect.c index 82aaafcb94e1..bd3f1240cbd6 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1275,6 +1275,12 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro DetectTransaction no_tx = NO_TX; return no_tx; } + const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags); + bool updated = (flow_flags & STREAM_TOSERVER) ? txd->updated_ts : txd->updated_tc; + if (!updated && tx_progress < tx_end_state && ((flow_flags & STREAM_EOF) == 0)) { + DetectTransaction no_tx = NO_TX; + return no_tx; + } uint64_t detect_flags = (flow_flags & STREAM_TOSERVER) ? txd->detect_flags_ts : txd->detect_flags_tc; if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) { @@ -1291,7 +1297,6 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro return no_tx; } - const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags); const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1; DetectEngineState *tx_de_state = txd->de_state; DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL; diff --git a/src/output-tx.c b/src/output-tx.c index 042b424adec9..0936bd054f33 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -394,7 +394,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser); uint64_t max_id = tx_id; int logged = 0; - int gap = 0; + bool gap = false; const bool support_files = AppLayerParserSupportsFiles(ipproto, alproto); const uint8_t pkt_dir = STREAM_FLAGS_FOR_PACKET(p); @@ -417,15 +417,6 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) tx_id = ires.tx_id; SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx); - const int tx_progress_ts = - AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags); - const int tx_progress_tc = - AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); - const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc); - - SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file, - op_thread_data->filedata); - AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx); if (unlikely(txd == NULL)) { SCLogDebug("NO TXD"); @@ -435,6 +426,15 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) goto next_tx; } + const int tx_progress_ts = + AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags); + const int tx_progress_tc = + AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); + const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc); + + SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file, + op_thread_data->filedata); + if (file_logging_active) { if (AppLayerParserIsFileTx(txd)) { // need to process each tx that might be a file tx, // even if there are not files (yet) @@ -469,6 +469,11 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) } } SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags); + if (!txd->updated_tc && !txd->updated_ts && !(tx_progress_ts == complete_ts) && + !(tx_progress_tc == complete_tc) && !ts_eof && !tc_eof) { + gap = true; + goto next_tx; + } if (list[ALPROTO_UNKNOWN] != 0) { OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id); @@ -519,7 +524,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) max_id = tx_id; SCLogDebug("max_id %" PRIu64, max_id); } else { - gap = 1; + gap = true; } next_tx: if (!ires.has_next) diff --git a/src/stream-tcp.c b/src/stream-tcp.c index 46b379f7de16..b847fb6f8a00 100644 --- a/src/stream-tcp.c +++ b/src/stream-tcp.c @@ -5815,11 +5815,7 @@ TmEcode StreamTcp (ThreadVars *tv, Packet *p, void *data, PacketQueueNoLock *pq) StatsIncr(tv, stt->counter_tcp_invalid_checksum); return TM_ECODE_OK; } - } else { - p->flags |= PKT_IGNORE_CHECKSUM; } - } else { - p->flags |= PKT_IGNORE_CHECKSUM; //TODO check that this is set at creation } AppLayerProfilingReset(stt->ra_ctx->app_tctx);