diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs index 97db321e2249..b0df9159dfdc 100644 --- a/rust/src/applayer.rs +++ b/rust/src/applayer.rs @@ -477,6 +477,7 @@ pub const APP_LAYER_PARSER_EOF_TS : u16 = BIT_U16!(5); pub const APP_LAYER_PARSER_EOF_TC : u16 = BIT_U16!(6); pub const APP_LAYER_PARSER_TRUNC_TS : u16 = BIT_U16!(7); pub const APP_LAYER_PARSER_TRUNC_TC : u16 = BIT_U16!(8); +pub const APP_LAYER_PARSER_LAYERED_PACKET : u16 = BIT_U16!(11); pub const APP_LAYER_PARSER_OPT_ACCEPT_GAPS: u32 = BIT_U32!(0); diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index 14d7b47dfb03..55e41c384618 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -25,6 +25,9 @@ use crate::conf::conf_get; use crate::core::*; use crate::filecontainer::*; use crate::filetracker::*; + +use crate::dns::dns::rs_dns_state_new; + use nom7::Err; use std; use std::collections::VecDeque; @@ -144,6 +147,8 @@ pub struct HTTP2Transaction { pub escaped: Vec>, pub req_line: Vec, pub resp_line: Vec, + + pub doh: Vec, } impl Transaction for HTTP2Transaction { @@ -175,6 +180,7 @@ impl HTTP2Transaction { escaped: Vec::with_capacity(16), req_line: Vec::new(), resp_line: Vec::new(), + doh: Vec::new(), } } @@ -202,12 +208,27 @@ impl HTTP2Transaction { self.tx_data.set_event(event as u8); } - fn handle_headers(&mut self, blocks: &[parser::HTTP2FrameHeaderBlock], dir: Direction) { + fn handle_headers(&mut self, blocks: &[parser::HTTP2FrameHeaderBlock], dir: Direction) -> Option> { let mut authority = None; + let mut path = None; + let mut doh = false; let mut host = None; for block in blocks { if block.name == b"content-encoding" { self.decoder.http2_encoding_fromvec(&block.value, dir); + } else if block.name == b"accept" { + //TODO? faster pattern matching + if block.value == b"application/dns-message" { + doh = true; + } + } else if block.name == b"content-type" { + if block.value == b"application/dns-message" { + // push original 2-bytes DNS/TCP header + self.doh.push(0); + self.doh.push(0); + } + } else if block.name == b":path" { + path = Some(&block.value); } else if block.name.eq_ignore_ascii_case(b":authority") { authority = Some(&block.value); if block.value.iter().any(|&x| x == b'@') { @@ -229,6 +250,14 @@ impl HTTP2Transaction { } } } + if doh { + if let Some(p) = path { + if let Ok((_, dns_req)) = parser::doh_extract_request(p) { + return Some(dns_req); + } + } + } + return None; } pub fn update_file_flags(&mut self, flow_file_flags: u16) { @@ -237,10 +266,9 @@ impl HTTP2Transaction { } fn decompress<'a>( - &'a mut self, input: &'a [u8], dir: Direction, sfcm: &'static SuricataFileContext, over: bool, flow: *const Flow, + &'a mut self, input: &'a [u8], output: &'a mut Vec, dir: Direction, sfcm: &'static SuricataFileContext, over: bool, flow: *const Flow, ) -> io::Result<()> { - let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE); - let decompressed = self.decoder.decompress(input, &mut output, dir)?; + let decompressed = self.decoder.decompress(input, output, dir)?; let xid: u32 = self.tx_id as u32; if dir == Direction::ToClient { self.ft_tc.tx_id = self.tx_id - 1; @@ -299,13 +327,17 @@ impl HTTP2Transaction { &xid, ); }; + if !self.doh.is_empty() { + self.doh.extend_from_slice(decompressed); + } return Ok(()); } fn handle_frame( &mut self, header: &parser::HTTP2FrameHeader, data: &HTTP2FrameTypeData, dir: Direction, - ) { + ) -> Option> { //handle child_stream_id changes + let mut r = None; match data { HTTP2FrameTypeData::PUSHPROMISE(hs) => { if dir == Direction::ToClient { @@ -315,7 +347,7 @@ impl HTTP2Transaction { } self.state = HTTP2TransactionState::HTTP2StateReserved; } - self.handle_headers(&hs.blocks, dir); + r = self.handle_headers(&hs.blocks, dir); } HTTP2FrameTypeData::CONTINUATION(hs) => { if dir == Direction::ToClient @@ -323,13 +355,13 @@ impl HTTP2Transaction { { self.child_stream_id = 0; } - self.handle_headers(&hs.blocks, dir); + r = self.handle_headers(&hs.blocks, dir); } HTTP2FrameTypeData::HEADERS(hs) => { if dir == Direction::ToClient { self.child_stream_id = 0; } - self.handle_headers(&hs.blocks, dir); + r = self.handle_headers(&hs.blocks, dir); } HTTP2FrameTypeData::RSTSTREAM(_) => { self.child_stream_id = 0; @@ -376,6 +408,7 @@ impl HTTP2Transaction { } _ => {} } + return r; } } @@ -441,6 +474,9 @@ pub struct HTTP2State { dynamic_headers_tc: HTTP2DynTable, transactions: VecDeque, progress: HTTP2ConnectionState, + // layered packets contents for DNS over HTTP2 + layered: Vec>, + dns_state: Option<*mut std::os::raw::c_void>, } impl State for HTTP2State { @@ -473,6 +509,8 @@ impl HTTP2State { dynamic_headers_tc: HTTP2DynTable::new(), transactions: VecDeque::new(), progress: HTTP2ConnectionState::Http2StateInit, + layered: Vec::new(), + dns_state: None, } } @@ -925,6 +963,7 @@ impl HTTP2State { fn parse_frames( &mut self, mut input: &[u8], il: usize, dir: Direction, flow: *const Flow, + pstate: *mut std::os::raw::c_void ) -> AppLayerResult { while !input.is_empty() { match parser::http2_parse_frame_header(input) { @@ -970,7 +1009,7 @@ impl HTTP2State { ); let tx = self.find_or_create_tx(&head, &txdata, dir); - tx.handle_frame(&head, &txdata, dir); + let odoh = tx.handle_frame(&head, &txdata, dir); let over = head.flags & parser::HTTP2_FLAG_HEADER_EOS != 0; let ftype = head.ftype; let sid = head.stream_id; @@ -1002,19 +1041,58 @@ impl HTTP2State { if padded && !rem.is_empty() && usize::from(rem[0]) < hlsafe{ dinput = &rem[1..hlsafe - usize::from(rem[0])]; } - if tx_same.decompress( + let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE); + match tx_same.decompress( dinput, + &mut output, dir, sfcm, over, - flow).is_err() { - self.set_event(HTTP2Event::FailedDecompression); + flow) { + Ok(_) => { + if !tx_same.doh.is_empty() { + if over { + // fix DNS/TCP header length field + tx_same.doh[0] = ((tx_same.doh.len() - 2) >> 8) as u8; + tx_same.doh[1] = ((tx_same.doh.len() - 2) & 0xFF) as u8; + self.layered.push(tx_same.doh.to_vec()); + if self.dns_state.is_none() { + unsafe { + self.dns_state = Some(rs_dns_state_new(std::ptr::null_mut(), ALPROTO_HTTP2)); + } + } + unsafe { + AppLayerParserStateSetFlag( + pstate, + APP_LAYER_PARSER_LAYERED_PACKET, + ); + } + } + } + } + _ => { + self.set_event(HTTP2Event::FailedDecompression); + } } } } None => panic!("no SURICATA_HTTP2_FILE_CONFIG"), } } + if let Some(doh) = odoh { + self.layered.push(doh); + unsafe { + AppLayerParserStateSetFlag( + pstate, + APP_LAYER_PARSER_LAYERED_PACKET, + ); + } + if self.dns_state.is_none() { + unsafe { + self.dns_state = Some(rs_dns_state_new(std::ptr::null_mut(), ALPROTO_HTTP2)); + } + } + } input = &rem[hlsafe..]; } Err(Err::Incomplete(_)) => { @@ -1033,7 +1111,7 @@ impl HTTP2State { return AppLayerResult::ok(); } - fn parse_ts(&mut self, mut input: &[u8], flow: *const Flow) -> AppLayerResult { + fn parse_ts(&mut self, mut input: &[u8], flow: *const Flow, pstate: *mut std::os::raw::c_void) -> AppLayerResult { //very first : skip magic let mut magic_consumed = 0; if self.progress < HTTP2ConnectionState::Http2StateMagicDone { @@ -1073,7 +1151,7 @@ impl HTTP2State { } //then parse all we can - let r = self.parse_frames(input, il, Direction::ToServer, flow); + let r = self.parse_frames(input, il, Direction::ToServer, flow, pstate); if r.status == 1 { //adds bytes consumed by banner to incomplete result return AppLayerResult::incomplete(r.consumed + magic_consumed as u32, r.needed); @@ -1082,7 +1160,7 @@ impl HTTP2State { } } - fn parse_tc(&mut self, mut input: &[u8], flow: *const Flow) -> AppLayerResult { + fn parse_tc(&mut self, mut input: &[u8], flow: *const Flow, pstate: *mut std::os::raw::c_void) -> AppLayerResult { //first consume frame bytes let il = input.len(); if self.response_frame_size > 0 { @@ -1097,10 +1175,43 @@ impl HTTP2State { } } //then parse all we can - return self.parse_frames(input, il, Direction::ToClient, flow); + return self.parse_frames(input, il, Direction::ToClient, flow, pstate); + } +} + +#[no_mangle] +pub unsafe extern "C" fn SCHttp2ClearLayered( + s: &mut HTTP2State +) { + s.layered.clear(); +} + +#[no_mangle] +pub unsafe extern "C" fn SCHttp2GetLayeredState( + s: &HTTP2State +) -> *mut std::os::raw::c_void { + match s.dns_state { + Some(d) => d, + _ => std::ptr::null_mut(), } } +#[no_mangle] +pub unsafe extern "C" fn SCHttp2GetLayered( + s: &HTTP2State, buffer: *mut *const u8, buffer_len: *mut u32, i: u32, +) -> bool { + let i = i as usize; + if i < s.layered.len() { + *buffer = s.layered[i].as_ptr(); + *buffer_len = s.layered[i].len() as u32; + return true; + } + + *buffer = std::ptr::null(); + *buffer_len = 0; + return false; +} + // C exports. export_tx_data_get!(rs_http2_get_tx_data, HTTP2Transaction); @@ -1175,22 +1286,22 @@ pub unsafe extern "C" fn rs_http2_state_tx_free(state: *mut std::os::raw::c_void #[no_mangle] pub unsafe extern "C" fn rs_http2_parse_ts( - flow: *const Flow, state: *mut std::os::raw::c_void, _pstate: *mut std::os::raw::c_void, + flow: *const Flow, state: *mut std::os::raw::c_void, pstate: *mut std::os::raw::c_void, stream_slice: StreamSlice, _data: *const std::os::raw::c_void, ) -> AppLayerResult { let state = cast_pointer!(state, HTTP2State); let buf = stream_slice.as_slice(); - return state.parse_ts(buf, flow); + return state.parse_ts(buf, flow, pstate); } #[no_mangle] pub unsafe extern "C" fn rs_http2_parse_tc( - flow: *const Flow, state: *mut std::os::raw::c_void, _pstate: *mut std::os::raw::c_void, + flow: *const Flow, state: *mut std::os::raw::c_void, pstate: *mut std::os::raw::c_void, stream_slice: StreamSlice, _data: *const std::os::raw::c_void, ) -> AppLayerResult { let state = cast_pointer!(state, HTTP2State); let buf = stream_slice.as_slice(); - return state.parse_tc(buf, flow); + return state.parse_tc(buf, flow, pstate); } #[no_mangle] diff --git a/rust/src/http2/parser.rs b/rust/src/http2/parser.rs index adabeb28c6e4..5990c6c734c6 100644 --- a/rust/src/http2/parser.rs +++ b/rust/src/http2/parser.rs @@ -28,6 +28,7 @@ use nom7::multi::many0; use nom7::number::streaming::{be_u16, be_u24, be_u32, be_u8}; use nom7::sequence::tuple; use nom7::{Err, IResult}; +use nom7::bytes::complete::tag; use std::fmt; use std::str::FromStr; @@ -751,6 +752,22 @@ pub fn http2_parse_frame_settings(i: &[u8]) -> IResult<&[u8], Vec IResult<&[u8], Vec> { + let (i, _) = tag("/dns-query?dns=")(i)?; + match base64::decode(i) { + Ok(mut dec) => { + // adds 2 byte tcp header with length + dec.insert(0, (dec.len() & 0xFF) as u8); + dec.insert(0, (dec.len() >> 8) as u8); + // i is unused + return Ok((i, dec)); + } + _ => { + return Err(Err::Error(make_error(i, ErrorKind::MapOpt))); + } + } +} + #[cfg(test)] mod tests { diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 1f6066471757..4179d24f2c51 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -1397,8 +1397,11 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow } #endif /* invoke the parser */ - AppLayerResult res = p->Parser[direction](f, alstate, pstate, stream_slice, - alp_tctx->alproto_local_storage[f->protomap][alproto]); + void *storage = NULL; + if (alp_tctx) { + storage = alp_tctx->alproto_local_storage[f->protomap][alproto]; + } + AppLayerResult res = p->Parser[direction](f, alstate, pstate, stream_slice, storage); if (res.status < 0) { AppLayerIncParserErrorCounter(tv, f); goto error; @@ -1438,6 +1441,32 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow } } + if (pstate->flags & APP_LAYER_PARSER_LAYERED_PACKET) { + // hardcoded logic + if (alproto == ALPROTO_HTTP2) { + const uint8_t *b = NULL; + uint32_t b_len = 0; + for (uint32_t i = 0;; i++) { + if (!SCHttp2GetLayered(alstate, &b, &b_len, i)) { + SCHttp2ClearLayered(alstate); + break; + } + Packet *np = PacketPseudoFromFlow(f, f->protoctx, direction); + if (np == NULL) { + // TODO log warning + continue; + } + PKT_SET_SRC(np, PKT_SRC_APP_LAYER_LAYERED); + // TODO np->flags |= PKT_PSEUDO_DETECTLOG_FLUSH; + np->payload = SCMalloc(b_len); + memcpy(np->payload, b, b_len); + np->payload_len = b_len; + PacketEnqueueNoLock(&tv->decode_pq, np); + // TODO StatsIncr(tv, stt->counter_tcp_pseudo); + } + } + } + /* set the packets to no inspection and reassembly if required */ if (pstate->flags & APP_LAYER_PARSER_NO_INSPECTION) { AppLayerParserSetEOF(pstate); diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index e9f8cf55e925..95125202a848 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -42,6 +42,7 @@ #define APP_LAYER_PARSER_TRUNC_TC BIT_U16(8) #define APP_LAYER_PARSER_SFRAME_TS BIT_U16(9) #define APP_LAYER_PARSER_SFRAME_TC BIT_U16(10) +#define APP_LAYER_PARSER_LAYERED_PACKET BIT_U16(11) /* Flags for AppLayerParserProtoCtx. */ #define APP_LAYER_PARSER_OPT_ACCEPT_GAPS BIT_U32(0) diff --git a/src/decode.c b/src/decode.c index d302c7654675..4ea2bf2cc077 100644 --- a/src/decode.c +++ b/src/decode.c @@ -779,6 +779,9 @@ const char *PktSrcToString(enum PktSrcEnum pkt_src) case PKT_SRC_SHUTDOWN_FLUSH: pkt_src_str = "shutdown flush"; break; + case PKT_SRC_APP_LAYER_LAYERED: + pkt_src_str = "app-layer (detect/log)"; + break; } DEBUG_VALIDATE_BUG_ON(pkt_src_str == NULL); return pkt_src_str; diff --git a/src/decode.h b/src/decode.h index 6392f3361e58..7423af97f53d 100644 --- a/src/decode.h +++ b/src/decode.h @@ -64,6 +64,7 @@ enum PktSrcEnum { PKT_SRC_CAPTURE_TIMEOUT, PKT_SRC_DECODER_GENEVE, PKT_SRC_SHUTDOWN_FLUSH, + PKT_SRC_APP_LAYER_LAYERED, }; #include "source-nflog.h" diff --git a/src/flow-worker.c b/src/flow-worker.c index 6980570d3ce1..9fc7e12210b7 100644 --- a/src/flow-worker.c +++ b/src/flow-worker.c @@ -380,6 +380,36 @@ static inline void FlowWorkerStreamTCPUpdate(ThreadVars *tv, FlowWorkerThreadDat /* Packets here can safely access p->flow as it's locked */ SCLogDebug("packet %"PRIu64": extra packets %u", p->pcap_cnt, fw->pq.len); Packet *x; + while ((x = PacketDequeueNoLock(&tv->decode_pq))) { + // switch flow to new protocol + void *alstate_orig = p->flow->alstate; + AppProto alproto_orig = p->flow->alproto; + // hardcoded logic + if (alproto_orig == ALPROTO_HTTP2) { + p->flow->alproto = ALPROTO_DNS; + p->flow->alstate = SCHttp2GetLayeredState(p->flow->alstate); + } + // run parsing, detection and logging + AppLayerParserParse(tv, NULL, p->flow, p->flow->alproto, + (x->flowflags & FLOW_PKT_TOSERVER) ? STREAM_TOSERVER : STREAM_TOCLIENT, x->payload, + x->payload_len); + if (detect_thread != NULL) { + FLOWWORKER_PROFILING_START(x, PROFILE_FLOWWORKER_DETECT); + Detect(tv, x, detect_thread); + FLOWWORKER_PROFILING_END(x, PROFILE_FLOWWORKER_DETECT); + } + OutputLoggerLog(tv, x, fw->output_thread); + FramesPrune(x->flow, x); + + // switch back to real protocol + p->flow->alstate = alstate_orig; + p->flow->alproto = alproto_orig; + + // releases memory associated to the packet + SCFree(x->payload); + FlowDeReference(&x->flow); + TmqhOutputPacketpool(tv, x); + } while ((x = PacketDequeueNoLock(&fw->pq))) { SCLogDebug("packet %"PRIu64" extra packet %p", p->pcap_cnt, x); diff --git a/src/stream-tcp.c b/src/stream-tcp.c index b77423161800..5ac22db9cfd3 100644 --- a/src/stream-tcp.c +++ b/src/stream-tcp.c @@ -6478,29 +6478,12 @@ void StreamTcpSetSessionBypassFlag(TcpSession *ssn) ssn->flags |= STREAMTCP_FLAG_BYPASS; } -/** \brief Create a pseudo packet injected into the engine to signal the - * opposing direction of this stream trigger detection/logging. - * - * \param parent real packet - * \param pq packet queue to store the new pseudo packet in - * \param dir 0 ts 1 tc - */ -static void StreamTcpPseudoPacketCreateDetectLogFlush(ThreadVars *tv, - StreamTcpThread *stt, Packet *parent, - TcpSession *ssn, PacketQueueNoLock *pq, int dir) +Packet *PacketPseudoFromFlow(Flow *f, TcpSession *ssn, int dir) { - SCEnter(); - Flow *f = parent->flow; - - if (parent->flags & PKT_PSEUDO_DETECTLOG_FLUSH) { - SCReturn; - } - Packet *np = PacketPoolGetPacket(); if (np == NULL) { - SCReturn; + return NULL; } - PKT_SET_SRC(np, PKT_SRC_STREAM_TCP_DETECTLOG_FLUSH); np->tenant_id = f->tenant_id; np->datalink = DLT_RAW; @@ -6509,7 +6492,6 @@ static void StreamTcpPseudoPacketCreateDetectLogFlush(ThreadVars *tv, np->flags |= PKT_STREAM_EST; np->flags |= PKT_HAS_FLOW; np->flags |= PKT_IGNORE_CHECKSUM; - np->flags |= PKT_PSEUDO_DETECTLOG_FLUSH; memcpy(&np->vlan_id[0], &f->vlan_id[0], sizeof(np->vlan_id)); np->vlan_idx = f->vlan_idx; np->livedev = (struct LiveDevice_ *)f->livedev; @@ -6652,7 +6634,36 @@ static void StreamTcpPseudoPacketCreateDetectLogFlush(ThreadVars *tv, np->tcph->th_seq = htonl(ssn->server.next_seq); np->tcph->th_ack = htonl(ssn->client.last_ack); } + return np; +error: + FlowDeReference(&np->flow); + PacketPoolReturnPacket(np); + return NULL; +} +/** \brief Create a pseudo packet injected into the engine to signal the + * opposing direction of this stream trigger detection/logging. + * + * \param parent real packet + * \param pq packet queue to store the new pseudo packet in + * \param dir 0 ts 1 tc + */ +static void StreamTcpPseudoPacketCreateDetectLogFlush(ThreadVars *tv, StreamTcpThread *stt, + Packet *parent, TcpSession *ssn, PacketQueueNoLock *pq, int dir) +{ + SCEnter(); + Flow *f = parent->flow; + + if (parent->flags & PKT_PSEUDO_DETECTLOG_FLUSH) { + SCReturn; + } + + Packet *np = PacketPseudoFromFlow(f, ssn, dir); + if (np == NULL) { + SCReturn; + } + PKT_SET_SRC(np, PKT_SRC_STREAM_TCP_DETECTLOG_FLUSH); + np->flags |= PKT_PSEUDO_DETECTLOG_FLUSH; /* use parent time stamp */ np->ts = parent->ts; @@ -6661,9 +6672,6 @@ static void StreamTcpPseudoPacketCreateDetectLogFlush(ThreadVars *tv, StatsIncr(tv, stt->counter_tcp_pseudo); SCReturn; -error: - FlowDeReference(&np->flow); - SCReturn; } /** \brief create packets in both directions to flush out logging diff --git a/src/stream-tcp.h b/src/stream-tcp.h index ff8a0998cb4e..04e809c5886c 100644 --- a/src/stream-tcp.h +++ b/src/stream-tcp.h @@ -205,5 +205,8 @@ uint64_t StreamDataRightEdge(const TcpStream *stream, const bool eof); void StreamTcpThreadCacheEnable(void); void StreamTcpThreadCacheCleanup(void); +// move elsewhere +Packet *PacketPseudoFromFlow(Flow *f, TcpSession *ssn, int dir); + #endif /* __STREAM_TCP_H__ */