diff --git a/rules/smtp-events.rules b/rules/smtp-events.rules index 2898bdc299f4..9c601260133a 100644 --- a/rules/smtp-events.rules +++ b/rules/smtp-events.rules @@ -31,4 +31,6 @@ alert smtp any any -> any any (msg:"SURICATA SMTP duplicate fields"; flow:establ alert smtp any any -> any any (msg:"SURICATA SMTP unparsable content"; flow:established,to_server; app-layer-event:smtp.unparsable_content; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220019; rev:1;) alert smtp any any -> any any (msg:"SURICATA SMTP filename truncated"; flow:established,to_server; app-layer-event:smtp.mime_long_filename; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220020; rev:1;) alert smtp any any -> any any (msg:"SURICATA SMTP failed protocol change"; flow:established,to_client; app-layer-event:smtp.failed_protocol_change; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220021; rev:2;) -# next sid 2220022 +alert smtp any any -> any any (msg:"SURICATA SMTP invalid BDAT command"; flow:established,to_server; app-layer-event:smtp.invalid_bdat; flowint:smtp.anomaly.count,+,1; classtype:protocol-command-decode; sid:2220022; rev:1;) + +# next sid 2220023 diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 7938a94a0429..ebf2b0522501 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -95,9 +95,15 @@ /* All other commands are represented by this var */ #define SMTP_COMMAND_OTHER_CMD 5 #define SMTP_COMMAND_RSET 6 +#define SMTP_COMMAND_QUIT 7 +/* Pseudo command used to match the final BDAT reply to its transaction. */ +#define SMTP_COMMAND_BDAT_LAST 8 #define SMTP_DEFAULT_MAX_TX 256 +/* command buffer tx id for commands with no owning transaction */ +#define SMTP_NO_TX_ID UINT64_MAX + typedef struct SMTPInput_ { /* current input that is being parsed */ const uint8_t *buf; @@ -126,6 +132,7 @@ SCEnumCharMap smtp_decoder_event_table[] = { { "MAX_REPLY_LINE_LEN_EXCEEDED", SMTP_DECODER_EVENT_MAX_REPLY_LINE_LEN_EXCEEDED }, { "INVALID_PIPELINED_SEQUENCE", SMTP_DECODER_EVENT_INVALID_PIPELINED_SEQUENCE }, { "BDAT_CHUNK_LEN_EXCEEDED", SMTP_DECODER_EVENT_BDAT_CHUNK_LEN_EXCEEDED }, + { "INVALID_BDAT", SMTP_DECODER_EVENT_INVALID_BDAT }, { "NO_SERVER_WELCOME_MESSAGE", SMTP_DECODER_EVENT_NO_SERVER_WELCOME_MESSAGE }, { "TLS_REJECTED", SMTP_DECODER_EVENT_TLS_REJECTED }, { "DATA_COMMAND_REJECTED", SMTP_DECODER_EVENT_DATA_COMMAND_REJECTED }, @@ -186,6 +193,76 @@ static const char *SMTPGetFrameNameById(const uint8_t frame_id) return name; } +static SCEnumCharMap smtp_state_client_table[] = { + { "request_started", SMTP_REQUEST_STARTED }, + { "request_data", SMTP_REQUEST_DATA }, + { "request_complete", SMTP_REQUEST_COMPLETE }, + { NULL, -1 }, +}; + +static SCEnumCharMap smtp_state_server_table[] = { + { "response_started", SMTP_RESPONSE_STARTED }, + { "response_data", SMTP_RESPONSE_DATA }, + { "response_complete", SMTP_RESPONSE_COMPLETE }, + { NULL, -1 }, +}; + +static int SMTPStateGetStateIdByName(const char *name, const uint8_t direction) +{ + SCEnumCharMap *map = + direction == STREAM_TOSERVER ? smtp_state_client_table : smtp_state_server_table; + int id = SCMapEnumNameToValue(name, map); + if (id < 0) { + return -1; + } + return id; +} + +static const char *SMTPStateGetStateNameById(const int id, const uint8_t direction) +{ + SCEnumCharMap *map = + direction == STREAM_TOSERVER ? smtp_state_client_table : smtp_state_server_table; + return SCMapEnumValueToName(id, map); +} + +static inline void SMTPSetProgressTS(SMTPTransaction *tx, uint8_t progress) +{ + if (tx != NULL && tx->progress_ts < progress) { + tx->progress_ts = progress; + } +} + +static inline void SMTPSetProgressTC(SMTPTransaction *tx, uint8_t progress) +{ + if (tx != NULL && tx->progress_tc < progress) { + tx->progress_tc = progress; + tx->tx_data.updated_tc = true; + } +} + +static inline void SMTPTransactionCompleteTS(SMTPTransaction *tx) +{ + DEBUG_VALIDATE_BUG_ON(tx == NULL); + if (tx) { + SMTPSetProgressTS(tx, SMTP_REQUEST_COMPLETE); + SCLogDebug("marked tx as ts complete"); + } +} + +static inline void SMTPTransactionCompleteTC(SMTPTransaction *tx) +{ + DEBUG_VALIDATE_BUG_ON(tx == NULL); + if (tx) { + SMTPSetProgressTC(tx, SMTP_RESPONSE_COMPLETE); + SCLogDebug("marked tx as tc complete"); + } +} + +static bool SMTPTransactionRequestIsComplete(const SMTPTransaction *tx) +{ + return tx && tx->progress_ts == SMTP_REQUEST_COMPLETE; +} + typedef struct SMTPThreadCtx_ { MpmThreadCtx *smtp_mpm_thread_ctx; PrefilterRuleStore *pmq; @@ -483,6 +560,34 @@ static SMTPTransaction *SMTPTransactionCreate(SMTPState *state) return tx; } +static SMTPTransaction *SMTPStateGetTxById(SMTPState *state, uint64_t tx_id) +{ + SMTPTransaction *tx = NULL; + TAILQ_FOREACH (tx, &state->tx_list, next) { + if (tx->tx_id == tx_id) { + return tx; + } + if (tx->tx_id > tx_id) { + break; + } + } + return NULL; +} + +static SMTPTransaction *SMTPGetReplyTx(SMTPState *state) +{ + if (state->cmds_idx >= state->cmds_cnt) { + return state->curr_tx; + } + + /* a command with no owning tx, or whose tx is gone, must not resolve + * to another tx */ + if (state->cmds_tx_ids[state->cmds_idx] == SMTP_NO_TX_ID) { + return NULL; + } + return SMTPStateGetTxById(state, state->cmds_tx_ids[state->cmds_idx]); +} + static void FlagDetectStateNewFile(SMTPTransaction *tx) { if (tx && tx->tx_data.de_state) { @@ -548,8 +653,9 @@ static AppLayerResult SMTPGetLine(Flow *f, StreamSlice *slice, SMTPState *state, } else if (direction == 1) { frame = AppLayerFrameNewByPointer( f, slice, input->buf + input->consumed, -1, 1, SMTP_FRAME_RESPONSE_LINE); - if (frame != NULL && state->curr_tx) { - AppLayerFrameSetTxId(frame, state->curr_tx->tx_id); + SMTPTransaction *reply_tx = SMTPGetReplyTx(state); + if (frame != NULL && reply_tx != NULL) { + AppLayerFrameSetTxId(frame, reply_tx->tx_id); } } } @@ -613,7 +719,8 @@ static AppLayerResult SMTPGetLine(Flow *f, StreamSlice *slice, SMTPState *state, } } -static int SMTPInsertCommandIntoCommandBuffer(uint8_t command, SMTPState *state) +static int SMTPInsertCommandIntoCommandBuffer( + SMTPState *state, uint8_t command, const SMTPTransaction *tx) { SCEnter(); void *ptmp; @@ -628,12 +735,26 @@ static int SMTPInsertCommandIntoCommandBuffer(uint8_t command, SMTPState *state) sizeof(uint8_t) * (state->cmds_buffer_len + increment)); if (ptmp == NULL) { SCFree(state->cmds); + SCFree(state->cmds_tx_ids); state->cmds = NULL; + state->cmds_tx_ids = NULL; SCLogDebug("SCRealloc failure"); return -1; } state->cmds = ptmp; + ptmp = SCRealloc( + state->cmds_tx_ids, sizeof(uint64_t) * (state->cmds_buffer_len + increment)); + if (ptmp == NULL) { + SCFree(state->cmds); + SCFree(state->cmds_tx_ids); + state->cmds = NULL; + state->cmds_tx_ids = NULL; + SCLogDebug("SCRealloc failure"); + return -1; + } + state->cmds_tx_ids = ptmp; + state->cmds_buffer_len += increment; } if (state->cmds_cnt >= 1 && @@ -652,12 +773,13 @@ static int SMTPInsertCommandIntoCommandBuffer(uint8_t command, SMTPState *state) } state->cmds[state->cmds_cnt] = command; + state->cmds_tx_ids[state->cmds_cnt] = tx != NULL ? tx->tx_id : SMTP_NO_TX_ID; state->cmds_cnt++; return 0; } -static int SMTPProcessCommandBDAT(SMTPState *state, const SMTPLine *line) +static int SMTPProcessCommandBDAT(SMTPState *state, SMTPTransaction *tx, const SMTPLine *line) { SCEnter(); @@ -669,6 +791,9 @@ static int SMTPProcessCommandBDAT(SMTPState *state, const SMTPLine *line) SCReturnInt(-1); } else if (state->bdat_chunk_idx == state->bdat_chunk_len) { state->parser_state &= ~SMTP_PARSER_STATE_COMMAND_DATA_MODE; + if (state->current_command == SMTP_COMMAND_BDAT_LAST) { + SMTPTransactionCompleteTS(tx); + } } SCReturnInt(0); @@ -706,11 +831,13 @@ static void SetMimeEvents(SMTPState *state, uint32_t events) } } -static inline void SMTPTransactionComplete(SMTPState *state) +static inline void SMTPTransactionComplete(SMTPTransaction *tx) { - DEBUG_VALIDATE_BUG_ON(state->curr_tx == NULL); - if (state->curr_tx) - state->curr_tx->done = true; + DEBUG_VALIDATE_BUG_ON(tx == NULL); + if (tx) { + SMTPSetProgressTS(tx, SMTP_REQUEST_COMPLETE); + SMTPSetProgressTC(tx, SMTP_RESPONSE_COMPLETE); + } } /** @@ -735,7 +862,7 @@ static int SMTPProcessCommandDATA( * acknowledged with a reply. We insert a dummy command to * the command buffer to be used by the reply handler to match * the reply received */ - SMTPInsertCommandIntoCommandBuffer(SMTP_COMMAND_DATA_MODE, state); + SMTPInsertCommandIntoCommandBuffer(state, SMTP_COMMAND_DATA_MODE, tx); if (smtp_config.raw_extraction) { /* we use this as the signal that message data is complete. */ FileCloseFile(&tx->files_ts, &smtp_config.sbcfg, NULL, 0, 0); @@ -747,8 +874,7 @@ static int SMTPProcessCommandDATA( FileFlowToFlags(f, STREAM_TOSERVER)); } } - SMTPTransactionComplete(state); - SCLogDebug("marked tx as done"); + SMTPTransactionCompleteTS(tx); } else if (smtp_config.raw_extraction) { // message not over, store the line. This is a substitution of // ProcessDataChunk @@ -845,8 +971,7 @@ static int SMTPProcessCommandDATA( static inline bool IsReplyToCommand(const SMTPState *state, const uint8_t cmd) { - return (state->cmds_idx < state->cmds_buffer_len && - state->cmds[state->cmds_idx] == cmd); + return (state->cmds_idx < state->cmds_cnt && state->cmds[state->cmds_idx] == cmd); } static int SMTPProcessReply( @@ -859,8 +984,9 @@ static int SMTPProcessReply( return 0; // to continue processing further } - if (state->curr_tx) { - state->curr_tx->tx_data.updated_tc = true; + SMTPTransaction *reply_tx = SMTPGetReplyTx(state); + if (reply_tx != NULL) { + reply_tx->tx_data.updated_tc = true; } /* the reply code has to contain at least 3 bytes, to hold the 3 digit * reply code */ @@ -931,8 +1057,8 @@ static int SMTPProcessReply( if (!SCAppLayerRequestProtocolTLSUpgrade(f)) { SMTPSetEvent(state, SMTP_DECODER_EVENT_FAILED_PROTOCOL_CHANGE); } - if (state->curr_tx) { - SMTPTransactionComplete(state); + if (reply_tx) { + SMTPTransactionComplete(reply_tx); } } else { /* decoder event */ @@ -940,6 +1066,7 @@ static int SMTPProcessReply( } } else if (IsReplyToCommand(state, SMTP_COMMAND_DATA)) { if (reply_code == SMTP_REPLY_354) { + SMTPSetProgressTC(reply_tx, SMTP_RESPONSE_DATA); /* Next comes the mail for the DATA command in toserver direction */ state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE; } else { @@ -950,10 +1077,25 @@ static int SMTPProcessReply( } SMTPSetEvent(state, SMTP_DECODER_EVENT_DATA_COMMAND_REJECTED); } + } else if (IsReplyToCommand(state, SMTP_COMMAND_BDAT)) { + SMTPSetProgressTC(reply_tx, SMTP_RESPONSE_DATA); + } else if (IsReplyToCommand(state, SMTP_COMMAND_BDAT_LAST)) { + if (reply_tx && !(state->parser_state & SMTP_PARSER_STATE_PARSING_MULTILINE_REPLY)) { + SMTPTransactionCompleteTC(reply_tx); + } + } else if (IsReplyToCommand(state, SMTP_COMMAND_DATA_MODE)) { + if (reply_tx && !(state->parser_state & SMTP_PARSER_STATE_PARSING_MULTILINE_REPLY)) { + SMTPTransactionCompleteTC(reply_tx); + } } else if (IsReplyToCommand(state, SMTP_COMMAND_RSET)) { - if (reply_code == SMTP_REPLY_250 && state->curr_tx && + if (reply_code == SMTP_REPLY_250 && reply_tx && !(state->parser_state & SMTP_PARSER_STATE_PARSING_MULTILINE_REPLY)) { - SMTPTransactionComplete(state); + SMTPTransactionComplete(reply_tx); + } + } else if (IsReplyToCommand(state, SMTP_COMMAND_QUIT)) { + if (reply_code == SMTP_REPLY_221 && reply_tx && + !(state->parser_state & SMTP_PARSER_STATE_PARSING_MULTILINE_REPLY)) { + SMTPTransactionComplete(reply_tx); } } else { /* we don't care for any other command for now */ @@ -982,10 +1124,12 @@ static int SMTPProcessReply( return 0; } -static int SMTPParseCommandBDAT(SMTPState *state, const SMTPLine *line) +static int SMTPParseCommandBDAT(SMTPState *state, const SMTPLine *line, bool *last) { SCEnter(); + *last = false; + int i = 4; while (i < line->len) { if (line->buf[i] != ' ') { @@ -1009,10 +1153,25 @@ static int SMTPParseCommandBDAT(SMTPState *state, const SMTPLine *line) } memcpy(strbuf, line->buf + i, len); strbuf[len] = '\0'; - if (ByteExtractStringUint32(&state->bdat_chunk_len, 10, 0, strbuf) < 0) { + int parsed = ByteExtractStringUint32(&state->bdat_chunk_len, 10, 0, strbuf); + if (parsed < 0) { /* decoder event */ return -1; } + state->bdat_chunk_idx = 0; + + i += parsed; + if (i < line->len && line->buf[i] != ' ') { + return -1; + } + while (i < line->len && line->buf[i] == ' ') { + i++; + } + if (line->len - i == 4 && SCMemcmpLowercase("last", line->buf + i, 4) == 0) { + *last = true; + } else if (i != line->len) { + return -1; + } return 0; } @@ -1151,7 +1310,9 @@ static int SMTPProcessRequest( if (line->len == 0 && line->delim_len == 0) { return 0; } - if (state->curr_tx == NULL || (state->curr_tx->done && !NoNewTx(state, line))) { + const bool no_new_tx = NoNewTx(state, line); + if ((state->curr_tx == NULL && (state->tx_cnt == 0 || !no_new_tx)) || + (SMTPTransactionRequestIsComplete(state->curr_tx) && !no_new_tx)) { tx = SMTPTransactionCreate(state); if (tx == NULL) return -1; @@ -1167,7 +1328,9 @@ static int SMTPProcessRequest( if (frame != NULL && state->curr_tx) { AppLayerFrameSetTxId(frame, state->curr_tx->tx_id); } - tx->tx_data.updated_ts = true; + if (tx != NULL) { + tx->tx_data.updated_ts = true; + } state->toserver_data_count += (line->len + line->delim_len); @@ -1181,10 +1344,17 @@ static int SMTPProcessRequest( int r = 0; AppLayerParserTriggerRawStreamInspection(f, STREAM_TOSERVER); - if (line->len >= 8 && SCMemcmpLowercase("starttls", line->buf, 8) == 0) { + if (tx == NULL) { + DEBUG_VALIDATE_BUG_ON(!no_new_tx); + const bool is_rset = SCMemcmpLowercase("rset", line->buf, 4) == 0; + if (is_rset) + state->bdat_chunk_idx = 0; + state->current_command = is_rset ? SMTP_COMMAND_RSET : SMTP_COMMAND_QUIT; + } else if (line->len >= 8 && SCMemcmpLowercase("starttls", line->buf, 8) == 0) { state->current_command = SMTP_COMMAND_STARTTLS; } else if (line->len >= 4 && SCMemcmpLowercase("data", line->buf, 4) == 0) { state->current_command = SMTP_COMMAND_DATA; + SMTPSetProgressTS(tx, SMTP_REQUEST_DATA); if (state->curr_tx->is_data) { // We did not receive a confirmation from server // And now client sends a next DATA @@ -1220,19 +1390,40 @@ static int SMTPProcessRequest( state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE; } } else if (line->len >= 4 && SCMemcmpLowercase("bdat", line->buf, 4) == 0) { - r = SMTPParseCommandBDAT(state, line); + bool last = false; + r = SMTPParseCommandBDAT(state, line, &last); if (r == -1) { - SCReturnInt(-1); + /* Invalid BDAT syntax is recoverable: the server rejects the + * command and the session continues. */ + SMTPSetEvent(state, SMTP_DECODER_EVENT_INVALID_BDAT); + state->current_command = SMTP_COMMAND_OTHER_CMD; + r = 0; + } else { + state->current_command = last ? SMTP_COMMAND_BDAT_LAST : SMTP_COMMAND_BDAT; + SMTPSetProgressTS(tx, SMTP_REQUEST_DATA); + if (state->bdat_chunk_len > 0) { + state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE; + } else if (last) { + SMTPTransactionCompleteTS(tx); + } } - state->current_command = SMTP_COMMAND_BDAT; - state->parser_state |= SMTP_PARSER_STATE_COMMAND_DATA_MODE; } else if (line->len >= 4 && ((SCMemcmpLowercase("helo", line->buf, 4) == 0) || SCMemcmpLowercase("ehlo", line->buf, 4) == 0)) { r = SMTPParseCommandHELO(state, line); if (r == -1) { SCReturnInt(-1); } - state->current_command = SMTP_COMMAND_OTHER_CMD; + if (state->curr_tx->mail_from != NULL || !TAILQ_EMPTY(&state->curr_tx->rcpt_to_list) || + state->curr_tx->progress_ts != SMTP_REQUEST_STARTED) { + /* Mid-session HELO/EHLO resets the state as if a RSET + * had been issued (RFC 5321 4.1.4). The progress check + * catches a transaction with no envelope but an attempted + * DATA or BDAT, such as a rejected envelope-less DATA. */ + state->bdat_chunk_idx = 0; + state->current_command = SMTP_COMMAND_RSET; + } else { + state->current_command = SMTP_COMMAND_OTHER_CMD; + } } else if (line->len >= 9 && SCMemcmpLowercase("mail from", line->buf, 9) == 0) { r = SMTPParseCommandMAILFROM(state, line); if (r == -1) { @@ -1249,13 +1440,15 @@ static int SMTPProcessRequest( // Resets chunk index in case of connection reuse state->bdat_chunk_idx = 0; state->current_command = SMTP_COMMAND_RSET; + } else if (line->len >= 4 && SCMemcmpLowercase("quit", line->buf, 4) == 0) { + state->current_command = SMTP_COMMAND_QUIT; } else { state->current_command = SMTP_COMMAND_OTHER_CMD; } /* Every command is inserted into a command buffer, to be matched * against reply(ies) sent by the server */ - if (SMTPInsertCommandIntoCommandBuffer(state->current_command, state) == -1) { + if (SMTPInsertCommandIntoCommandBuffer(state, state->current_command, tx) == -1) { SCReturnInt(-1); } @@ -1267,7 +1460,8 @@ static int SMTPProcessRequest( return SMTPProcessCommandDATA(state, tx, f, line); case SMTP_COMMAND_BDAT: - return SMTPProcessCommandBDAT(state, line); + case SMTP_COMMAND_BDAT_LAST: + return SMTPProcessCommandBDAT(state, tx, line); default: /* we have nothing to do with any other command at this instant. @@ -1285,16 +1479,37 @@ static inline void ResetLine(SMTPLine *line) } } +static int SMTPPreProcessCommandBDAT( + SMTPState *state, Flow *f, StreamSlice *slice, SMTPInput *input, SMTPLine *line) +{ + if (state->bdat_chunk_idx >= state->bdat_chunk_len) { + /* The BDAT chunk is already complete; data mode was set by another + * command, such as a pipelined DATA reply. Leave data mode and let + * the line parser handle the input as a new command. */ + state->parser_state &= ~SMTP_PARSER_STATE_COMMAND_DATA_MODE; + return 1; + } + uint32_t remaining = state->bdat_chunk_len - state->bdat_chunk_idx; + uint32_t consumed = MIN((uint32_t)input->len, remaining); + line->buf = input->buf + input->consumed; + line->len = consumed; + input->consumed += consumed; + input->len -= consumed; + int ret = SMTPProcessRequest(state, f, input, line, slice); + ResetLine(line); + return ret; +} + /* - * @brief Pre Process the data that comes in DATA mode. + * @brief Pre-process command data. + * + * If the command being processed is DATA, its data must be handled by this + * function so the line limit used by GetLine is not applied. GetLine caps lines + * at SMTP_LINE_BUFFER_LIMIT, which could truncate file data or parts of an + * e-mail if a line were too long. * - * If currently, the command that is being processed is DATA, whatever data - * comes as a part of it must be handled by this function. This is because - * there should be no char limit imposition on the line arriving in the DATA - * mode. Such limits are in place for any lines passed to the GetLine function - * and the lines are capped there at SMTP_LINE_BUFFER_LIMIT. - * One such limit in DATA mode may lead to file data or parts of e-mail being - * truncated if the line were too long. + * BDAT data is octet-counted and must be consumed only up to the declared chunk + * boundary. * * @param state Pointer to the current SMTPState * @param f Pointer to the current Flow @@ -1312,6 +1527,11 @@ static int SMTPPreProcessCommands( DEBUG_VALIDATE_BUG_ON(line->len != 0); DEBUG_VALIDATE_BUG_ON(line->delim_len != 0); + if (state->current_command == SMTP_COMMAND_BDAT || + state->current_command == SMTP_COMMAND_BDAT_LAST) { + return SMTPPreProcessCommandBDAT(state, f, slice, input, line); + } + /* fall back to strict line parsing for mime header parsing */ if (state->curr_tx && state->curr_tx->mime_state && SCMimeSmtpGetState(state->curr_tx->mime_state) < MimeSmtpBody) @@ -1403,7 +1623,8 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state, /* toserver */ if (direction == 0) { if (((state->current_command == SMTP_COMMAND_DATA) || - (state->current_command == SMTP_COMMAND_BDAT)) && + (state->current_command == SMTP_COMMAND_BDAT) || + (state->current_command == SMTP_COMMAND_BDAT_LAST)) && (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { int ret = SMTPPreProcessCommands(state, f, &stream_slice, &input, &line); DEBUG_VALIDATE_BUG_ON(ret != 0 && ret != -1 && ret != 1); @@ -1430,11 +1651,15 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state, * wherever it had to be */ ResetLine(&line); - /* If DATA mode was entered in the middle of input parsing, exempt it from GetLine as we - * don't want input limits to be exercised on DATA data. Here, SMTPPreProcessCommands - * should either consume all the data or return in case it encounters another boundary. - * In case of another boundary, the control should be passed to SMTPGetLine */ - if ((input.len > 0) && (state->current_command == SMTP_COMMAND_DATA) && + /* If command data mode was entered in the middle of input parsing, first pass it to + * SMTPPreProcessCommands so input limits are not applied to DATA bodies and BDAT data + * is not consumed past its chunk boundary. SMTPPreProcessCommands should either + * consume all remaining input or stop at a MIME or BDAT chunk boundary, after which + * control is passed to SMTPGetLine. */ + if ((input.len > 0) && + ((state->current_command == SMTP_COMMAND_DATA) || + (state->current_command == SMTP_COMMAND_BDAT) || + (state->current_command == SMTP_COMMAND_BDAT_LAST)) && (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { int ret = SMTPPreProcessCommands(state, f, &stream_slice, &input, &line); DEBUG_VALIDATE_BUG_ON(ret != 0 && ret != -1 && ret != 1); @@ -1505,6 +1730,12 @@ void *SMTPStateAlloc(void *orig_state, AppProto proto_orig) SCFree(smtp_state); return NULL; } + smtp_state->cmds_tx_ids = SCMalloc(sizeof(uint64_t) * SMTP_COMMAND_BUFFER_STEPS); + if (smtp_state->cmds_tx_ids == NULL) { + SCFree(smtp_state->cmds); + SCFree(smtp_state); + return NULL; + } smtp_state->cmds_buffer_len = SMTP_COMMAND_BUFFER_STEPS; TAILQ_INIT(&smtp_state->tx_list); @@ -1602,6 +1833,9 @@ static void SMTPStateFree(void *p) if (smtp_state->cmds != NULL) { SCFree(smtp_state->cmds); } + if (smtp_state->cmds_tx_ids != NULL) { + SCFree(smtp_state->cmds_tx_ids); + } if (smtp_state->helo) { SCFree(smtp_state->helo); @@ -1790,7 +2024,10 @@ static void *SMTPStateGetTx(void *state, uint64_t id) static int SMTPStateGetAlstateProgress(void *vtx, uint8_t direction) { SMTPTransaction *tx = vtx; - return tx->done; + if (direction & STREAM_TOSERVER) { + return tx->progress_ts; + } + return tx->progress_tc; } static AppLayerGetFileState SMTPGetTxFiles(void *txv, uint8_t direction) @@ -1893,9 +2130,12 @@ void RegisterSMTPParsers(void) AppLayerParserRegisterGetTxIterator(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxIterator); AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxData); AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetStateData); - AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_SMTP, 1, 1); + AppLayerParserRegisterStateProgressCompletionStatus( + ALPROTO_SMTP, SMTP_REQUEST_COMPLETE, SMTP_RESPONSE_COMPLETE); AppLayerParserRegisterGetFrameFuncs( IPPROTO_TCP, ALPROTO_SMTP, SMTPGetFrameIdByName, SMTPGetFrameNameById); + AppLayerParserRegisterGetStateFuncs( + IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetStateIdByName, SMTPStateGetStateNameById); } else { SCLogInfo("Parser disabled for %s protocol. Protocol detection still on.", proto_name); } @@ -2708,7 +2948,7 @@ static int SMTPParserTest02(void) goto end; } if (smtp_state->cmds_cnt != 1 || smtp_state->cmds_idx != 0 || - smtp_state->cmds[0] != SMTP_COMMAND_OTHER_CMD || + smtp_state->cmds[0] != SMTP_COMMAND_QUIT || smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) { printf("smtp parser in inconsistent state\n"); goto end; @@ -3190,7 +3430,7 @@ static int SMTPParserTest05(void) goto end; } if (smtp_state->cmds_cnt != 1 || smtp_state->cmds_idx != 0 || - smtp_state->cmds[0] != SMTP_COMMAND_OTHER_CMD || + smtp_state->cmds[0] != SMTP_COMMAND_QUIT || smtp_state->parser_state != (SMTP_PARSER_STATE_FIRST_REPLY_SEEN | SMTP_PARSER_STATE_PIPELINING_SERVER)) { printf("smtp parser in inconsistent state\n"); @@ -4213,7 +4453,7 @@ static int SMTPParserTest14(void) goto end; } if (smtp_state->cmds_cnt != 1 || smtp_state->cmds_idx != 0 || - smtp_state->cmds[0] != SMTP_COMMAND_OTHER_CMD || + smtp_state->cmds[0] != SMTP_COMMAND_QUIT || smtp_state->parser_state != SMTP_PARSER_STATE_FIRST_REPLY_SEEN) { printf("smtp parser in inconsistent state l.%d\n", __LINE__); goto end; @@ -4240,6 +4480,7 @@ static int SMTPParserTest14(void) FLOW_DESTROY(&f); return result; } + #endif /* UNITTESTS */ void SMTPParserRegisterTests(void) diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h index cd9c614b966a..b6cb3964d281 100644 --- a/src/app-layer-smtp.h +++ b/src/app-layer-smtp.h @@ -38,6 +38,7 @@ enum { SMTP_DECODER_EVENT_MAX_REPLY_LINE_LEN_EXCEEDED, SMTP_DECODER_EVENT_INVALID_PIPELINED_SEQUENCE, SMTP_DECODER_EVENT_BDAT_CHUNK_LEN_EXCEEDED, + SMTP_DECODER_EVENT_INVALID_BDAT, SMTP_DECODER_EVENT_NO_SERVER_WELCOME_MESSAGE, SMTP_DECODER_EVENT_TLS_REJECTED, SMTP_DECODER_EVENT_DATA_COMMAND_REJECTED, @@ -69,14 +70,28 @@ typedef struct SMTPString_ { TAILQ_ENTRY(SMTPString_) next; } SMTPString; +enum SMTPRequestProgress { + SMTP_REQUEST_STARTED = 0, + SMTP_REQUEST_DATA = 1, + SMTP_REQUEST_COMPLETE = 2, +}; + +enum SMTPResponseProgress { + SMTP_RESPONSE_STARTED = 0, + SMTP_RESPONSE_DATA = 1, + SMTP_RESPONSE_COMPLETE = 2, +}; + typedef struct SMTPTransaction_ { /** id of this tx, starting at 0 */ uint64_t tx_id; AppLayerTxData tx_data; - /** the tx is complete and can be logged and cleaned */ - bool done; + /** to-server firewall progress state. */ + uint8_t progress_ts; + /** to-client firewall progress state. */ + uint8_t progress_tc; /** the tx has seen a DATA command */ // another DATA command within the same context // will trigger an app-layer event. @@ -138,6 +153,8 @@ typedef struct SMTPState_ { * stored command in the buffer to match the reply(ies) with the command */ /** the command buffer */ uint8_t *cmds; + /** tx id for each stored command */ + uint64_t *cmds_tx_ids; /** the buffer length */ uint16_t cmds_buffer_len; /** no of commands stored in the above buffer */ diff --git a/src/detect-email.c b/src/detect-email.c index 26bd4974ce53..f5538c83dd5d 100644 --- a/src/detect-email.c +++ b/src/detect-email.c @@ -235,8 +235,9 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailFromSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_from_buffer_id = SCDetectHelperBufferMpmRegister( - "email.from", "MIME EMAIL FROM", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailFromData); + g_mime_email_from_buffer_id = + SCDetectHelperBufferProgressMpmRegister("email.from", "MIME EMAIL FROM", ALPROTO_SMTP, + STREAM_TOSERVER, GetMimeEmailFromData, SMTP_REQUEST_DATA); kw.name = "email.subject"; kw.desc = "'Subject' field from an email"; @@ -244,8 +245,9 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailSubjectSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_subject_buffer_id = SCDetectHelperBufferMpmRegister("email.subject", - "MIME EMAIL SUBJECT", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailSubjectData); + g_mime_email_subject_buffer_id = + SCDetectHelperBufferProgressMpmRegister("email.subject", "MIME EMAIL SUBJECT", + ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailSubjectData, SMTP_REQUEST_DATA); kw.name = "email.to"; kw.desc = "'To' field from an email"; @@ -253,8 +255,8 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailToSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_to_buffer_id = SCDetectHelperBufferMpmRegister( - "email.to", "MIME EMAIL TO", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailToData); + g_mime_email_to_buffer_id = SCDetectHelperBufferProgressMpmRegister("email.to", "MIME EMAIL TO", + ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailToData, SMTP_REQUEST_DATA); kw.name = "email.cc"; kw.desc = "'Cc' field from an email"; @@ -262,8 +264,8 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailCcSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_cc_buffer_id = SCDetectHelperBufferMpmRegister( - "email.cc", "MIME EMAIL CC", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailCcData); + g_mime_email_cc_buffer_id = SCDetectHelperBufferProgressMpmRegister("email.cc", "MIME EMAIL CC", + ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailCcData, SMTP_REQUEST_DATA); kw.name = "email.date"; kw.desc = "'Date' field from an email"; @@ -271,8 +273,9 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailDateSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_date_buffer_id = SCDetectHelperBufferMpmRegister( - "email.date", "MIME EMAIL DATE", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailDateData); + g_mime_email_date_buffer_id = + SCDetectHelperBufferProgressMpmRegister("email.date", "MIME EMAIL DATE", ALPROTO_SMTP, + STREAM_TOSERVER, GetMimeEmailDateData, SMTP_REQUEST_DATA); kw.name = "email.message_id"; kw.desc = "'Message-Id' field from an email"; @@ -280,8 +283,9 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailMessageIdSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_message_id_buffer_id = SCDetectHelperBufferMpmRegister("email.message_id", - "MIME EMAIL Message-Id", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailMessageIdData); + g_mime_email_message_id_buffer_id = + SCDetectHelperBufferProgressMpmRegister("email.message_id", "MIME EMAIL Message-Id", + ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailMessageIdData, SMTP_REQUEST_DATA); kw.name = "email.x_mailer"; kw.desc = "'X-Mailer' field from an email"; @@ -289,8 +293,9 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailXMailerSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_x_mailer_buffer_id = SCDetectHelperBufferMpmRegister("email.x_mailer", - "MIME EMAIL X-Mailer", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailXMailerData); + g_mime_email_x_mailer_buffer_id = + SCDetectHelperBufferProgressMpmRegister("email.x_mailer", "MIME EMAIL X-Mailer", + ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailXMailerData, SMTP_REQUEST_DATA); kw.name = "email.url"; kw.desc = "'Url' extracted from an email"; @@ -298,8 +303,9 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailUrlSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_url_buffer_id = SCDetectHelperMultiBufferMpmRegister( - "email.url", "MIME EMAIL URL", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailUrlData); + g_mime_email_url_buffer_id = + SCDetectHelperMultiBufferProgressMpmRegister("email.url", "MIME EMAIL URL", + ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailUrlData, SMTP_REQUEST_DATA); kw.name = "email.received"; kw.desc = "'Received' field from an email"; @@ -307,6 +313,7 @@ void DetectEmailRegister(void) kw.Setup = DetectMimeEmailReceivedSetup; kw.flags = SIGMATCH_NOOPT | SIGMATCH_INFO_STICKY_BUFFER; SCDetectHelperKeywordRegister(&kw); - g_mime_email_received_buffer_id = SCDetectHelperMultiBufferMpmRegister("email.received", - "MIME EMAIL RECEIVED", ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailReceivedData); + g_mime_email_received_buffer_id = + SCDetectHelperMultiBufferProgressMpmRegister("email.received", "MIME EMAIL RECEIVED", + ALPROTO_SMTP, STREAM_TOSERVER, GetMimeEmailReceivedData, SMTP_REQUEST_DATA); } diff --git a/src/detect-file-data.c b/src/detect-file-data.c index e5f28d8b9f4b..77f68de01ca1 100644 --- a/src/detect-file-data.c +++ b/src/detect-file-data.c @@ -93,7 +93,10 @@ DetectFileHandlerProtocol_t al_protocols[ALPROTO_WITHFILES_MAX] = { .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT, .to_client_progress = HTTP2StateDataServer, .to_server_progress = HTTP2StateDataClient }, - { .alproto = ALPROTO_SMTP, .direction = SIG_FLAG_TOSERVER }, { .alproto = ALPROTO_UNKNOWN } + { .alproto = ALPROTO_SMTP, + .direction = SIG_FLAG_TOSERVER, + .to_server_progress = SMTP_REQUEST_DATA }, + { .alproto = ALPROTO_UNKNOWN } }; void DetectFileRegisterProto(