From db0f9ddc697eff5d6f5bc1f2f0d06f7307a62750 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 14 Sep 2022 15:38:04 +0200 Subject: [PATCH] files/tx: inspection, logging and loop optimizations Introduce AppLayerTxData::file_tx as direction(s) indicator for transactions. When set to 0, its not a file tx and it will not be considered for file inspection, logging and housekeeping tasks. Various tx loop optimizations in housekeeping and output. Update the "file capable" app-layers to set the fields based on their directional file support as well as on the traffic. --- rust/src/applayer.rs | 8 +++++ rust/src/http2/http2.rs | 3 ++ rust/src/nfs/nfs.rs | 1 + rust/src/smb/files.rs | 1 + src/app-layer-ftp.c | 3 ++ src/app-layer-htp.c | 3 ++ src/app-layer-parser.c | 24 +++++++++----- src/app-layer-parser.h | 22 +++++++++++++ src/app-layer-smtp.c | 1 + src/output-tx.c | 69 +++++++++++++++++++++++++++++------------ 10 files changed, 108 insertions(+), 27 deletions(-) diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs index 85cfedb45b..bea80bb02c 100644 --- a/rust/src/applayer.rs +++ b/rust/src/applayer.rs @@ -111,6 +111,13 @@ pub struct AppLayerTxData { pub file_flags: u16, + /// Indicated if a file tracking tx, and if so in which direction: + /// 0: not a file tx + /// STREAM_TOSERVER: file tx, files only in toserver dir + /// STREAM_TOCLIENT: file tx , files only in toclient dir + /// STREAM_TOSERVER|STREAM_TOCLIENT: files possible in both dirs + pub file_tx: u8, + /// detection engine flags for use by detection engine detect_flags_ts: u64, detect_flags_tc: u64, @@ -145,6 +152,7 @@ impl AppLayerTxData { files_logged: 0, files_stored: 0, file_flags: 0, + file_tx: 0, detect_flags_ts: 0, detect_flags_tc: 0, de_state: std::ptr::null_mut(), diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index e9789dd482..1586e22c99 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -551,6 +551,8 @@ impl HTTP2State { tx.tx_id = self.tx_id; tx.state = HTTP2TransactionState::HTTP2StateGlobal; tx.tx_data.update_file_flags(self.state_data.file_flags); + // TODO can this tx hold files? + tx.tx_data.file_tx = STREAM_TOSERVER|STREAM_TOCLIENT; // might hold files in both directions tx.update_file_flags(tx.tx_data.file_flags); self.transactions.push_back(tx); return self.transactions.back_mut().unwrap(); @@ -611,6 +613,7 @@ impl HTTP2State { } tx.tx_data.update_file_flags(self.state_data.file_flags); tx.update_file_flags(tx.tx_data.file_flags); + tx.tx_data.file_tx = STREAM_TOSERVER|STREAM_TOCLIENT; // might hold files in both directions self.transactions.push_back(tx); return self.transactions.back_mut().unwrap(); } diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs index 7e48897abe..76e8a2180f 100644 --- a/rust/src/nfs/nfs.rs +++ b/rust/src/nfs/nfs.rs @@ -701,6 +701,7 @@ impl NFSState { d.update_file_flags(tx.tx_data.file_flags); } tx.tx_data.init_files_opened(); + tx.tx_data.file_tx = if direction == Direction::ToServer { STREAM_TOSERVER } else { STREAM_TOCLIENT }; // TODO direction to flag func? SCLogDebug!("new_file_tx: TX FILE created: ID {} NAME {}", tx.id, String::from_utf8_lossy(file_name)); self.transactions.push(tx); diff --git a/rust/src/smb/files.rs b/rust/src/smb/files.rs index d7d6625217..e58f037e4b 100644 --- a/rust/src/smb/files.rs +++ b/rust/src/smb/files.rs @@ -81,6 +81,7 @@ impl SMBState { _ => { }, } tx.tx_data.init_files_opened(); + tx.tx_data.file_tx = if direction == Direction::ToServer { STREAM_TOSERVER } else { STREAM_TOCLIENT }; // TODO direction to flag func? SCLogDebug!("SMB: new_file_tx: TX FILE created: ID {} NAME {}", tx.id, String::from_utf8_lossy(file_name)); self.transactions.push(tx); diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index e508258a24..72be86cf41 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -1102,6 +1102,9 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state, : AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TC) != 0; ftpdata_state->tx_data.file_flags |= ftpdata_state->state_data.file_flags; + if (ftpdata_state->tx_data.file_tx == 0) + ftpdata_state->tx_data.file_tx = direction & (STREAM_TOSERVER | STREAM_TOCLIENT); + /* we depend on detection engine for file pruning */ const uint16_t flags = FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction) | FILE_USE_DETECT; diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 3ea59f5f7e..d7a8cd0a38 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -2121,6 +2121,7 @@ static int HTPCallbackRequestStart(htp_tx_t *tx) if (unlikely(tx_ud == NULL)) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files htp_tx_set_user_data(tx, tx_ud); } SCReturnInt(HTP_OK); @@ -2159,6 +2160,8 @@ static int HTPCallbackResponseStart(htp_tx_t *tx) if (unlikely(tx_ud == NULL)) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.file_tx = + STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed. htp_tx_set_user_data(tx, tx_ud); } SCReturnInt(HTP_OK); diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index db179d92b9..3b33912bd1 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -920,16 +920,14 @@ FileContainer *AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t d SCReturnPtr(ptr, "FileContainer *"); } -static void AppLayerParserFileTxHousekeeping(const Flow *f, void *tx, const uint8_t pkt_dir) +static void AppLayerParserFileTxHousekeeping( + const Flow *f, void *tx, const uint8_t pkt_dir, const bool trunc) { FileContainer *fc = AppLayerParserGetTxFiles(f, tx, pkt_dir); if (fc) { - if (AppLayerParserStateIssetFlag(f->alparser, (pkt_dir == STREAM_TOSERVER) - ? APP_LAYER_PARSER_TRUNC_TS - : APP_LAYER_PARSER_TRUNC_TC) != 0) { + if (trunc) { FileTruncateAllOpenFiles(fc); } - FilePrune(fc); } } @@ -969,6 +967,8 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER); const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT); + int pkt_dir_trunc = -1; + AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto); AppLayerGetTxIterState state; memset(&state, 0, sizeof(state)); @@ -978,6 +978,7 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) bool skipped = false; const bool is_unidir = AppLayerParserGetOptionFlags(f->protomap, f->alproto) & APP_LAYER_PARSER_OPT_UNIDIR_TXS; + // const bool support_files = AppLayerParserSupportsFiles(f->proto, f->alproto); while (1) { AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, i, total_txs, &state); @@ -989,8 +990,16 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) i = ires.tx_id; // actual tx id for the tx the IterFunc returned SCLogDebug("%p/%"PRIu64" checking", tx, i); - - AppLayerParserFileTxHousekeeping(f, tx, pkt_dir); + AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx); + if (txd != NULL && AppLayerParserHasFilesInDir(txd, pkt_dir)) { + if (pkt_dir_trunc == -1) + pkt_dir_trunc = + AppLayerParserStateIssetFlag(f->alparser, + (pkt_dir == STREAM_TOSERVER) ? APP_LAYER_PARSER_TRUNC_TS + : APP_LAYER_PARSER_TRUNC_TC) != 0; + + AppLayerParserFileTxHousekeeping(f, tx, pkt_dir, (bool)pkt_dir_trunc); + } const int tx_progress_tc = AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); @@ -1007,7 +1016,6 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) goto next; } - AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx); bool inspected = false; if (txd && has_tx_detect_flags) { if (!IS_DISRUPTED(ts_disrupt_flags) && f->sgh_toserver != NULL) { diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index 181c3745d5..71ab593ceb 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -263,6 +263,28 @@ AppLayerStateData *AppLayerParserGetStateData(uint8_t ipproto, AppProto alproto, void AppLayerParserApplyTxConfig(uint8_t ipproto, AppProto alproto, void *state, void *tx, enum ConfigAction mode, AppLayerTxConfig); +static inline bool AppLayerParserIsFileTx(const AppLayerTxData *txd) +{ + if (txd->file_tx != 0) { + return true; + } + return false; +} + +static inline bool AppLayerParserIsFileTxInDir(const AppLayerTxData *txd, const uint8_t direction) +{ + if ((txd->file_tx & direction) != 0) { + return true; + } + return false; +} + +/** \brief check if tx (possibly) has files in this tx for the direction */ +static inline bool AppLayerParserHasFilesInDir(const AppLayerTxData *txd, const uint8_t direction) +{ + return (txd->files_opened && AppLayerParserIsFileTxInDir(txd, direction)); +} + /***** General *****/ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *tctx, Flow *f, AppProto alproto, diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 7fb65050d1..f30c372a99 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -439,6 +439,7 @@ static SMTPTransaction *SMTPTransactionCreate(void) TAILQ_INIT(&tx->rcpt_to_list); tx->mime_state = NULL; + tx->tx_data.file_tx = STREAM_TOSERVER; // can xfer files return tx; } diff --git a/src/output-tx.c b/src/output-tx.c index 310455fd8d..2e2d3a578d 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -27,6 +27,7 @@ #include "tm-modules.h" #include "output.h" #include "output-tx.h" +#include "stream.h" #include "app-layer.h" #include "app-layer-parser.h" #include "util-profiling.h" @@ -384,13 +385,21 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) uint64_t max_id = tx_id; int logged = 0; int gap = 0; + const bool file_logging_active = (op_thread_data->file || op_thread_data->filedata); + const bool support_files = AppLayerParserSupportsFiles(p->proto, alproto); + const uint8_t pkt_dir = STREAM_FLAGS_FOR_PACKET(p); - SCLogDebug("tx_id %" PRIu64 " total_txs %" PRIu64, tx_id, total_txs); + SCLogDebug("pcap_cnt %" PRIu64 ": tx_id %" PRIu64 " total_txs %" PRIu64, p->pcap_cnt, tx_id, + total_txs); AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto); AppLayerGetTxIterState state; memset(&state, 0, sizeof(state)); + const int complete_ts = + AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOSERVER); + const int complete_tc = + AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOCLIENT); while (1) { AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state); if (ires.tx_ptr == NULL) @@ -403,35 +412,57 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) AppLayerParserGetStateProgress(p->proto, alproto, tx, ts_disrupt_flags); const int tx_progress_tc = AppLayerParserGetStateProgress(p->proto, alproto, tx, tc_disrupt_flags); - const bool tx_complete = (tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus( - alproto, STREAM_TOSERVER) && - tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus( - alproto, STREAM_TOCLIENT)); - const bool ts_ready = tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus( - alproto, STREAM_TOSERVER); - const bool tc_ready = tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus( - alproto, STREAM_TOCLIENT); - SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready); + const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc); SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file, op_thread_data->filedata); AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx); - if (txd && (op_thread_data->file || op_thread_data->filedata) && - AppLayerParserSupportsFiles(p->proto, alproto)) { - OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx, tx_id, - txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof); - } - if (txd) - SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags); - - if (txd == NULL) { + if (unlikely(txd == NULL)) { + SCLogDebug("NO TXD"); /* make sure this tx, which can't be properly logged is skipped */ logged = 1; max_id = tx_id; goto next_tx; } + if (file_logging_active) { + if (AppLayerParserIsFileTx(txd)) { // need to process each tx that might be a file tx, + // even if there are not files (yet) + const bool ts_ready = (tx_progress_ts == complete_ts); + const bool tc_ready = (tx_progress_tc == complete_tc); + SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready); + + const bool eval_files = ts_ready | tc_ready | tx_complete | ts_eof | tc_eof | eof; + + SCLogDebug("eval_files: %u, ts_ready %u, tc_ready %u, tx_complete %u, ts_eof %u, " + "tc_eof %u, eof %u", + eval_files, ts_ready, tc_ready, tx_complete, ts_eof, tc_eof, eof); + SCLogDebug("txd->file_tx & pkt_dir: %02x & %02x -> %02x", txd->file_tx, pkt_dir, + (txd->file_tx & pkt_dir)); + + /* call only for the correct direction, except when it looks anything like a end of + * transaction or end of stream. Since OutputTxLogFiles has complicated logic around + * that, we just leave it to that function to sort things out for now. */ + if (eval_files || AppLayerParserIsFileTxInDir( + txd, pkt_dir)) { // need to process each tx that might + // be a file tx, even if there + OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx, + tx_id, txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof); + } + } else if (support_files) { + if (op_thread_data->file) { + txd->logged.flags |= BIT_U32(LOGGER_FILE); + SCLogDebug("not a file_tx: setting LOGGER_FILE => %08x", txd->logged.flags); + } + if (op_thread_data->filedata) { + txd->logged.flags |= BIT_U32(LOGGER_FILEDATA); + SCLogDebug("not a file_tx: setting LOGGER_FILEDATA => %08x", txd->logged.flags); + } + } + } + SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags); + if (list[ALPROTO_UNKNOWN] != 0) { OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id); if (list[alproto] == NULL)