files/tx: inspection, logging and loop optimizations

Introduce AppLayerTxData::file_tx as direction(s) indicator for transactions.
When set to 0, its not a file tx and it will not be considered for file
inspection, logging and housekeeping tasks.

Various tx loop optimizations in housekeeping and output.

Update the "file capable" app-layers to set the fields based on their
directional file support as well as on the traffic.

rust/src/applayer.rs

@@ -111,6 +111,13 @@ pub struct AppLayerTxData {
 
     pub file_flags: u16,
 
+    /// Indicated if a file tracking tx, and if so in which direction:
+    ///  0: not a file tx
+    /// STREAM_TOSERVER: file tx, files only in toserver dir
+    /// STREAM_TOCLIENT: file tx , files only in toclient dir
+    /// STREAM_TOSERVER|STREAM_TOCLIENT: files possible in both dirs
+    pub file_tx: u8,
+
     /// detection engine flags for use by detection engine
     detect_flags_ts: u64,
     detect_flags_tc: u64,
@@ -145,6 +152,7 @@ impl AppLayerTxData {
             files_logged: 0,
             files_stored: 0,
             file_flags: 0,
+            file_tx: 0,
             detect_flags_ts: 0,
             detect_flags_tc: 0,
             de_state: std::ptr::null_mut(),

rust/src/http2/http2.rs

@@ -551,6 +551,8 @@ impl HTTP2State {
         tx.tx_id = self.tx_id;
         tx.state = HTTP2TransactionState::HTTP2StateGlobal;
         tx.tx_data.update_file_flags(self.state_data.file_flags);
+        // TODO can this tx hold files?
+        tx.tx_data.file_tx = STREAM_TOSERVER|STREAM_TOCLIENT; // might hold files in both directions
         tx.update_file_flags(tx.tx_data.file_flags);
         self.transactions.push_back(tx);
         return self.transactions.back_mut().unwrap();
@@ -611,6 +613,7 @@ impl HTTP2State {
             }
             tx.tx_data.update_file_flags(self.state_data.file_flags);
             tx.update_file_flags(tx.tx_data.file_flags);
+            tx.tx_data.file_tx = STREAM_TOSERVER|STREAM_TOCLIENT; // might hold files in both directions
             self.transactions.push_back(tx);
             return self.transactions.back_mut().unwrap();
         }

rust/src/nfs/nfs.rs

@@ -701,6 +701,7 @@ impl NFSState {
             d.update_file_flags(tx.tx_data.file_flags);
         }
         tx.tx_data.init_files_opened();
+        tx.tx_data.file_tx = if direction == Direction::ToServer { STREAM_TOSERVER } else { STREAM_TOCLIENT }; // TODO direction to flag func?
         SCLogDebug!("new_file_tx: TX FILE created: ID {} NAME {}",
                 tx.id, String::from_utf8_lossy(file_name));
         self.transactions.push(tx);

rust/src/smb/files.rs

@@ -81,6 +81,7 @@ impl SMBState {
             _ => { },
         }
         tx.tx_data.init_files_opened();
+        tx.tx_data.file_tx = if direction == Direction::ToServer { STREAM_TOSERVER } else { STREAM_TOCLIENT }; // TODO direction to flag func?
         SCLogDebug!("SMB: new_file_tx: TX FILE created: ID {} NAME {}",
                 tx.id, String::from_utf8_lossy(file_name));
         self.transactions.push(tx);

src/app-layer-ftp.c

@@ -1102,6 +1102,9 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state,
                              : AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TC) != 0;
 
     ftpdata_state->tx_data.file_flags |= ftpdata_state->state_data.file_flags;
+    if (ftpdata_state->tx_data.file_tx == 0)
+        ftpdata_state->tx_data.file_tx = direction & (STREAM_TOSERVER | STREAM_TOCLIENT);
+
     /* we depend on detection engine for file pruning */
     const uint16_t flags =
             FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction) | FILE_USE_DETECT;

src/app-layer-htp.c

@@ -2121,6 +2121,7 @@ static int HTPCallbackRequestStart(htp_tx_t *tx)
         if (unlikely(tx_ud == NULL)) {
             SCReturnInt(HTP_OK);
         }
+        tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files
         htp_tx_set_user_data(tx, tx_ud);
     }
     SCReturnInt(HTP_OK);
@@ -2159,6 +2160,8 @@ static int HTPCallbackResponseStart(htp_tx_t *tx)
         if (unlikely(tx_ud == NULL)) {
             SCReturnInt(HTP_OK);
         }
+        tx_ud->tx_data.file_tx =
+                STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed.
         htp_tx_set_user_data(tx, tx_ud);
     }
     SCReturnInt(HTP_OK);

src/app-layer-parser.c

@@ -920,16 +920,14 @@ FileContainer *AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t d
     SCReturnPtr(ptr, "FileContainer *");
 }
 
-static void AppLayerParserFileTxHousekeeping(const Flow *f, void *tx, const uint8_t pkt_dir)
+static void AppLayerParserFileTxHousekeeping(
+        const Flow *f, void *tx, const uint8_t pkt_dir, const bool trunc)
 {
     FileContainer *fc = AppLayerParserGetTxFiles(f, tx, pkt_dir);
     if (fc) {
-        if (AppLayerParserStateIssetFlag(f->alparser, (pkt_dir == STREAM_TOSERVER)
-                                                              ? APP_LAYER_PARSER_TRUNC_TS
-                                                              : APP_LAYER_PARSER_TRUNC_TC) != 0) {
+        if (trunc) {
             FileTruncateAllOpenFiles(fc);
         }
-
         FilePrune(fc);
     }
 }
@@ -969,6 +967,8 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir)
     const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER);
     const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT);
 
+    int pkt_dir_trunc = -1;
+
     AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
     AppLayerGetTxIterState state;
     memset(&state, 0, sizeof(state));
@@ -978,6 +978,7 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir)
     bool skipped = false;
     const bool is_unidir =
             AppLayerParserGetOptionFlags(f->protomap, f->alproto) & APP_LAYER_PARSER_OPT_UNIDIR_TXS;
+    // const bool support_files = AppLayerParserSupportsFiles(f->proto, f->alproto);
 
     while (1) {
         AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, i, total_txs, &state);
@@ -989,8 +990,16 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir)
         i = ires.tx_id; // actual tx id for the tx the IterFunc returned
 
         SCLogDebug("%p/%"PRIu64" checking", tx, i);
+        AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx);
+        if (txd != NULL && AppLayerParserHasFilesInDir(txd, pkt_dir)) {
+            if (pkt_dir_trunc == -1)
+                pkt_dir_trunc =
+                        AppLayerParserStateIssetFlag(f->alparser,
+                                (pkt_dir == STREAM_TOSERVER) ? APP_LAYER_PARSER_TRUNC_TS
+                                                             : APP_LAYER_PARSER_TRUNC_TC) != 0;
 
-        AppLayerParserFileTxHousekeeping(f, tx, pkt_dir);
+            AppLayerParserFileTxHousekeeping(f, tx, pkt_dir, (bool)pkt_dir_trunc);
+        }
 
         const int tx_progress_tc =
                 AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags);
@@ -1007,7 +1016,6 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir)
             goto next;
         }
 
-        AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx);
         bool inspected = false;
         if (txd && has_tx_detect_flags) {
             if (!IS_DISRUPTED(ts_disrupt_flags) && f->sgh_toserver != NULL) {

src/app-layer-parser.h

@@ -263,6 +263,28 @@ AppLayerStateData *AppLayerParserGetStateData(uint8_t ipproto, AppProto alproto,
 void AppLayerParserApplyTxConfig(uint8_t ipproto, AppProto alproto,
         void *state, void *tx, enum ConfigAction mode, AppLayerTxConfig);
 
+static inline bool AppLayerParserIsFileTx(const AppLayerTxData *txd)
+{
+    if (txd->file_tx != 0) {
+        return true;
+    }
+    return false;
+}
+
+static inline bool AppLayerParserIsFileTxInDir(const AppLayerTxData *txd, const uint8_t direction)
+{
+    if ((txd->file_tx & direction) != 0) {
+        return true;
+    }
+    return false;
+}
+
+/** \brief check if tx (possibly) has files in this tx for the direction */
+static inline bool AppLayerParserHasFilesInDir(const AppLayerTxData *txd, const uint8_t direction)
+{
+    return (txd->files_opened && AppLayerParserIsFileTxInDir(txd, direction));
+}
+
 /***** General *****/
 
 int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *tctx, Flow *f, AppProto alproto,

src/app-layer-smtp.c

@@ -439,6 +439,7 @@ static SMTPTransaction *SMTPTransactionCreate(void)
 
     TAILQ_INIT(&tx->rcpt_to_list);
     tx->mime_state = NULL;
+    tx->tx_data.file_tx = STREAM_TOSERVER; // can xfer files
     return tx;
 }
 

src/output-tx.c

@@ -27,6 +27,7 @@
 #include "tm-modules.h"
 #include "output.h"
 #include "output-tx.h"
+#include "stream.h"
 #include "app-layer.h"
 #include "app-layer-parser.h"
 #include "util-profiling.h"
@@ -384,13 +385,21 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
     uint64_t max_id = tx_id;
     int logged = 0;
     int gap = 0;
+    const bool file_logging_active = (op_thread_data->file || op_thread_data->filedata);
+    const bool support_files = AppLayerParserSupportsFiles(p->proto, alproto);
+    const uint8_t pkt_dir = STREAM_FLAGS_FOR_PACKET(p);
 
-    SCLogDebug("tx_id %" PRIu64 " total_txs %" PRIu64, tx_id, total_txs);
+    SCLogDebug("pcap_cnt %" PRIu64 ": tx_id %" PRIu64 " total_txs %" PRIu64, p->pcap_cnt, tx_id,
+            total_txs);
 
     AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
     AppLayerGetTxIterState state;
     memset(&state, 0, sizeof(state));
 
+    const int complete_ts =
+            AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOSERVER);
+    const int complete_tc =
+            AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOCLIENT);
     while (1) {
         AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
         if (ires.tx_ptr == NULL)
@@ -403,35 +412,57 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
                 AppLayerParserGetStateProgress(p->proto, alproto, tx, ts_disrupt_flags);
         const int tx_progress_tc =
                 AppLayerParserGetStateProgress(p->proto, alproto, tx, tc_disrupt_flags);
-        const bool tx_complete = (tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus(
-                                                            alproto, STREAM_TOSERVER) &&
-                                  tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus(
-                                                            alproto, STREAM_TOCLIENT));
-        const bool ts_ready = tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus(
-                                                        alproto, STREAM_TOSERVER);
-        const bool tc_ready = tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus(
-                                                        alproto, STREAM_TOCLIENT);
-        SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready);
+        const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc);
 
         SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file,
                 op_thread_data->filedata);
 
         AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx);
-        if (txd && (op_thread_data->file || op_thread_data->filedata) &&
-                AppLayerParserSupportsFiles(p->proto, alproto)) {
-            OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx, tx_id,
-                    txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof);
-        }
-        if (txd)
-            SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
-
-        if (txd == NULL) {
+        if (unlikely(txd == NULL)) {
+            SCLogDebug("NO TXD");
             /* make sure this tx, which can't be properly logged is skipped */
             logged = 1;
             max_id = tx_id;
             goto next_tx;
         }
 
+        if (file_logging_active) {
+            if (AppLayerParserIsFileTx(txd)) { // need to process each tx that might be a file tx,
+                                               // even if there are not files (yet)
+                const bool ts_ready = (tx_progress_ts == complete_ts);
+                const bool tc_ready = (tx_progress_tc == complete_tc);
+                SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready);
+
+                const bool eval_files = ts_ready | tc_ready | tx_complete | ts_eof | tc_eof | eof;
+
+                SCLogDebug("eval_files: %u, ts_ready %u, tc_ready %u, tx_complete %u, ts_eof %u, "
+                           "tc_eof %u, eof %u",
+                        eval_files, ts_ready, tc_ready, tx_complete, ts_eof, tc_eof, eof);
+                SCLogDebug("txd->file_tx & pkt_dir: %02x & %02x -> %02x", txd->file_tx, pkt_dir,
+                        (txd->file_tx & pkt_dir));
+
+                /* call only for the correct direction, except when it looks anything like a end of
+                 * transaction or end of stream. Since OutputTxLogFiles has complicated logic around
+                 * that, we just leave it to that function to sort things out for now. */
+                if (eval_files || AppLayerParserIsFileTxInDir(
+                                          txd, pkt_dir)) { // need to process each tx that might
+                                                           // be a file tx, even if there
+                    OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx,
+                            tx_id, txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof);
+                }
+            } else if (support_files) {
+                if (op_thread_data->file) {
+                    txd->logged.flags |= BIT_U32(LOGGER_FILE);
+                    SCLogDebug("not a file_tx: setting LOGGER_FILE => %08x", txd->logged.flags);
+                }
+                if (op_thread_data->filedata) {
+                    txd->logged.flags |= BIT_U32(LOGGER_FILEDATA);
+                    SCLogDebug("not a file_tx: setting LOGGER_FILEDATA => %08x", txd->logged.flags);
+                }
+            }
+        }
+        SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
+
         if (list[ALPROTO_UNKNOWN] != 0) {
             OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id);
             if (list[alproto] == NULL)