filestore: track files getting stored per tx

Avoid evicting a tx before the filedata logger has decided it is
done.

rust/src/applayer.rs

@@ -62,6 +62,7 @@ pub struct AppLayerTxData {
     /// track file open/logs so we can know how long to keep the tx
     pub files_opened: u32,
     pub files_logged: u32,
+    pub files_stored: u32,
 
     /// detection engine flags for use by detection engine
     detect_flags_ts: u64,
@@ -75,6 +76,7 @@ impl AppLayerTxData {
             logged: LoggerFlags::new(),
             files_opened: 0,
             files_logged: 0,
+            files_stored: 0,
             detect_flags_ts: 0,
             detect_flags_tc: 0,
         }

src/app-layer-parser.c

@@ -877,6 +877,7 @@ FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
 
 extern int g_detect_disabled;
 extern bool g_file_logger_enabled;
+extern bool g_filedata_logger_enabled;
 
 /**
  * \brief remove obsolete (inspected and logged) transactions
@@ -998,8 +999,12 @@ void AppLayerParserTransactionsCleanup(Flow *f)
 
         /* if file logging is enabled, we keep a tx active while some of the files aren't
          * logged yet. */
-        if (txd && txd->files_opened && g_file_logger_enabled) {
-            if (txd->files_opened != txd->files_logged) {
+        if (txd && txd->files_opened) {
+            if (g_file_logger_enabled && txd->files_opened != txd->files_logged) {
+                skipped = true;
+                goto next;
+            }
+            if (g_filedata_logger_enabled && txd->files_opened != txd->files_stored) {
                 skipped = true;
                 goto next;
             }

src/output-filedata.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2014 Open Information Security Foundation
+/* Copyright (C) 2007-2021 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -35,6 +35,8 @@
 #include "util-validate.h"
 #include "util-magic.h"
 
+bool g_filedata_logger_enabled = false;
+
 typedef struct OutputLoggerThreadStore_ {
     void *thread_data;
     struct OutputLoggerThreadStore_ *next;
@@ -97,6 +99,7 @@ int OutputRegisterFiledataLogger(LoggerId id, const char *name,
     }
 
     SCLogDebug("OutputRegisterFiledataLogger happy");
+    g_filedata_logger_enabled = true;
     return 0;
 }
 
@@ -129,6 +132,17 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
     return file_logged;
 }
 
+static void CloseFile(const Packet *p, Flow *f, File *file)
+{
+    void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, file->txid);
+    if (txv) {
+        AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv);
+        if (txd)
+            txd->files_stored++;
+    }
+    file->flags |= FILE_STORED;
+}
+
 static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadData *td,
         Packet *p, FileContainer *ffc, const uint8_t call_flags,
         const bool file_close, const bool file_trunc, const uint8_t dir)
@@ -162,7 +176,7 @@ static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadData *td,
                     FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
                 }
                 CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
-                ff->flags |= FILE_STORED;
+                CloseFile(p, p->flow, ff);
                 continue;
             }
 
@@ -201,7 +215,7 @@ static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadData *td,
 
                 /* all done */
                 if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
-                    ff->flags |= FILE_STORED;
+                    CloseFile(p, p->flow, ff);
                 }
             }
         }