file extraction: always prune files after detect

If a keyword like filemd5 was being used without a filestore,
or a file output enabled, it would be pruned before detection
had a chance to match.

Consolidate file pruning to the end of the flow worker so files
are available for detection even when a file output is not
enabled.

Redmine issue:
https://redmine.openinfosecfoundation.org/issues/2490

src/app-layer-ftp.c

@@ -1118,9 +1118,6 @@ static int FTPDataParse(Flow *f, FtpDataState *ftpdata_state,
     }
 
 out:
-    if (ftpdata_state->files) {
-        FilePrune(ftpdata_state->files);
-    }
     return ret;
 }
 

src/app-layer-htp-file.c

@@ -146,7 +146,6 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
 
     FileSetTx(files->tail, txid);
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }
@@ -301,7 +300,6 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len,
         retval = -2;
     }
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }
@@ -353,7 +351,6 @@ int HTPFileClose(HtpState *s, const uint8_t *data, uint32_t data_len,
         retval = -2;
     }
 
-    FilePrune(files);
 end:
     SCReturnInt(retval);
 }

src/app-layer-smtp.c

@@ -521,7 +521,6 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
 
     if (files != NULL) {
         SMTPPruneFiles(files);
-        FilePrune(files);
     }
 
     SCReturnInt(ret);

src/flow-worker.c

@@ -179,6 +179,18 @@ static TmEcode FlowWorkerThreadDeinit(ThreadVars *tv, void *data)
 TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq);
 TmEcode StreamTcp (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *);
 
+static void FlowPruneFiles(Packet *p)
+{
+    if (p->flow && p->flow->alstate) {
+        Flow *f = p->flow;
+        FileContainer *fc = AppLayerParserGetFiles(p->proto, f->alproto,
+            f->alstate, PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT);
+        if (fc != NULL) {
+            FilePrune(fc);
+        }
+    }
+}
+
 static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data, PacketQueue *preq, PacketQueue *unused)
 {
     FlowWorkerThreadData *fw = data;
@@ -281,6 +293,9 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data, PacketQueue *pr
     // Outputs.
     OutputLoggerLog(tv, p, fw->output_thread);
 
+    /* Prune any stored files. */
+    FlowPruneFiles(p);
+
     /*  Release tcp segments. Done here after alerting can use them. */
     if (p->flow != NULL && p->proto == IPPROTO_TCP) {
         FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);

src/output-file.c

@@ -178,11 +178,6 @@ static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data)
     OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER);
     OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT);
 
-    if (ffc_ts && (p->flowflags & FLOW_PKT_TOSERVER))
-        FilePrune(ffc_ts);
-    if (ffc_tc && (p->flowflags & FLOW_PKT_TOCLIENT))
-        FilePrune(ffc_tc);
-
     return TM_ECODE_OK;
 }
 

src/output-filedata.c

@@ -200,8 +200,6 @@ static void OutputFiledataLogFfc(ThreadVars *tv, OutputLoggerThreadStore *store,
                 }
             }
         }
-
-        FilePrune(ffc);
     }
 }