file: optimize file pruning

FilePrune would clear the files, but not free them and remove them
from the list. This lead to ever growing lists in some cases.
Especially in HTTP sessions with many transactions, this could slow
us down.

src/util-file.c

@@ -132,7 +132,7 @@ static int FileAppendFileData(FileContainer *ffc, FileData *ffd)
 
 
 
-static void FilePruneFile(File *file)
+static int FilePruneFile(File *file)
 {
     SCEnter();
 
@@ -141,7 +141,7 @@ static void FilePruneFile(File *file)
     if (!(file->flags & FILE_NOMAGIC)) {
         /* need magic but haven't set it yet, bail out */
         if (file->magic == NULL)
-            SCReturn;
+            SCReturnInt(0);
         else
             SCLogDebug("file->magic %s", file->magic);
     } else {
@@ -168,19 +168,36 @@ static void FilePruneFile(File *file)
 #endif
         } else if (fd->stored == 0) {
             fd = NULL;
+            SCReturnInt(0);
             break;
         }
     }
 
-    SCReturn;
+    if (file->state >= FILE_STATE_CLOSED)
+        SCReturnInt(1);
+    else
+        SCReturnInt(0);
 }
 
 void FilePrune(FileContainer *ffc)
 {
-    File *file;
+    File *file = ffc->head;
+
+    while (file) {
+        if (FilePruneFile(file) == 0)
+            break;
+
+        BUG_ON(file != ffc->head);
+
+        File *file_next = file->next;
+
+        /* update head and tail */
+        ffc->head = file_next;
+        if (file == ffc->tail)
+            ffc->tail = NULL;
 
-    for (file = ffc->head; file != NULL; file = file->next) {
-        FilePruneFile(file);
+        FileFree(file);
+        file = file_next;
     }
 }