detect/files: fix file sigs state handling

Make sure all file sig mismatches indicate this in their return
code, not just the ones with filestore enabled. This is needed
to tell the stateful detect engine that it is dealing with a file
sig, so it can make sure these are inspected correctly even if
there are possibly multiple files per tx.

src/detect-engine-file.c

@@ -147,7 +147,7 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
                         FileMatch(tv, det_ctx, f, flags, file, s, smd->ctx);
                     KEYWORD_PROFILING_END(det_ctx, smd->type, (match > 0));
                     if (match == 0) {
-                        r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
+                        r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
                         break;
                     } else if (smd->is_last) {
                         r = DETECT_ENGINE_INSPECT_SIG_MATCH;
@@ -165,11 +165,6 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
             if (r == DETECT_ENGINE_INSPECT_SIG_MATCH)
                 store_r = DETECT_ENGINE_INSPECT_SIG_MATCH;
 
-            /* if this is a filestore sig, and the sig can't match
-             * return 3 so we can distinguish */
-            if ((s->flags & SIG_FLAG_FILESTORE) && r == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH)
-                r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE;
-
             /* continue, this file may (or may not) be unable to match
              * maybe we have more that can :) */
         }
@@ -244,9 +239,9 @@ int DetectFileInspectGeneric(ThreadVars *tv,
     } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
         SCLogDebug("sid %u can't match on this transaction", s->id);
         r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
-    } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE) {
-        SCLogDebug("sid %u can't match on this transaction (filestore sig)", s->id);
-        r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE;
+    } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES) {
+        SCLogDebug("sid %u can't match on this transaction (file sig)", s->id);
+        r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
     } else if (match == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES) {
         SCLogDebug("match with more files ahead");
         r = match;

src/detect-engine-state.h

@@ -38,7 +38,10 @@
 #define DETECT_ENGINE_INSPECT_SIG_NO_MATCH 0
 #define DETECT_ENGINE_INSPECT_SIG_MATCH 1
 #define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH 2
-#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE 3
+/** indicate that the file inspection portion of a sig didn't match.
+ *  This is used to handle state keeping as the detect engine is still
+ *  only marginally aware of files. */
+#define DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES 3
 /** hack to work around a file inspection limitation. Since there can be
  *  multiple files in a TX and the detection engine really don't know
  *  about that, we have to give the file inspection engine a way to

src/detect-filename.c

@@ -375,8 +375,6 @@ static int DetectEngineInspectFilename(
         const Signature *s,
         Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
 {
-    int r = 0;
-
     const DetectEngineTransforms *transforms = NULL;
     if (!engine->mpm) {
         transforms = engine->v2.transforms;
@@ -388,9 +386,9 @@ static int DetectEngineInspectFilename(
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     }
 
+    int r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     int local_file_id = 0;
-    File *file = ffc->head;
-    for (; file != NULL; file = file->next) {
+    for (File *file = ffc->head; file != NULL; file = file->next) {
         if (file->txid != tx_id)
             continue;
 
@@ -409,16 +407,13 @@ static int DetectEngineInspectFilename(
                                               buffer->inspect_offset, DETECT_CI_FLAGS_SINGLE,
                                               DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL);
         if (match == 1) {
-            r = 1;
-            break;
+            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        } else {
+            r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
         }
         local_file_id++;
     }
-
-    if (r == 1)
-        return DETECT_ENGINE_INSPECT_SIG_MATCH;
-    else
-        return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE;
+    return r;
 }
 
 typedef struct PrefilterMpmFilename {

src/detect.c

@@ -1231,7 +1231,7 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
             } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
                 inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
                 inspect_flags |= BIT_U32(engine->id);
-            } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE) {
+            } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES) {
                 inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
                 inspect_flags |= BIT_U32(engine->id);
                 file_no_match = 1;