detect: change mask logic

Previously the MPM/Prefilter engines would suggest the same rule
candidates multiple times.

For example, while processing the request body, the http headers
would be inspected by MPM multiple times.

The mask check was one way to quickly decide which rules could be
skipped.

Now that the MPM engines normally return a rule just once, this
mask check no longer makes sense. If the rule meets the ip/port/
direction based conditions, it needs to be evaluated if the MPM
said so. Even if not all conditions are yet true.

WIP disable mask as it no longer makes sense

WIP redo mask match

src/detect.c

@@ -1161,8 +1161,25 @@ void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineT
 
         SCLogDebug("inspecting signature id %"PRIu32"", s->id);
 
-        if ((s->mask & mask) != s->mask)
-            goto next;
+        if (sflags & SIG_FLAG_STATE_MATCH) {
+            if (det_ctx->de_state_sig_array[s->num] & DE_STATE_MATCH_NO_NEW_STATE)
+                goto next;
+        } else {
+            /* don't run mask check for stateful rules.
+             * There we depend on prefilter */
+            if ((s->mask & mask) != s->mask) {
+                SCLogDebug("mask mismatch %x & %x != %x", s->mask, mask, s->mask);
+                goto next;
+            }
+
+            if (unlikely(sflags & SIG_FLAG_DSIZE)) {
+                if (likely(p->payload_len < s->dsize_low || p->payload_len > s->dsize_high)) {
+                    SCLogDebug("kicked out as p->payload_len %u, dsize low %u, hi %u",
+                            p->payload_len, s->dsize_low, s->dsize_high);
+                    goto next;
+                }
+            }
+        }
 
         /* if the sig has alproto and the session as well they should match */
         if (likely(sflags & SIG_FLAG_APPLAYER)) {
@@ -1179,19 +1196,6 @@ void SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineT
             }
         }
 
-        if (unlikely(sflags & SIG_FLAG_DSIZE)) {
-            if (likely(p->payload_len < s->dsize_low || p->payload_len > s->dsize_high)) {
-                SCLogDebug("kicked out as p->payload_len %u, dsize low %u, hi %u",
-                           p->payload_len, s->dsize_low, s->dsize_high);
-                goto next;
-            }
-        }
-
-        if (sflags & SIG_FLAG_STATE_MATCH) {
-            if (det_ctx->de_state_sig_array[s->num] & DE_STATE_MATCH_NO_NEW_STATE)
-                goto next;
-        }
-
         /* check if this signature has a requirement for flowvars of some type
          * and if so, if we actually have any in the flow. If not, the sig
          * can't match and we skip it. */