detect/flow: optimize flow check

Flow direction doesn't need explicit checking as the rule groups (sgh)
are already per direction. So if a rule sets only flow:to_server or
flow:to_client, we can avoid adding a sigmatch to the signature.

src/detect-engine-build.c

@@ -192,6 +192,12 @@ int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
     if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
         return 0;
 
+    /* if flow dir is set we can't process it in ip-only */
+    if (!(((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) ||
+            (s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) ==
+            (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)))
+        return 0;
+
     /* for now assume that all registered buffer types are incompatible */
     const int nlists = s->init_data->smlists_array_size;
     for (int i = 0; i < nlists; i++) {

src/detect-engine-iponly.c

@@ -54,6 +54,7 @@
 #include "util-unittest-helper.h"
 #include "util-print.h"
 #include "util-profiling.h"
+#include "util-validate.h"
 
 #ifdef OS_WIN32
 #include <winsock.h>
@@ -949,7 +950,7 @@ int IPOnlyMatchCompatSMs(ThreadVars *tv,
     SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_MATCH];
     if (smd) {
         while (1) {
-            BUG_ON(!(sigmatch_table[smd->type].flags & SIGMATCH_IPONLY_COMPAT));
+            DEBUG_VALIDATE_BUG_ON(!(sigmatch_table[smd->type].flags & SIGMATCH_IPONLY_COMPAT));
             KEYWORD_PROFILING_START;
             if (sigmatch_table[smd->type].Match(tv, det_ctx, p, s, smd->ctx) > 0) {
                 KEYWORD_PROFILING_END(det_ctx, smd->type, 1);

src/detect-flow.c

@@ -376,8 +376,6 @@ int DetectFlowSetup (DetectEngineCtx *de_ctx, Signature *s, const char *flowstr)
     sm->type = DETECT_FLOW;
     sm->ctx = (SigMatchCtx *)fd;
 
-    SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_MATCH);
-
     /* set the signature direction flags */
     if (fd->flags & DETECT_FLOW_FLAG_TOSERVER) {
         s->flags |= SIG_FLAG_TOSERVER;
@@ -395,12 +393,17 @@ int DetectFlowSetup (DetectEngineCtx *de_ctx, Signature *s, const char *flowstr)
     } else if (fd->flags == DETECT_FLOW_FLAG_TOSERVER ||
                fd->flags == DETECT_FLOW_FLAG_TOCLIENT)
     {
-        // no direct flow is needed for just direction
-
+        /* no direct flow is needed for just direction,
+         * no sigmatch is needed either. */
+        SigMatchFree(sm);
+        sm = NULL;
     } else {
         s->init_data->init_flags |= SIG_FLAG_INIT_FLOW;
     }
 
+    if (sm != NULL) {
+        SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_MATCH);
+    }
     return 0;
 
 error: