detect: shrink inspect engine by using 'id' as state flag

src/detect-engine-state.c

@@ -538,7 +538,7 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
                                              tx, tx_id);
                     SCLogDebug("engine %p match %d", engine, match);
                     if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) {
-                        inspect_flags |= engine->inspect_flags;
+                        inspect_flags |= BIT_U32(engine->id);
                         engine = engine->next;
                         total_matches++;
                         continue;
@@ -551,10 +551,10 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
                         continue;
                     } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
                         inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
-                        inspect_flags |= engine->inspect_flags;
+                        inspect_flags |= BIT_U32(engine->id);;
                     } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE) {
                         inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
-                        inspect_flags |= engine->inspect_flags;
+                        inspect_flags |= BIT_U32(engine->id);
                         file_no_match++;
                     }
                     break;
@@ -854,7 +854,7 @@ static int DoInspectItem(ThreadVars *tv,
     }
 
     while (engine != NULL) {
-        if (!(item->flags & engine->inspect_flags) &&
+        if (!(item->flags & BIT_U32(engine->id)) &&
                 direction == engine->dir)
         {
             SCLogDebug("inspect_flags %x", inspect_flags);
@@ -862,7 +862,7 @@ static int DoInspectItem(ThreadVars *tv,
             int match = engine->Callback(tv, de_ctx, det_ctx, s, f,
                     flags, alstate, inspect_tx, inspect_tx_id);
             if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) {
-                inspect_flags |= engine->inspect_flags;
+                inspect_flags |= BIT_U32(engine->id);
                 engine = engine->next;
                 total_matches++;
                 continue;
@@ -875,10 +875,10 @@ static int DoInspectItem(ThreadVars *tv,
                 continue;
             } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) {
                 inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
-                inspect_flags |= engine->inspect_flags;
+                inspect_flags |= BIT_U32(engine->id);
             } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE) {
                 inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH;
-                inspect_flags |= engine->inspect_flags;
+                inspect_flags |= BIT_U32(engine->id);
                 (*file_no_match)++;
             }
             break;

src/detect-engine.c

@@ -178,7 +178,7 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
 
         if (s->app_inspect == NULL) {
             s->app_inspect = new_engine;
-            new_engine->inspect_flags = BIT_U32(DE_STATE_FLAG_BASE); // start beyond the built-ins
+            new_engine->id = DE_STATE_FLAG_BASE; /* id is used as flag in stateful detect */
         } else {
             DetectEngineAppInspectionEngine *a = s->app_inspect;
             while (a->next != NULL) {
@@ -187,7 +187,6 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
 
             a->next = new_engine;
             new_engine->id = a->id + 1;
-            new_engine->inspect_flags = BIT_U32(new_engine->id + DE_STATE_FLAG_BASE);
         }
         SCLogDebug("sid %u: engine %p/%u added", s->id, new_engine, new_engine->id);
 

src/detect.h

@@ -378,9 +378,8 @@ struct DetectEngineThreadCtx_;// DetectEngineThreadCtx;
 typedef struct DetectEngineAppInspectionEngine_ {
     AppProto alproto;
     uint8_t dir;
-    uint8_t id;
+    uint8_t id;     /**< per sig id used in state keeping */
     int sm_list;
-    uint32_t inspect_flags;
 
     /* \retval 0 No match.  Don't discontinue matching yet.  We need more data.
      *         1 Match.