profiling: support prefilter engines

src/decode.h

@@ -347,6 +347,15 @@ typedef struct PktProfilingLoggerData_ {
     uint64_t ticks_spent;
 } PktProfilingLoggerData;
 
+typedef struct PktProfilingPrefilterEngine_ {
+    uint64_t ticks_spent;
+} PktProfilingPrefilterEngine;
+
+typedef struct PktProfilingPrefilterData_ {
+    PktProfilingPrefilterEngine *engines;
+    uint32_t size;          /**< array size */
+} PktProfilingPrefilterData;
+
 /** \brief Per pkt stats storage */
 typedef struct PktProfiling_ {
     uint64_t ticks_start;
@@ -357,6 +366,7 @@ typedef struct PktProfiling_ {
     PktProfilingAppData app[ALPROTO_MAX];
     PktProfilingDetectData detect[PROF_DETECT_SIZE];
     PktProfilingLoggerData logger[LOGGER_SIZE];
+    PktProfilingPrefilterData prefilter;
     uint64_t proto_detect;
 } PktProfiling;
 

src/detect-engine-dns.c

@@ -135,7 +135,7 @@ int PrefilterTxDnsQueryRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxDnsQuery,
         ALPROTO_DNS, 1,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "dns_query");
 }
 
 int DetectEngineInspectDnsRequest(ThreadVars *tv,

src/detect-engine-filedata-smtp.c

@@ -281,7 +281,7 @@ int PrefilterTxSmtpFiledataRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxSmtpFiledata,
         ALPROTO_SMTP, 0,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "file_data (smtp)");
 }
 
 #ifdef UNITTESTS

src/detect-engine-hcbd.c

@@ -253,7 +253,7 @@ int PrefilterTxHttpRequestBodyRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxHttpRequestBody,
         ALPROTO_HTTP, HTP_REQUEST_BODY,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_client_body");
 }
 
 int DetectEngineInspectHttpClientBody(ThreadVars *tv,

src/detect-engine-hcd.c

@@ -103,7 +103,7 @@ int PrefilterTxRequestCookieRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxRequestCookie,
         ALPROTO_HTTP, HTP_REQUEST_HEADERS,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_cookie (request)");
 }
 
 /** \brief HTTP Cookie Mpm prefilter callback
@@ -149,7 +149,7 @@ int PrefilterTxResponseCookieRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxResponseCookie,
         ALPROTO_HTTP, HTP_RESPONSE_HEADERS,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_cookie (response)");
 }
 
 /**

src/detect-engine-hhd.c

@@ -256,12 +256,12 @@ int PrefilterTxHttpRequestHeadersRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     int r = PrefilterAppendTxEngine(sgh, PrefilterTxHttpRequestHeaders,
         ALPROTO_HTTP, HTP_REQUEST_HEADERS,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_header (request)");
     if (r != 0)
         return r;
     return PrefilterAppendTxEngine(sgh, PrefilterTxHttpRequestHeaders,
         ALPROTO_HTTP, HTP_REQUEST_TRAILER,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_header (request)");
 }
 
 /** \brief HTTP Headers Mpm prefilter callback
@@ -305,12 +305,12 @@ int PrefilterTxHttpResponseHeadersRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     int r = PrefilterAppendTxEngine(sgh, PrefilterTxHttpResponseHeaders,
         ALPROTO_HTTP, HTP_RESPONSE_HEADERS,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_header (response)");
     if (r != 0)
         return r;
     return PrefilterAppendTxEngine(sgh, PrefilterTxHttpResponseHeaders,
         ALPROTO_HTTP, HTP_RESPONSE_TRAILER,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_header (response)");
 }
 
 int DetectEngineInspectHttpHeader(ThreadVars *tv,

src/detect-engine-hhhd.c

@@ -98,7 +98,7 @@ int PrefilterTxHostnameRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxHostname,
         ALPROTO_HTTP, HTP_REQUEST_HEADERS,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_host");
 }
 
 /**

src/detect-engine-hmd.c

@@ -95,7 +95,7 @@ int PrefilterTxMethodRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 {
     return PrefilterAppendTxEngine(sgh, PrefilterTxMethod,
         ALPROTO_HTTP, HTP_REQUEST_LINE,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_method");
 }
 
 /**

src/detect-engine-hrhd.c

@@ -97,12 +97,12 @@ int PrefilterTxRequestHeadersRawRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     int r = PrefilterAppendTxEngine(sgh, PrefilterTxRequestHeadersRaw,
         ALPROTO_HTTP, HTP_REQUEST_HEADERS+1, /* inspect when headers complete */
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_raw_header (request)");
     if (r != 0)
         return r;
     return PrefilterAppendTxEngine(sgh, PrefilterTxRequestHeadersRaw,
         ALPROTO_HTTP, HTP_REQUEST_TRAILER+1, /* inspect when trailer complete */
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_raw_header (request)");
 }
 
 /** \brief HTTP Raw Header Mpm prefilter callback
@@ -141,12 +141,12 @@ int PrefilterTxResponseHeadersRawRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     int r = PrefilterAppendTxEngine(sgh, PrefilterTxResponseHeadersRaw,
         ALPROTO_HTTP, HTP_RESPONSE_HEADERS+1, /* inspect when headers complete */
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_raw_header (response)");
     if (r != 0)
         return r;
     return PrefilterAppendTxEngine(sgh, PrefilterTxResponseHeadersRaw,
         ALPROTO_HTTP, HTP_RESPONSE_TRAILER+1, /* inspect when trailer complete */
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_raw_header (response)");
 }
 
 /**

src/detect-engine-hrhhd.c

@@ -110,7 +110,7 @@ int PrefilterTxHostnameRawRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxHostnameRaw,
         ALPROTO_HTTP, HTP_REQUEST_HEADERS,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_raw_host");
 }
 
 /**

src/detect-engine-hrud.c

@@ -95,7 +95,7 @@ int PrefilterTxRawUriRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxRawUri,
         ALPROTO_HTTP, HTP_REQUEST_LINE,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_raw_uri");
 }
 
 /**

src/detect-engine-hsbd.c

@@ -257,7 +257,7 @@ int PrefilterTxHttpResponseBodyRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxHttpResponseBody,
         ALPROTO_HTTP, HTP_RESPONSE_BODY,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "file_data (http response)");
 }
 
 

src/detect-engine-hscd.c

@@ -94,7 +94,7 @@ int PrefilterTxHttpStatCodeRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
     return PrefilterAppendTxEngine(sgh, PrefilterTxHttpStatCode,
         ALPROTO_HTTP,
         HTP_RESPONSE_LINE+1, /* inspect when response line completely parsed */
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_stat_code");
 }
 
 /**

src/detect-engine-hsmd.c

@@ -94,7 +94,7 @@ int PrefilterTxHttpStatMsgRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
     return PrefilterAppendTxEngine(sgh, PrefilterTxHttpStatMsg,
         ALPROTO_HTTP,
         HTP_RESPONSE_LINE+1, /* inspect when response line completely parsed */
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_stat_msg");
 }
 
 /**

src/detect-engine-hua.c

@@ -103,7 +103,7 @@ int PrefilterTxUARegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxUA,
         ALPROTO_HTTP, HTP_REQUEST_HEADERS,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_user_agent");
 }
 
 /**

src/detect-engine-payload.c

@@ -83,7 +83,7 @@ static void PrefilterPktStream(DetectEngineThreadCtx *det_ctx,
 
 int PrefilterPktStreamRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 {
-    return PrefilterAppendEngine(sgh, PrefilterPktStream, mpm_ctx, NULL);
+    return PrefilterAppendEngine(sgh, PrefilterPktStream, mpm_ctx, NULL, "stream");
 }
 
 static void PrefilterPktPayload(DetectEngineThreadCtx *det_ctx,
@@ -112,7 +112,7 @@ static void PrefilterPktPayload(DetectEngineThreadCtx *det_ctx,
 
 int PrefilterPktPayloadRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 {
-    return PrefilterAppendEngine(sgh, PrefilterPktPayload, mpm_ctx, NULL);
+    return PrefilterAppendEngine(sgh, PrefilterPktPayload, mpm_ctx, NULL, "payload");
 }
 
 

src/detect-engine-prefilter-common.c

@@ -107,7 +107,8 @@ SetupEngineForPacketHeader(SigGroupHead *sgh, int sm_type,
         }
     }
 
-    PrefilterAppendEngine(sgh, Match, ctx, PrefilterPacketHeaderFree);
+    PrefilterAppendEngine(sgh, Match, ctx, PrefilterPacketHeaderFree,
+            sigmatch_table[sm_type].name);
     return 0;
 }
 
@@ -205,7 +206,8 @@ SetupEngineForPacketHeaderPrefilterPacketU8HashCtx(SigGroupHead *sgh, int sm_typ
 
     if (cnt) {
         PrefilterAppendEngine(sgh, Match, ctx,
-                PrefilterPacketU8HashCtxFree);
+                PrefilterPacketU8HashCtxFree,
+                sigmatch_table[sm_type].name);
     } else {
         PrefilterPacketU8HashCtxFree(ctx);
     }

src/detect-engine-prefilter.c

@@ -52,6 +52,12 @@
 #include "app-layer-parser.h"
 #include "app-layer-htp.h"
 
+#include "util-profiling.h"
+
+#ifdef PROFILING
+static int PrefilterStoreGetId(const char *name);
+#endif
+
 static inline void PrefilterTx(DetectEngineThreadCtx *det_ctx,
         const SigGroupHead *sgh, Packet *p, const uint8_t flags)
 {
@@ -92,8 +98,10 @@ static inline void PrefilterTx(DetectEngineThreadCtx *det_ctx,
             if (engine->tx_min_progress > tx_progress)
                 goto next;
 
+            PROFILING_PREFILTER_START(p);
             engine->PrefilterTx(det_ctx, engine->pectx,
                     p, p->flow, tx, idx, flags);
+            PROFILING_PREFILTER_END(p, engine->profile_id);
         next:
             engine = engine->next;
         } while (engine);
@@ -105,10 +113,15 @@ void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh,
 {
     SCEnter();
 
+    PROFILING_PREFILTER_RESET(p, det_ctx->de_ctx->profile_prefilter_maxid);
+
     /* run packet engines */
     PrefilterEngine *engine = sgh->engines;
     while (engine) {
+        PROFILING_PREFILTER_START(p);
         engine->Prefilter(det_ctx, p, engine->pectx);
+        PROFILING_PREFILTER_END(p, engine->profile_id);
+
         engine = engine->next;
     }
 
@@ -124,7 +137,8 @@ void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh,
 
 int PrefilterAppendEngine(SigGroupHead *sgh,
         void (*Prefilter)(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx),
-        void *pectx, void (*FreeFunc)(void *pectx))
+        void *pectx, void (*FreeFunc)(void *pectx),
+        const char *name)
 {
     if (sgh == NULL || Prefilter == NULL || pectx == NULL)
         return -1;
@@ -139,16 +153,21 @@ int PrefilterAppendEngine(SigGroupHead *sgh,
 
     if (sgh->engines == NULL) {
         sgh->engines = e;
-        return 0;
-    }
+    } else {
+        PrefilterEngine *t = sgh->engines;
+        while (t->next != NULL) {
+            t = t->next;
+        }
 
-    PrefilterEngine *t = sgh->engines;
-    while (t->next != NULL) {
-        t = t->next;
+        t->next = e;
+        e->id = t->id + 1;
     }
 
-    t->next = e;
-    e->id = t->id + 1;
+#ifdef PROFILING
+    sgh->engines_cnt = e->id;
+    e->name = name;
+    e->profile_id = PrefilterStoreGetId(e->name);
+#endif
     return 0;
 }
 
@@ -157,7 +176,8 @@ int PrefilterAppendTxEngine(SigGroupHead *sgh,
             Packet *p, Flow *f, void *tx,
             const uint64_t idx, const uint8_t flags),
         AppProto alproto, int tx_min_progress,
-        void *pectx, void (*FreeFunc)(void *pectx))
+        void *pectx, void (*FreeFunc)(void *pectx),
+        const char *name)
 {
     if (sgh == NULL || PrefilterTx == NULL || pectx == NULL)
         return -1;
@@ -174,16 +194,20 @@ int PrefilterAppendTxEngine(SigGroupHead *sgh,
 
     if (sgh->tx_engines == NULL) {
         sgh->tx_engines = e;
-        return 0;
-    }
+    } else {
+        PrefilterEngine *t = sgh->tx_engines;
+        while (t->next != NULL) {
+            t = t->next;
+        }
 
-    PrefilterEngine *t = sgh->tx_engines;
-    while (t->next != NULL) {
-        t = t->next;
+        t->next = e;
+        e->id = t->id + 1;
     }
-
-    t->next = e;
-    e->id = t->id + 1;
+#ifdef PROFILING
+    sgh->tx_engines_cnt = e->id;
+    e->name = name;
+    e->profile_id = PrefilterStoreGetId(e->name);
+#endif
     return 0;
 }
 
@@ -205,3 +229,124 @@ void PrefilterFreeEngines(PrefilterEngine *list)
         t = next;
     }
 }
+
+#ifdef PROFILING
+/* hash table for assigning a unique id to each engine type. */
+
+typedef struct PrefilterStore_ {
+    const char *name;
+    uint32_t id;
+} PrefilterStore;
+
+static uint32_t PrefilterStoreHashFunc(HashListTable *ht, void *data, uint16_t datalen)
+{
+    PrefilterStore *ctx = data;
+
+    uint32_t hash = strlen(ctx->name);
+    uint16_t u;
+
+    for (u = 0; u < strlen(ctx->name); u++) {
+        hash += ctx->name[u];
+    }
+
+    hash %= ht->array_size;
+    return hash;
+}
+
+static char PrefilterStoreCompareFunc(void *data1, uint16_t len1,
+                                      void *data2, uint16_t len2)
+{
+    PrefilterStore *ctx1 = data1;
+    PrefilterStore *ctx2 = data2;
+    return (strcmp(ctx1->name, ctx2->name) == 0);
+}
+
+static void PrefilterStoreFreeFunc(void *ptr)
+{
+    SCFree(ptr);
+}
+
+static SCMutex g_prefilter_mutex = SCMUTEX_INITIALIZER;
+static uint32_t g_prefilter_id = 0;
+static HashListTable *g_prefilter_hash_table = NULL;
+
+static void PrefilterDeinit(void)
+{
+    SCMutexLock(&g_prefilter_mutex);
+    BUG_ON(g_prefilter_hash_table == NULL);
+    HashListTableFree(g_prefilter_hash_table);
+    SCMutexUnlock(&g_prefilter_mutex);
+}
+
+static void PrefilterInit(void)
+{
+    SCMutexLock(&g_prefilter_mutex);
+    BUG_ON(g_prefilter_hash_table != NULL);
+
+    g_prefilter_hash_table = HashListTableInit(256,
+            PrefilterStoreHashFunc,
+            PrefilterStoreCompareFunc,
+            PrefilterStoreFreeFunc);
+    BUG_ON(g_prefilter_hash_table == NULL);
+    atexit(PrefilterDeinit);
+    SCMutexUnlock(&g_prefilter_mutex);
+}
+
+static int PrefilterStoreGetId(const char *name)
+{
+    PrefilterStore ctx = { name, 0 };
+
+    if (g_prefilter_hash_table == NULL) {
+        PrefilterInit();
+    }
+
+    SCLogDebug("looking up %s", name);
+
+    SCMutexLock(&g_prefilter_mutex);
+    PrefilterStore *rctx = HashListTableLookup(g_prefilter_hash_table, (void *)&ctx, 0);
+    if (rctx != NULL) {
+        SCMutexUnlock(&g_prefilter_mutex);
+        return rctx->id;
+    }
+
+    PrefilterStore *actx = SCCalloc(1, sizeof(*actx));
+    if (actx == NULL) {
+        SCMutexUnlock(&g_prefilter_mutex);
+        return -1;
+    }
+
+    actx->name = name;
+    actx->id = g_prefilter_id++;
+    SCLogDebug("prefilter engine %s has profile id %u", actx->name, actx->id);
+
+    int ret = HashListTableAdd(g_prefilter_hash_table, actx, 0);
+    if (ret != 0) {
+        SCMutexUnlock(&g_prefilter_mutex);
+        SCFree(actx);
+        return -1;
+    }
+
+    int r = actx->id;
+    SCMutexUnlock(&g_prefilter_mutex);
+    return r;
+}
+
+/** \warning slow */
+const char *PrefilterStoreGetName(const uint32_t id)
+{
+    const char *name = NULL;
+    SCMutexLock(&g_prefilter_mutex);
+    if (g_prefilter_hash_table != NULL) {
+        HashListTableBucket *hb = HashListTableGetListHead(g_prefilter_hash_table);
+        for ( ; hb != NULL; hb = HashListTableGetListNext(hb)) {
+            PrefilterStore *ctx = HashListTableGetListData(hb);
+            if (ctx->id == id) {
+                name = ctx->name;
+                break;
+            }
+        }
+    }
+    SCMutexUnlock(&g_prefilter_mutex);
+    return name;
+}
+#endif /* PROFILING */

src/detect-engine-prefilter.h

@@ -29,14 +29,20 @@ void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p,
 
 int PrefilterAppendEngine(SigGroupHead *sgh,
         void (*Prefilter)(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx),
-        void *pectx, void (*FreeFunc)(void *pectx));
+        void *pectx, void (*FreeFunc)(void *pectx),
+        const char *name);
 int PrefilterAppendTxEngine(SigGroupHead *sgh,
         void (*PrefilterTx)(DetectEngineThreadCtx *det_ctx, const void *pectx,
             Packet *p, Flow *f, void *tx,
             const uint64_t idx, const uint8_t flags),
         const AppProto alproto, const int tx_min_progress,
-        void *pectx, void (*FreeFunc)(void *pectx));
+        void *pectx, void (*FreeFunc)(void *pectx),
+        const char *name);
 
 void PrefilterFreeEngines(PrefilterEngine *list);
 
+#ifdef PROFILING
+const char *PrefilterStoreGetName(const uint32_t id);
+#endif
+
 #endif

src/detect-engine-tls.c

@@ -81,7 +81,7 @@ int PrefilterTxTlsSniRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxTlsSni,
         ALPROTO_TLS, 0, // TODO a special 'cert ready' state might be good to add
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "tls_sni");
 }
 
 /** \brief Do the content inspection and validation for a signature
@@ -157,7 +157,7 @@ int PrefilterTxTlsIssuerRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxTlsIssuer,
         ALPROTO_TLS, 0, // TODO a special 'cert ready' state might be good to add
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "tls_cert_issuer");
 }
 
 /** \brief Do the content inspection and validation for a signature
@@ -233,7 +233,7 @@ int PrefilterTxTlsSubjectRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxTlsSubject,
         ALPROTO_TLS, 0, // TODO a special 'cert ready' state might be good to add
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "tls_cert_subject");
 }
 
 /** \brief Do the content inspection and validation for a signature

src/detect-engine-uri.c

@@ -86,7 +86,7 @@ int PrefilterTxUriRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx)
 
     return PrefilterAppendTxEngine(sgh, PrefilterTxUri,
         ALPROTO_HTTP, HTP_REQUEST_LINE,
-        mpm_ctx, NULL);
+        mpm_ctx, NULL, "http_uri");
 }
 
 /**

src/detect.c

@@ -853,41 +853,6 @@ static void QuickSortSigIntId(SigIntId *sids, uint32_t n)
 #define SMS_USE_FLOW_SGH        0x01
 #define SMS_USED_PM             0x02
 
-/**
- * \internal
- * \brief Run mpm on packet, stream and other buffers based on
- *        alproto, sgh state.
- *
- * \param de_ctx       Pointer to the detection engine context.
- * \param det_ctx      Pointer to the detection engine thread context.
- * \param smsg         The stream segment to inspect for stream mpm.
- * \param p            Packet.
- * \param flags        Flags.
- * \param alproto      Flow alproto.
- * \param has_state    Bool indicating we have (al)state
- * \param sms_runflags Used to store state by detection engine.
- */
-static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx,
-        DetectEngineThreadCtx *det_ctx, StreamMsg *smsg, Packet *p,
-        const uint8_t flags, const AppProto alproto,
-        const int has_state, uint8_t *sms_runflags)
-{
-    SCEnter();
-
-    /* have a look at the reassembled stream (if any) */
-    if (p->flowflags & FLOW_PKT_ESTABLISHED) {
-        SCLogDebug("p->flowflags & FLOW_PKT_ESTABLISHED");
-    } else {
-        SCLogDebug("NOT p->flowflags & FLOW_PKT_ESTABLISHED");
-    }
-
-    /* Sort the rule list to lets look at pmq.
-     * NOTE due to merging of 'stream' pmqs we *MAY* have duplicate entries */
-    if (det_ctx->pmq.rule_id_array_cnt > 1) {
-        QuickSortSigIntId(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt);
-    }
-}
-
 #ifdef DEBUG
 static void DebugInspectIds(Packet *p, Flow *f, StreamMsg *smsg)
 {
@@ -1229,14 +1194,16 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     }
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_NONMPMLIST);
 
+    PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PREFILTER);
     /* run the prefilter engines */
     Prefilter(det_ctx, det_ctx->sgh, p, flow_flags, has_state);
-
-    /* run the mpm for each type */
-    PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM);
-    DetectMpmPrefilter(de_ctx, det_ctx, det_ctx->smsg, p, flow_flags,
-            alproto, has_state, &sms_runflags);
-    PACKET_PROFILING_DETECT_END(p, PROF_DETECT_MPM);
+    /* Sort the rule list to lets look at pmq.
+     * NOTE due to merging of 'stream' pmqs we *MAY* have duplicate entries */
+    if (det_ctx->pmq.rule_id_array_cnt > 1) {
+        QuickSortSigIntId(det_ctx->pmq.rule_id_array, det_ctx->pmq.rule_id_array_cnt);
+    }
+    DetectPrefilterMergeSort(de_ctx, det_ctx);
+    PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PREFILTER);
 
 #ifdef PROFILING
     if (th_v) {
@@ -1250,9 +1217,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
     }
 #endif
 
-    PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PREFILTER);
-    DetectPrefilterMergeSort(de_ctx, det_ctx);
-    PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PREFILTER);
 
     PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES);
     /* inspect the sigs against the packet */
@@ -3840,6 +3804,22 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
             }
         }
 
+#ifdef PROFILING
+        PrefilterEngine *e;
+        uint32_t engines = 0;
+        uint32_t tx_engines = 0;
+
+        for (e = sgh->engines ; e != NULL; e = e->next) {
+            engines++;
+            de_ctx->profile_prefilter_maxid = MAX(de_ctx->profile_prefilter_maxid, e->profile_id);
+        }
+        for (e = sgh->tx_engines ; e != NULL; e = e->next) {
+            tx_engines++;
+            de_ctx->profile_prefilter_maxid = MAX(de_ctx->profile_prefilter_maxid, e->profile_id);
+        }
+        SCLogDebug("SGH %p: engines %u tx_engines %u", sgh, engines, tx_engines);
+#endif
+
         SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh);
 
         sgh->id = idx;

src/detect.h

@@ -669,6 +669,7 @@ typedef struct DetectEngineCtx_ {
     struct SCProfileKeywordDetectCtx_ *profile_keyword_ctx_per_list[DETECT_SM_LIST_MAX];
     struct SCProfileSghDetectCtx_ *profile_sgh_ctx;
     uint32_t profile_match_logging_threshold;
+    uint32_t profile_prefilter_maxid;
 #endif
 
     char config_prefix[64];
@@ -983,6 +984,10 @@ typedef struct PrefilterEngine_ {
     /** Free function for pectx data. If NULL the memory is not freed. */
     void (*Free)(void *pectx);
 
+    const char *name;
+#ifdef PROFILING
+    uint32_t profile_id;
+#endif
 } PrefilterEngine;
 
 typedef struct SigGroupHeadInitData_ {
@@ -1023,6 +1028,11 @@ typedef struct SigGroupHead_ {
     PrefilterEngine *engines;
     PrefilterEngine *tx_engines;
 
+#ifdef PROFILING
+    uint32_t engines_cnt;
+    uint32_t tx_engines_cnt;
+#endif
+
     /** Array with sig ptrs... size is sig_cnt * sizeof(Signature *) */
     Signature **match_array;
 

src/suricata-common.h

@@ -310,27 +310,6 @@
 #define BIT_U64(n) (1ULL << (n))
 
 typedef enum PacketProfileDetectId_ {
-    PROF_DETECT_MPM,
-    PROF_DETECT_MPM_PACKET,         /* PKT MPM */
-    PROF_DETECT_MPM_PKT_STREAM,     /* PKT inspected with stream MPM */
-    PROF_DETECT_MPM_STREAM,         /* STREAM MPM */
-    PROF_DETECT_MPM_URI,
-    PROF_DETECT_MPM_HCBD,
-    PROF_DETECT_MPM_HSBD,
-    PROF_DETECT_MPM_HHD,
-    PROF_DETECT_MPM_HRHD,
-    PROF_DETECT_MPM_HMD,
-    PROF_DETECT_MPM_HCD,
-    PROF_DETECT_MPM_HRUD,
-    PROF_DETECT_MPM_HSMD,
-    PROF_DETECT_MPM_HSCD,
-    PROF_DETECT_MPM_HUAD,
-    PROF_DETECT_MPM_HHHD,
-    PROF_DETECT_MPM_HRHHD,
-    PROF_DETECT_MPM_DNSQUERY,
-    PROF_DETECT_MPM_TLSSNI,
-    PROF_DETECT_MPM_TLSISSUER,
-    PROF_DETECT_MPM_TLSSUBJECT,
     PROF_DETECT_IPONLY,
     PROF_DETECT_RULES,
     PROF_DETECT_STATEFUL,

src/util-profiling.c

@@ -29,6 +29,7 @@
 #include "suricata-common.h"
 #include "decode.h"
 #include "detect.h"
+#include "detect-engine-prefilter.h"
 #include "conf.h"
 #include "flow-worker.h"
 
@@ -94,6 +95,9 @@ struct ProfileProtoRecords {
     SCProfilePacketData records4[257];
     SCProfilePacketData records6[257];
 };
+static SCProfilePacketData prefilter4[256][256];
+static SCProfilePacketData prefilter6[256][256];
+
 
 struct ProfileProtoRecords packet_profile_flowworker_data[PROFILE_FLOWWORKER_SIZE];
 
@@ -174,6 +178,8 @@ SCProfilingInit(void)
             memset(&packet_profile_log_data4, 0, sizeof(packet_profile_log_data4));
             memset(&packet_profile_log_data6, 0, sizeof(packet_profile_log_data6));
             memset(&packet_profile_flowworker_data, 0, sizeof(packet_profile_flowworker_data));
+            memset(&prefilter4, 0, sizeof(prefilter4));
+            memset(&prefilter6, 0, sizeof(prefilter6));
 
             const char *filename = ConfNodeLookupChildValue(conf, "filename");
             if (filename != NULL) {
@@ -317,6 +323,85 @@ SCProfilingDump(void)
     SCLogPerf("Done dumping profiling data.");
 }
 
+static void DumpPrefilterIP(FILE *fp, int ipv, uint64_t total)
+{
+    char totalstr[256];
+
+    int i;
+    for (i = 0; i < 256; i++) {
+        const char *name = PrefilterStoreGetName(i);
+
+        for (int p = 0; p < 256; p++) {
+            SCProfilePacketData *pd = ipv == 4 ? &prefilter4[i][p] : &prefilter6[i][p];
+            if (pd->cnt == 0) {
+                continue;
+            }
+
+            FormatNumber(pd->tot, totalstr, sizeof(totalstr));
+            double percent = (long double)pd->tot /
+                (long double)total * 100;
+
+            fprintf(fp, "%-30s    IPv%d     %3d  %12"PRIu64"     %12"PRIu64"   %12"PRIu64"  %12"PRIu64"  %11s  %-6.2f\n",
+                    name, ipv, p, pd->cnt,
+                    pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent);
+        }
+    }
+}
+
+static void DumpPrefilter(FILE *fp)
+{
+    uint64_t total = 0;
+
+    int i;
+    for (i = 0; i < 256; i++) {
+        for (int p = 0; p < 256; p++) {
+            SCProfilePacketData *pd = &prefilter4[i][p];
+            total += pd->tot;
+            pd = &prefilter6[i][p];
+            total += pd->tot;
+        }
+    }
+
+    fprintf(fp, "\n%-30s   %-6s   %-5s   %-12s   %-12s   %-12s   %-12s   %-11s  %-3s\n",
+            "Prefilter", "IP ver", "Proto", "cnt", "min", "max", "avg", "tot", "%%");
+    fprintf(fp, "%-30s   %-6s   %-5s   %-12s   %-12s   %-12s   %-12s   %-11s  %-3s\n",
+            "--------------------", "------", "-----", "----------",
+            "------------", "------------", "-----------", "---------", "---");
+    DumpPrefilterIP(fp, 4, total);
+    DumpPrefilterIP(fp, 6, total);
+}
+
+static void SCProfilingUpdatePrefilterRecords(Packet *p)
+{
+    if (p->profile->prefilter.engines != NULL) {
+        uint32_t x;
+        for (x = 0; x < p->profile->prefilter.size; x++) {
+            uint64_t ticks = p->profile->prefilter.engines[x].ticks_spent;
+            if (ticks == 0)
+                continue;
+
+            SCProfilePacketData *pd = NULL;
+            if (PKT_IS_IPV4(p)) {
+                pd = &prefilter4[x][p->proto];
+            } else if (PKT_IS_IPV6(p)) {
+                pd = &prefilter6[x][p->proto];
+            } else {
+                continue;
+            }
+
+            if (pd->min == 0 || ticks < pd->min) {
+                pd->min = ticks;
+            }
+            if (pd->max < ticks) {
+                pd->max = ticks;
+            }
+
+            pd->tot += ticks;
+            pd->cnt ++;
+        }
+    }
+}
+
 static void DumpFlowWorkerIP(FILE *fp, int ipv, uint64_t total)
 {
     char totalstr[256];
@@ -743,6 +828,8 @@ void SCProfilingDumpPacketStats(void)
         }
     }
 
+    DumpPrefilter(fp);
+
     fprintf(fp, "\nGeneral detection engine stats:\n");
 
     total = 0;
@@ -1166,6 +1253,8 @@ void SCProfilingAddPacket(Packet *p)
             SCProfilingUpdatePacketDetectRecords(p);
             SCProfilingUpdatePacketLogRecords(p);
         }
+
+        SCProfilingUpdatePrefilterRecords(p);
     }
     pthread_mutex_unlock(&packet_profile_lock);
 }
@@ -1209,25 +1298,6 @@ int SCProfileRuleStart(Packet *p)
 const char * PacketProfileDetectIdToString(PacketProfileDetectId id)
 {
     switch (id) {
-        CASE_CODE (PROF_DETECT_MPM);
-        CASE_CODE (PROF_DETECT_MPM_PACKET);
-//        CASE_CODE (PROF_DETECT_MPM_PKT_STREAM);
-        CASE_CODE (PROF_DETECT_MPM_STREAM);
-        CASE_CODE (PROF_DETECT_MPM_URI);
-        CASE_CODE (PROF_DETECT_MPM_HCBD);
-        CASE_CODE (PROF_DETECT_MPM_HSBD);
-        CASE_CODE (PROF_DETECT_MPM_HHD);
-        CASE_CODE (PROF_DETECT_MPM_HRHD);
-        CASE_CODE (PROF_DETECT_MPM_HMD);
-        CASE_CODE (PROF_DETECT_MPM_HCD);
-        CASE_CODE (PROF_DETECT_MPM_HRUD);
-        CASE_CODE (PROF_DETECT_MPM_HSMD);
-        CASE_CODE (PROF_DETECT_MPM_HSCD);
-        CASE_CODE (PROF_DETECT_MPM_HUAD);
-        CASE_CODE (PROF_DETECT_MPM_DNSQUERY);
-        CASE_CODE (PROF_DETECT_MPM_TLSSNI);
-        CASE_CODE (PROF_DETECT_MPM_TLSISSUER);
-        CASE_CODE (PROF_DETECT_MPM_TLSSUBJECT);
         CASE_CODE (PROF_DETECT_IPONLY);
         CASE_CODE (PROF_DETECT_RULES);
         CASE_CODE (PROF_DETECT_PREFILTER);
@@ -1236,8 +1306,6 @@ const char * PacketProfileDetectIdToString(PacketProfileDetectId id)
         CASE_CODE (PROF_DETECT_CLEANUP);
         CASE_CODE (PROF_DETECT_GETSGH);
         CASE_CODE (PROF_DETECT_NONMPMLIST);
-        case PROF_DETECT_MPM_PKT_STREAM:
-            return "PROF_DETECT_MPM_PKT_STR";
         default:
             return "UNKNOWN";
     }

src/util-profiling.h

@@ -179,6 +179,7 @@ PktProfiling *SCProfilePacketStart(void);
 
 #define PACKET_PROFILING_RESET(p)                                   \
     if (profiling_packets_enabled && (p)->profile != NULL) {        \
+        SCFree((p)->profile->prefilter.engines);                    \
         SCFree((p)->profile);                                       \
         (p)->profile = NULL;                                        \
     }
@@ -274,6 +275,34 @@ PktProfiling *SCProfilePacketStart(void);
         SCProfilingSghUpdateCounter((det_ctx), (sgh));              \
     }
 
+#define PROFILING_PREFILTER_RESET(p, detectsize) \
+    if (profiling_packets_enabled  && (p)->profile != NULL) {       \
+        if ((p)->profile->prefilter.size != ((detectsize) + 1)) {   \
+            if ((p)->profile->prefilter.engines != NULL)            \
+                SCFree((p)->profile->prefilter.engines);            \
+            (p)->profile->prefilter.engines =                       \
+                SCCalloc((detectsize)+1, sizeof(PktProfilingPrefilterEngine)); \
+            (p)->profile->prefilter.size = (detectsize)+1;\
+        } else {                                                    \
+            memset((p)->profile->prefilter.engines, 0x00,           \
+                ((detectsize)+1 * sizeof(PktProfilingPrefilterEngine))); \
+        }                                                           \
+    }                                                               \
+
+#define PROFILING_PREFILTER_START(p) \
+    uint64_t ticks_start = 0; \
+    if (profiling_packets_enabled  && (p)->profile != NULL) {       \
+        ticks_start = UtilCpuGetTicks();                            \
+    }                                                               \
+
+#define PROFILING_PREFILTER_END(p, profile_id) \
+    if (profiling_packets_enabled && (p)->profile != NULL &&        \
+        ticks_start)                                                \
+    {                                                               \
+        uint64_t ticks_end = UtilCpuGetTicks();                     \
+        (p)->profile->prefilter.engines[(profile_id)].ticks_spent += (ticks_end - ticks_start);    \
+        ticks_start = 0;                                            \
+    }                                                               \
 
 void SCProfilingRulesGlobalInit(void);
 void SCProfilingRuleDestroyCtx(struct SCProfileDetectCtx_ *);
@@ -338,6 +367,10 @@ void SCProfilingDump(void);
 #define FLOWWORKER_PROFILING_START(p, id)
 #define FLOWWORKER_PROFILING_END(p, id)
 
+#define PROFILING_PREFILTER_RESET(p, detectsize)
+#define PROFILING_PREFILTER_START(p)
+#define PROFILING_PREFILTER_END(p, profile_id)
+
 #endif /* PROFILING */
 
 #endif /* ! __UTIL_PROFILE_H__ */