detect: introduce pkt mpm engines

Instead of the hardcode L4 matching in MPM that was recently introduced,
add an API similar to the AppLayer MPM and inspect engines.

Share part of the registration code with the AppLayer.

Implement for the tcp.hdr and udp.hdr keywords.

src/detect-dns-query.c

@@ -186,17 +186,17 @@ static void PrefilterMpmDnsQueryFree(void *ptr)
 
 static int PrefilterMpmDnsQueryRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     PrefilterMpmDnsQuery *pectx = SCCalloc(1, sizeof(*pectx));
     if (pectx == NULL)
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxDnsQuery,
-            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+            mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
             pectx, PrefilterMpmDnsQueryFree, mpm_reg->name);
 }
 

src/detect-engine-analyzer.c

@@ -534,6 +534,9 @@ static bool LooksLikeHTTPUA(const uint8_t *buf, uint16_t len)
 
 static void DumpMatches(RuleAnalyzer *ctx, json_t *js, const SigMatchData *smd)
 {
+    if (smd == NULL)
+        return;
+
     json_t *js_matches = json_array();
     if (js_matches == NULL) {
         return;
@@ -722,6 +725,36 @@ void EngineAnalysisRules2(const DetectEngineCtx *de_ctx, const Signature *s)
         json_object_set_new(ctx.js, "flags", js_flags);
     }
 
+    json_t *js_pkt_array = json_array();
+    const DetectEnginePktInspectionEngine *pkt = s->pkt_inspect;
+    for ( ; pkt != NULL; pkt = pkt->next) {
+        const char *name = DetectBufferTypeGetNameById(de_ctx, pkt->sm_list);
+        if (name == NULL) {
+            switch (pkt->sm_list) {
+                case DETECT_SM_LIST_PMATCH:
+                    name = "payload";
+                    break;
+                case DETECT_SM_LIST_MATCH:
+                    name = "packet";
+                    break;
+                default:
+                    name = "unknown";
+                    break;
+            }
+        }
+        json_t *js_engine = json_object();
+        if (js_engine != NULL) {
+            json_object_set_new(js_engine, "name", json_string(name));
+
+            json_object_set_new(js_engine, "is_mpm", json_boolean(pkt->mpm));
+
+            DumpMatches(&ctx, js_engine, pkt->smd);
+
+            json_array_append_new(js_pkt_array, js_engine);
+        }
+    }
+    json_object_set_new(ctx.js, "pkt_engines", js_pkt_array);
+
     if (s->init_data->init_flags & SIG_FLAG_INIT_STATE_MATCH) {
         bool has_stream = false;
         bool has_client_body_mpm = false;
@@ -897,7 +930,6 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
     uint32_t warn_no_direction = 0;
     uint32_t warn_both_direction = 0;
 
-    const int nlists = de_ctx->buffer_type_id;
     const int filedata_id = DetectBufferTypeGetByName("file_data");
     const int httpmethod_id = DetectBufferTypeGetByName("http_method");
     const int httpuri_id = DetectBufferTypeGetByName("http_uri");
@@ -932,7 +964,7 @@ void EngineAnalysisRules(const DetectEngineCtx *de_ctx,
         rule_ipv6_only += 1;
     }
 
-    for (list_id = 0; list_id < nlists; list_id++) {
+    for (list_id = 0; list_id < (int)s->init_data->smlists_array_size; list_id++) {
         SigMatch *sm = NULL;
         for (sm = s->init_data->smlists[list_id]; sm != NULL; sm = sm->next) {
             if (sm->type == DETECT_PCRE) {

src/detect-engine-build.c

@@ -191,8 +191,6 @@ int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
 
     if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
         return 0;
-    if (s->init_data->smlists[DETECT_SM_LIST_L4HDR] != NULL)
-        return 0;
 
     /* if flow dir is set we can't process it in ip-only */
     if (!(((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == 0) ||
@@ -583,9 +581,6 @@ static int SignatureCreateMask(Signature *s)
 static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx)
 {
     DetectMpmInitializeBuiltinMpms(de_ctx);
-    DetectMpmSetupAppMpms(de_ctx);
-
-    return;
 }
 
 /** \brief Pure-PCRE or bytetest rule */
@@ -1926,6 +1921,7 @@ int SigGroupBuild(DetectEngineCtx *de_ctx)
 
     int r = DetectMpmPrepareBuiltinMpms(de_ctx);
     r |= DetectMpmPrepareAppMpms(de_ctx);
+    r |= DetectMpmPreparePktMpms(de_ctx);
     if (r != 0) {
         SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
         exit(EXIT_FAILURE);
@@ -1947,8 +1943,6 @@ int SigGroupBuild(DetectEngineCtx *de_ctx)
 
     SCProfilingRuleInitCounters(de_ctx);
 #endif
-    SCFree(de_ctx->app_mpms);
-    de_ctx->app_mpms = NULL;
 
     if (!DetectEngineMultiTenantEnabled()) {
         VarNameStoreActivateStaging();

src/detect-engine-mpm.c

@@ -70,8 +70,6 @@ const char *builtin_mpms[] = {
     "toserver UDP packet",
     "toclient UDP packet",
     "other IP packet",
-    "toserver L4 header",
-    "toclient L4 header",
 
     NULL };
 
@@ -80,8 +78,8 @@ const char *builtin_mpms[] = {
  * Keywords are registered at engine start up
  */
 
-static DetectMpmAppLayerRegistery *g_app_mpms_list = NULL;
+static DetectBufferMpmRegistery *g_mpm_list[DETECT_BUFFER_MPM_TYPE_SIZE] = { NULL, NULL };
-static int g_app_mpms_list_cnt = 0;
+static int g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_SIZE] = { 0, 0 };
 
 /** \brief register a MPM engine
  *
@@ -91,7 +89,7 @@ void DetectAppLayerMpmRegister2(const char *name,
         int direction, int priority,
         int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
             SigGroupHead *sgh, MpmCtx *mpm_ctx,
-            const DetectMpmAppLayerRegistery *mpm_reg, int list_id),
+            const DetectBufferMpmRegistery *mpm_reg, int list_id),
         InspectionBufferGetDataPtr GetData,
         AppProto alproto, int tx_min_progress)
 {
@@ -111,23 +109,24 @@ void DetectAppLayerMpmRegister2(const char *name,
                 "MPM engine registration for %s failed", name);
     }
 
-    DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
+    DetectBufferMpmRegistery *am = SCCalloc(1, sizeof(*am));
     BUG_ON(am == NULL);
     am->name = name;
     snprintf(am->pname, sizeof(am->pname), "%s", am->name);
     am->direction = direction;
     am->sm_list = sm_list;
     am->priority = priority;
+    am->type = DETECT_BUFFER_MPM_TYPE_APP;
 
-    am->v2.PrefilterRegisterWithListId = PrefilterRegister;
+    am->PrefilterRegisterWithListId = PrefilterRegister;
-    am->v2.GetData = GetData;
+    am->app_v2.GetData = GetData;
-    am->v2.alproto = alproto;
+    am->app_v2.alproto = alproto;
-    am->v2.tx_min_progress = tx_min_progress;
+    am->app_v2.tx_min_progress = tx_min_progress;
 
-    if (g_app_mpms_list == NULL) {
+    if (g_mpm_list[DETECT_BUFFER_MPM_TYPE_APP] == NULL) {
-        g_app_mpms_list = am;
+        g_mpm_list[DETECT_BUFFER_MPM_TYPE_APP] = am;
     } else {
-        DetectMpmAppLayerRegistery *t = g_app_mpms_list;
+        DetectBufferMpmRegistery *t = g_mpm_list[DETECT_BUFFER_MPM_TYPE_APP];
         while (t->next != NULL) {
             t = t->next;
         }
@@ -135,101 +134,231 @@ void DetectAppLayerMpmRegister2(const char *name,
         t->next = am;
         am->id = t->id + 1;
     }
-    g_app_mpms_list_cnt++;
+    g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_APP]++;
 
     SupportFastPatternForSigMatchList(sm_list, priority);
 }
 
-void DetectAppLayerMpmRegister(const char *name,
+/** \brief copy a mpm engine from parent_id, add in transforms */
-        int direction, int priority,
+void DetectAppLayerMpmRegisterByParentId(DetectEngineCtx *de_ctx,
+        const int id, const int parent_id,
+        DetectEngineTransforms *transforms)
+{
+    SCLogDebug("registering %d/%d", id, parent_id);
+
+    DetectBufferMpmRegistery *t = de_ctx->app_mpms_list;
+    while (t) {
+        if (t->sm_list == parent_id) {
+            DetectBufferMpmRegistery *am = SCCalloc(1, sizeof(*am));
+            BUG_ON(am == NULL);
+            am->name = t->name;
+            snprintf(am->pname, sizeof(am->pname), "%s#%d", am->name, id);
+            am->direction = t->direction;
+            am->sm_list = id; // use new id
+            am->type = DETECT_BUFFER_MPM_TYPE_APP;
+            am->PrefilterRegisterWithListId = t->PrefilterRegisterWithListId;
+            am->app_v2.GetData = t->app_v2.GetData;
+            am->app_v2.alproto = t->app_v2.alproto;
+            am->app_v2.tx_min_progress = t->app_v2.tx_min_progress;
+            am->priority = t->priority;
+            am->sgh_mpm_context = t->sgh_mpm_context;
+            am->next = t->next;
+            if (transforms) {
+                memcpy(&am->transforms, transforms, sizeof(*transforms));
+            }
+            am->id = de_ctx->app_mpms_list_cnt++;
+
+            SupportFastPatternForSigMatchList(am->sm_list, am->priority);
+            t->next = am;
+            SCLogDebug("copied mpm registration for %s id %u "
+                    "with parent %u and GetData %p",
+                    t->name, id, parent_id, am->app_v2.GetData);
+            t = am;
+        }
+        t = t->next;
+    }
+}
+
+void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx)
+{
+    const DetectBufferMpmRegistery *list = g_mpm_list[DETECT_BUFFER_MPM_TYPE_APP];
+    while (list != NULL) {
+        DetectBufferMpmRegistery *n = SCCalloc(1, sizeof(*n));
+        BUG_ON(n == NULL);
+
+        *n = *list;
+        n->next = NULL;
+
+        if (de_ctx->app_mpms_list == NULL) {
+            de_ctx->app_mpms_list = n;
+        } else {
+            DetectBufferMpmRegistery *t = de_ctx->app_mpms_list;
+            while (t->next != NULL) {
+                t = t->next;
+            }
+            t->next = n;
+        }
+
+        /* default to whatever the global setting is */
+        int shared = (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);
+
+        /* see if we use a unique or shared mpm ctx for this type */
+        int confshared = 0;
+        char confstring[256] = "detect.mpm.";
+        strlcat(confstring, n->name, sizeof(confstring));
+        strlcat(confstring, ".shared", sizeof(confstring));
+        if (ConfGetBool(confstring, &confshared) == 1)
+            shared = confshared;
+
+        if (shared == 0) {
+            if (!(de_ctx->flags & DE_QUIET)) {
+                SCLogPerf("using unique mpm ctx' for %s", n->name);
+            }
+            n->sgh_mpm_context = MPM_CTX_FACTORY_UNIQUE_CONTEXT;
+        } else {
+            if (!(de_ctx->flags & DE_QUIET)) {
+                SCLogPerf("using shared mpm ctx' for %s", n->name);
+            }
+            n->sgh_mpm_context = MpmFactoryRegisterMpmCtxProfile(de_ctx, n->name);
+        }
+
+        list = list->next;
+    }
+    de_ctx->app_mpms_list_cnt = g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_APP];
+    SCLogDebug("mpm: de_ctx app_mpms_list %p %u",
+            de_ctx->app_mpms_list, de_ctx->app_mpms_list_cnt);
+}
+
+/**
+ *  \brief initialize mpm contexts for applayer buffers that are in
+ *         "single or "shared" mode.
+ */
+int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
+{
+    int r = 0;
+    const DetectBufferMpmRegistery *am = de_ctx->app_mpms_list;
+    while (am != NULL) {
+        int dir = (am->direction == SIG_FLAG_TOSERVER) ? 1 : 0;
+
+        if (am->sgh_mpm_context != MPM_CTX_FACTORY_UNIQUE_CONTEXT)
+        {
+            MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
+            if (mpm_ctx != NULL) {
+                if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
+                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                }
+            }
+        }
+        am = am->next;
+    }
+    return r;
+}
+
+/** \brief register a MPM engine
+ *
+ *  \note to be used at start up / registration only. Errors are fatal.
+ */
+void DetectPktMpmRegister(const char *name,
+        int priority,
         int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
-            SigGroupHead *sgh, MpmCtx *mpm_ctx))
+            SigGroupHead *sgh, MpmCtx *mpm_ctx,
+            const DetectBufferMpmRegistery *mpm_reg, int list_id),
+        InspectionBufferGetPktDataPtr GetData)
 {
-    SCLogDebug("registering %s/%d/%d/%p",
+    SCLogDebug("registering %s/%d/%p/%p", name, priority,
-            name, direction, priority, PrefilterRegister);
+            PrefilterRegister, GetData);
+
+    if (PrefilterRegister == PrefilterGenericMpmPktRegister && GetData == NULL) {
+        // must register GetData with PrefilterGenericMpmRegister
+        abort();
+    }
 
     DetectBufferTypeSupportsMpm(name);
+    DetectBufferTypeSupportsTransformations(name);
     int sm_list = DetectBufferTypeGetByName(name);
-    BUG_ON(sm_list == -1);
+    if (sm_list == -1) {
+        FatalError(SC_ERR_INITIALIZATION,
+                "MPM engine registration for %s failed", name);
+    }
 
-    DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
+    DetectBufferMpmRegistery *am = SCCalloc(1, sizeof(*am));
     BUG_ON(am == NULL);
     am->name = name;
     snprintf(am->pname, sizeof(am->pname), "%s", am->name);
-    am->direction = direction;
     am->sm_list = sm_list;
     am->priority = priority;
-    am->PrefilterRegister = PrefilterRegister;
+    am->type = DETECT_BUFFER_MPM_TYPE_PKT;
+
+    am->PrefilterRegisterWithListId = PrefilterRegister;
+    am->pkt_v1.GetData = GetData;
 
-    if (g_app_mpms_list == NULL) {
+    if (g_mpm_list[DETECT_BUFFER_MPM_TYPE_PKT] == NULL) {
-        g_app_mpms_list = am;
+        g_mpm_list[DETECT_BUFFER_MPM_TYPE_PKT] = am;
     } else {
-        DetectMpmAppLayerRegistery *t = g_app_mpms_list;
+        DetectBufferMpmRegistery *t = g_mpm_list[DETECT_BUFFER_MPM_TYPE_PKT];
         while (t->next != NULL) {
             t = t->next;
         }
-
         t->next = am;
         am->id = t->id + 1;
     }
-    g_app_mpms_list_cnt++;
+    g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_PKT]++;
 
     SupportFastPatternForSigMatchList(sm_list, priority);
+    SCLogDebug("%s/%d done", name, sm_list);
 }
 
 /** \brief copy a mpm engine from parent_id, add in transforms */
-void DetectAppLayerMpmRegisterByParentId(DetectEngineCtx *de_ctx,
+void DetectPktMpmRegisterByParentId(DetectEngineCtx *de_ctx,
         const int id, const int parent_id,
         DetectEngineTransforms *transforms)
 {
     SCLogDebug("registering %d/%d", id, parent_id);
 
-    DetectMpmAppLayerRegistery *t = de_ctx->app_mpms_list;
+    DetectBufferMpmRegistery *t = de_ctx->pkt_mpms_list;
     while (t) {
         if (t->sm_list == parent_id) {
-            DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
+            DetectBufferMpmRegistery *am = SCCalloc(1, sizeof(*am));
             BUG_ON(am == NULL);
             am->name = t->name;
             snprintf(am->pname, sizeof(am->pname), "%s#%d", am->name, id);
-            am->direction = t->direction;
             am->sm_list = id; // use new id
-            am->PrefilterRegister = t->PrefilterRegister;
+            am->type = DETECT_BUFFER_MPM_TYPE_PKT;
-            am->v2.PrefilterRegisterWithListId = t->v2.PrefilterRegisterWithListId;
+            am->PrefilterRegisterWithListId = t->PrefilterRegisterWithListId;
-            am->v2.GetData = t->v2.GetData;
+            am->pkt_v1.GetData = t->pkt_v1.GetData;
-            am->v2.alproto = t->v2.alproto;
-            am->v2.tx_min_progress = t->v2.tx_min_progress;
             am->priority = t->priority;
+            am->sgh_mpm_context = t->sgh_mpm_context;
             am->next = t->next;
             if (transforms) {
-                memcpy(&am->v2.transforms, transforms, sizeof(*transforms));
+                memcpy(&am->transforms, transforms, sizeof(*transforms));
             }
-            am->id = de_ctx->app_mpms_list_cnt++;
+            am->id = de_ctx->pkt_mpms_list_cnt++;
 
             SupportFastPatternForSigMatchList(am->sm_list, am->priority);
             t->next = am;
             SCLogDebug("copied mpm registration for %s id %u "
                     "with parent %u and GetData %p",
-                    t->name, id, parent_id, am->v2.GetData);
+                    t->name, id, parent_id, am->pkt_v1.GetData);
             t = am;
         }
         t = t->next;
     }
 }
 
-void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx)
+void DetectMpmInitializePktMpms(DetectEngineCtx *de_ctx)
 {
-    const DetectMpmAppLayerRegistery *list = g_app_mpms_list;
+    const DetectBufferMpmRegistery *list = g_mpm_list[DETECT_BUFFER_MPM_TYPE_PKT];
     while (list != NULL) {
-        DetectMpmAppLayerRegistery *n = SCCalloc(1, sizeof(*n));
+        DetectBufferMpmRegistery *n = SCCalloc(1, sizeof(*n));
         BUG_ON(n == NULL);
 
         *n = *list;
         n->next = NULL;
 
-        if (de_ctx->app_mpms_list == NULL) {
+        if (de_ctx->pkt_mpms_list == NULL) {
-            de_ctx->app_mpms_list = n;
+            de_ctx->pkt_mpms_list = n;
         } else {
-            DetectMpmAppLayerRegistery *t = de_ctx->app_mpms_list;
+            DetectBufferMpmRegistery *t = de_ctx->pkt_mpms_list;
             while (t->next != NULL) {
                 t = t->next;
             }
@@ -237,75 +366,58 @@ void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx)
             t->next = n;
         }
 
-        list = list->next;
-    }
-    de_ctx->app_mpms_list_cnt = g_app_mpms_list_cnt;
-    SCLogDebug("mpm: de_ctx app_mpms_list %p %u",
-            de_ctx->app_mpms_list, de_ctx->app_mpms_list_cnt);
-}
-
-void DetectMpmSetupAppMpms(DetectEngineCtx *de_ctx)
-{
-    BUG_ON(de_ctx->app_mpms_list_cnt == 0);
-
-    de_ctx->app_mpms = SCCalloc(de_ctx->app_mpms_list_cnt + 1, sizeof(DetectMpmAppLayerKeyword));
-    BUG_ON(de_ctx->app_mpms == NULL);
-
-    DetectMpmAppLayerRegistery *list = de_ctx->app_mpms_list;
-    while (list != NULL) {
-        DetectMpmAppLayerKeyword *am = &de_ctx->app_mpms[list->id];
-        am->reg = list;
-
         /* default to whatever the global setting is */
         int shared = (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);
 
         /* see if we use a unique or shared mpm ctx for this type */
         int confshared = 0;
         char confstring[256] = "detect.mpm.";
-        strlcat(confstring, am->reg->name, sizeof(confstring));
+        strlcat(confstring, n->name, sizeof(confstring));
         strlcat(confstring, ".shared", sizeof(confstring));
         if (ConfGetBool(confstring, &confshared) == 1)
             shared = confshared;
 
         if (shared == 0) {
             if (!(de_ctx->flags & DE_QUIET)) {
-                SCLogPerf("using unique mpm ctx' for %s", am->reg->name);
+                SCLogPerf("using unique mpm ctx' for %s", n->name);
             }
-            am->sgh_mpm_context = MPM_CTX_FACTORY_UNIQUE_CONTEXT;
+            n->sgh_mpm_context = MPM_CTX_FACTORY_UNIQUE_CONTEXT;
         } else {
             if (!(de_ctx->flags & DE_QUIET)) {
-                SCLogPerf("using shared mpm ctx' for %s", am->reg->name);
+                SCLogPerf("using shared mpm ctx' for %s", n->name);
             }
-            am->sgh_mpm_context = MpmFactoryRegisterMpmCtxProfile(de_ctx, am->reg->name);
+            n->sgh_mpm_context = MpmFactoryRegisterMpmCtxProfile(de_ctx, n->name);
         }
 
-        SCLogDebug("AppLayer MPM %s: %u", am->reg->name, am->sgh_mpm_context);
-
         list = list->next;
     }
+    de_ctx->pkt_mpms_list_cnt = g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_PKT];
+    SCLogDebug("mpm: de_ctx pkt_mpms_list %p %u",
+            de_ctx->pkt_mpms_list, de_ctx->pkt_mpms_list_cnt);
 }
 
 /**
  *  \brief initialize mpm contexts for applayer buffers that are in
  *         "single or "shared" mode.
  */
-int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
+int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx)
 {
+    SCLogDebug("preparing pkt mpm");
     int r = 0;
-    DetectMpmAppLayerKeyword *am = de_ctx->app_mpms;
+    const DetectBufferMpmRegistery *am = de_ctx->pkt_mpms_list;
-    while (am->reg != NULL) {
+    while (am != NULL) {
-        int dir = (am->reg->direction == SIG_FLAG_TOSERVER) ? 1 : 0;
+        SCLogDebug("%s", am->name);
-
         if (am->sgh_mpm_context != MPM_CTX_FACTORY_UNIQUE_CONTEXT)
         {
-            MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
+            MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, 0);
             if (mpm_ctx != NULL) {
                 if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
                     r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
+                    SCLogDebug("%s: %d", am->name, r);
                 }
             }
         }
-        am++;
+        am = am->next;
     }
     return r;
 }
@@ -341,7 +453,6 @@ void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
 
     de_ctx->sgh_mpm_context_proto_udp_packet = SetupBuiltinMpm(de_ctx, "udp-packet");
     de_ctx->sgh_mpm_context_proto_other_packet = SetupBuiltinMpm(de_ctx, "other-ip");
-    de_ctx->sgh_mpm_context_l4_header = SetupBuiltinMpm(de_ctx, "l4-header");
 }
 
 /**
@@ -393,17 +504,6 @@ int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
         }
     }
 
-    if (de_ctx->sgh_mpm_context_l4_header != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
-        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_l4_header, 0);
-        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
-        }
-        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_l4_header, 1);
-        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
-            r |= mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx);
-        }
-    }
-
     return r;
 }
 
@@ -951,12 +1051,7 @@ void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
 {
     HashListTableBucket *htb = NULL;
 
-    int app_mpms_cnt = 0;
+    int app_mpms_cnt = de_ctx->app_mpms_list_cnt;
-    DetectMpmAppLayerKeyword *a = de_ctx->app_mpms;
-    while (a->reg != NULL) {
-        a++;
-        app_mpms_cnt++;
-    }
     uint32_t stats[MPMB_MAX] = {0};
     uint32_t appstats[app_mpms_cnt + 1];    // +1 to silence scan-build
     memset(&appstats, 0x00, sizeof(appstats));
@@ -973,37 +1068,40 @@ void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
             stats[ms->buffer]++;
         else if (ms->sm_list != DETECT_SM_LIST_PMATCH) {
             int i = 0;
-            DetectMpmAppLayerKeyword *am = de_ctx->app_mpms;
+            const DetectBufferMpmRegistery *am = de_ctx->app_mpms_list;
-            while (am->reg != NULL) {
+            while (am != NULL) {
-                if (ms->sm_list == am->reg->sm_list &&
+                if (ms->sm_list == am->sm_list &&
-                    ms->direction == am->reg->direction)
+                    ms->direction == am->direction)
                 {
                     SCLogDebug("%s %s: %u patterns. Min %u, Max %u. Ctx %p",
-                            am->reg->name,
+                            am->name,
-                            am->reg->direction == SIG_FLAG_TOSERVER ? "toserver":"toclient",
+                            am->direction == SIG_FLAG_TOSERVER ? "toserver":"toclient",
                             ms->mpm_ctx->pattern_cnt,
                             ms->mpm_ctx->minlen, ms->mpm_ctx->maxlen,
                             ms->mpm_ctx);
-                    appstats[i]++;
+                    appstats[am->sm_list]++;
                     break;
                 }
                 i++;
-                am++;
+                am = am->next;
             }
         }
     }
 
     if (!(de_ctx->flags & DE_QUIET)) {
-        int x;
+        for (int x = 0; x < MPMB_MAX; x++) {
-        for (x = 0; x < MPMB_MAX; x++) {
             SCLogPerf("Builtin MPM \"%s\": %u", builtin_mpms[x], stats[x]);
         }
-        for (x = 0; x < app_mpms_cnt; x++) {
+        const DetectBufferMpmRegistery *am = de_ctx->app_mpms_list;
-            if (appstats[x] == 0)
+        while (am != NULL) {
-                continue;
+            if (appstats[am->sm_list] == 0)
-            const char *name = de_ctx->app_mpms[x].reg->name;
+                goto next;
-            const char *direction = de_ctx->app_mpms[x].reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient";
+
-            SCLogPerf("AppLayer MPM \"%s %s\": %u", direction, name, appstats[x]);
+            const char *name = am->name;
+            const char *direction = am->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient";
+            SCLogPerf("AppLayer MPM \"%s %s\": %u", direction, name, appstats[am->sm_list]);
+        next:
+            am = am->next;
         }
     }
 }
@@ -1028,18 +1126,16 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
 {
     const Signature *s = NULL;
     uint32_t sig;
-
     int dir = 0;
 
     if (ms->buffer != MPMB_MAX) {
-        BUG_ON(ms->sm_list != DETECT_SM_LIST_PMATCH && ms->sm_list != DETECT_SM_LIST_L4HDR);
+        BUG_ON(ms->sm_list != DETECT_SM_LIST_PMATCH);
 
         switch (ms->buffer) {
             /* TS is 1 */
             case MPMB_TCP_PKT_TS:
             case MPMB_TCP_STREAM_TS:
             case MPMB_UDP_TS:
-            case MPMB_L4HDR_TS:
                 dir = 1;
                 break;
 
@@ -1049,14 +1145,11 @@ static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
             case MPMB_TCP_STREAM_TC:
             case MPMB_TCP_PKT_TC:
             case MPMB_OTHERIP:          /**< use 0 for other */
-            case MPMB_L4HDR_TC:
                 dir = 0;
                 break;
         }
     } else {
         BUG_ON(ms->sm_list == DETECT_SM_LIST_PMATCH);
-        BUG_ON(ms->direction == 0);
-        BUG_ON(ms->direction == (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT));
 
         if (ms->direction == SIG_FLAG_TOSERVER)
             dir = 1;
@@ -1155,11 +1248,6 @@ MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
         case MPMB_OTHERIP:
             sgh_mpm_context = de_ctx->sgh_mpm_context_proto_other_packet;
             break;
-        case MPMB_L4HDR_TS:
-        case MPMB_L4HDR_TC:
-            sgh_mpm_context = de_ctx->sgh_mpm_context_l4_header;
-            sm_list = DETECT_SM_LIST_L4HDR;
-            break;
         default:
             break;
     }
@@ -1168,14 +1256,12 @@ MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
         case MPMB_TCP_PKT_TS:
         case MPMB_TCP_STREAM_TS:
         case MPMB_UDP_TS:
-        case MPMB_L4HDR_TS:
             direction = SIG_FLAG_TOSERVER;
             break;
 
         case MPMB_TCP_PKT_TC:
         case MPMB_TCP_STREAM_TC:
         case MPMB_UDP_TC:
-        case MPMB_L4HDR_TC:
             direction = SIG_FLAG_TOCLIENT;
             break;
 
@@ -1200,7 +1286,7 @@ MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
         if (list < 0)
             continue;
 
-        if (list != DETECT_SM_LIST_PMATCH && list != DETECT_SM_LIST_L4HDR)
+        if (list != DETECT_SM_LIST_PMATCH)
             continue;
 
         switch (buf) {
@@ -1229,11 +1315,6 @@ MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
                 sids_array[s->num / 8] |= 1 << (s->num % 8);
                 cnt++;
                 break;
-            case MPMB_L4HDR_TS:
-            case MPMB_L4HDR_TC:
-                sids_array[s->num / 8] |= 1 << (s->num % 8);
-                cnt++;
-                break;
             default:
                 break;
         }
@@ -1272,7 +1353,7 @@ MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 }
 
 static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, DetectMpmAppLayerKeyword *am)
+        SigGroupHead *sgh, const DetectBufferMpmRegistery *am)
 {
     const Signature *s = NULL;
     uint32_t sig;
@@ -1281,9 +1362,9 @@ static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx,
     uint8_t sids_array[max_sid];
     memset(sids_array, 0x00, max_sid);
 
-    SCLogDebug("handling %s direction %s for list %d", am->reg->name,
+    SCLogDebug("handling %s direction %s for list %d", am->name,
-            am->reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
+            am->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
-            am->reg->sm_list);
+            am->sm_list);
 
     for (sig = 0; sig < sgh->sig_cnt; sig++) {
         s = sgh->match_array[sig];
@@ -1297,10 +1378,10 @@ static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx,
         if (list < 0)
             continue;
 
-        if ((s->flags & am->reg->direction) == 0)
+        if ((s->flags & am->direction) == 0)
             continue;
 
-        if (list != am->reg->sm_list)
+        if (list != am->sm_list)
             continue;
 
         sids_array[s->num / 8] |= 1 << (s->num % 8);
@@ -1310,16 +1391,16 @@ static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx,
     if (cnt == 0)
         return NULL;
 
-    MpmStore lookup = { sids_array, max_sid, am->reg->direction,
+    MpmStore lookup = { sids_array, max_sid, am->direction,
-        MPMB_MAX, am->reg->sm_list, 0, NULL};
+        MPMB_MAX, am->sm_list, 0, NULL};
     SCLogDebug("am->direction %d am->sm_list %d",
-            am->reg->direction, am->reg->sm_list);
+            am->direction, am->sm_list);
 
     MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
     if (result == NULL) {
         SCLogDebug("new unique mpm for %s %s: %u patterns",
-                am->reg->name,
+                am->name,
-                am->reg->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
+                am->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
                 cnt);
 
         MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
@@ -1335,8 +1416,79 @@ static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx,
         copy->sid_array = sids;
         copy->sid_array_size = max_sid;
         copy->buffer = MPMB_MAX;
-        copy->direction = am->reg->direction;
+        copy->direction = am->direction;
-        copy->sm_list = am->reg->sm_list;
+        copy->sm_list = am->sm_list;
+        copy->sgh_mpm_context = am->sgh_mpm_context;
+
+        MpmStoreSetup(de_ctx, copy);
+        MpmStoreAdd(de_ctx, copy);
+        return copy;
+    } else {
+        SCLogDebug("using existing mpm %p", result);
+        return result;
+    }
+    return NULL;
+}
+
+static MpmStore *MpmStorePrepareBufferPkt(DetectEngineCtx *de_ctx,
+        SigGroupHead *sgh, const DetectBufferMpmRegistery *am)
+{
+    const Signature *s = NULL;
+    uint32_t sig;
+    uint32_t cnt = 0;
+    uint32_t max_sid = DetectEngineGetMaxSigId(de_ctx) / 8 + 1;
+    uint8_t sids_array[max_sid];
+    memset(sids_array, 0x00, max_sid);
+
+    SCLogDebug("handling %s for list %d", am->name,
+            am->sm_list);
+
+    for (sig = 0; sig < sgh->sig_cnt; sig++) {
+        s = sgh->match_array[sig];
+        if (s == NULL)
+            continue;
+
+        if (s->init_data->mpm_sm == NULL)
+            continue;
+
+        int list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm);
+        if (list < 0)
+            continue;
+
+        if (list != am->sm_list)
+            continue;
+
+        sids_array[s->num / 8] |= 1 << (s->num % 8);
+        cnt++;
+    }
+
+    if (cnt == 0)
+        return NULL;
+
+    MpmStore lookup = { sids_array, max_sid, SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT,
+        MPMB_MAX, am->sm_list, 0, NULL};
+    SCLogDebug("am->sm_list %d", am->sm_list);
+
+    MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
+    if (result == NULL) {
+        SCLogDebug("new unique mpm for %s: %u patterns",
+                am->name, cnt);
+
+        MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
+        if (copy == NULL)
+            return NULL;
+        uint8_t *sids = SCCalloc(1, max_sid);
+        if (sids == NULL) {
+            SCFree(copy);
+            return NULL;
+        }
+
+        memcpy(sids, sids_array, max_sid);
+        copy->sid_array = sids;
+        copy->sid_array_size = max_sid;
+        copy->buffer = MPMB_MAX;
+        copy->direction = SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT;
+        copy->sm_list = am->sm_list;
         copy->sgh_mpm_context = am->sgh_mpm_context;
 
         MpmStoreSetup(de_ctx, copy);
@@ -1368,6 +1520,72 @@ static void SetRawReassemblyFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     SCLogDebug("rule group %p does NOT have SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
 }
 
+static void PrepareAppMpms(DetectEngineCtx *de_ctx, SigGroupHead *sh)
+{
+    if (de_ctx->app_mpms_list_cnt == 0)
+        return;
+
+    sh->init->app_mpms = SCCalloc(de_ctx->app_mpms_list_cnt, sizeof(MpmCtx *));
+    BUG_ON(sh->init->app_mpms == NULL);
+
+    DetectBufferMpmRegistery *a = de_ctx->app_mpms_list;
+    while (a != NULL) {
+        if ((a->direction == SIG_FLAG_TOSERVER && SGH_DIRECTION_TS(sh)) ||
+            (a->direction == SIG_FLAG_TOCLIENT && SGH_DIRECTION_TC(sh)))
+        {
+            MpmStore *mpm_store = MpmStorePrepareBufferAppLayer(de_ctx, sh, a);
+            if (mpm_store != NULL) {
+                sh->init->app_mpms[a->id] = mpm_store->mpm_ctx;
+
+                SCLogDebug("a %p a->name %s a->PrefilterRegisterWithListId %p "
+                        "mpm_store->mpm_ctx %p", a, a->name,
+                        a->PrefilterRegisterWithListId, mpm_store->mpm_ctx);
+
+                /* if we have just certain types of negated patterns,
+                 * mpm_ctx can be NULL */
+                if (a->PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
+                    BUG_ON(a->PrefilterRegisterWithListId(de_ctx,
+                                sh, mpm_store->mpm_ctx,
+                                a, a->sm_list) != 0);
+                    SCLogDebug("mpm %s %d set up", a->name, a->sm_list);
+                }
+            }
+        }
+        a = a->next;
+    }
+}
+
+static void PreparePktMpms(DetectEngineCtx *de_ctx, SigGroupHead *sh)
+{
+    if (de_ctx->pkt_mpms_list_cnt == 0)
+        return;
+
+    sh->init->pkt_mpms = SCCalloc(de_ctx->pkt_mpms_list_cnt, sizeof(MpmCtx *));
+    BUG_ON(sh->init->pkt_mpms == NULL);
+
+    DetectBufferMpmRegistery *a = de_ctx->pkt_mpms_list;
+    while (a != NULL) {
+        MpmStore *mpm_store = MpmStorePrepareBufferPkt(de_ctx, sh, a);
+        if (mpm_store != NULL) {
+            sh->init->pkt_mpms[a->id] = mpm_store->mpm_ctx;
+
+            SCLogDebug("a %p a->name %s a->reg->PrefilterRegisterWithListId %p "
+                    "mpm_store->mpm_ctx %p", a, a->name,
+                    a->PrefilterRegisterWithListId, mpm_store->mpm_ctx);
+
+            /* if we have just certain types of negated patterns,
+             * mpm_ctx can be NULL */
+            if (a->PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
+                BUG_ON(a->PrefilterRegisterWithListId(de_ctx,
+                            sh, mpm_store->mpm_ctx,
+                            a, a->sm_list) != 0);
+                SCLogDebug("mpm %s %d set up", a->name, a->sm_list);
+            }
+        }
+        a = a->next;
+    }
+}
+
 /** \brief Prepare the pattern matcher ctx in a sig group head.
  *
  */
@@ -1387,11 +1605,6 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             }
 
             SetRawReassemblyFlag(de_ctx, sh);
-
-            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_L4HDR_TS);
-            if (mpm_store != NULL) {
-                PrefilterTcpHeaderRegister(de_ctx, sh, mpm_store->mpm_ctx);
-            }
         }
         if (SGH_DIRECTION_TC(sh)) {
             mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_PKT_TC);
@@ -1405,11 +1618,6 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             }
 
             SetRawReassemblyFlag(de_ctx, sh);
-
-            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_L4HDR_TC);
-            if (mpm_store != NULL) {
-                PrefilterTcpHeaderRegister(de_ctx, sh, mpm_store->mpm_ctx);
-            }
        }
     } else if (SGH_PROTO(sh, IPPROTO_UDP)) {
         if (SGH_DIRECTION_TS(sh)) {
@@ -1417,20 +1625,12 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             if (mpm_store != NULL) {
                 PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
             }
-            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_L4HDR_TS);
-            if (mpm_store != NULL) {
-                PrefilterUdpHeaderRegister(de_ctx, sh, mpm_store->mpm_ctx);
-            }
         }
         if (SGH_DIRECTION_TC(sh)) {
             mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_UDP_TC);
             if (mpm_store != NULL) {
                 PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
             }
-            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_L4HDR_TC);
-            if (mpm_store != NULL) {
-                PrefilterUdpHeaderRegister(de_ctx, sh, mpm_store->mpm_ctx);
-            }
         }
     } else {
         mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_OTHERIP);
@@ -1439,50 +1639,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         }
     }
 
-    int i = 0;
+    PrepareAppMpms(de_ctx, sh);
-    DetectMpmAppLayerKeyword *a = de_ctx->app_mpms;
+    PreparePktMpms(de_ctx, sh);
-    while (a->reg != NULL) {
-        i++;
-        a++;
-    }
-    if (i == 0)
-        return 0;
-
-    sh->init->app_mpms = SCCalloc(i, sizeof(MpmCtx *));
-    BUG_ON(sh->init->app_mpms == NULL);
-
-    a = de_ctx->app_mpms;
-    while (a->reg != NULL) {
-        if ((a->reg->direction == SIG_FLAG_TOSERVER && SGH_DIRECTION_TS(sh)) ||
-            (a->reg->direction == SIG_FLAG_TOCLIENT && SGH_DIRECTION_TC(sh)))
-        {
-            mpm_store = MpmStorePrepareBufferAppLayer(de_ctx, sh, a);
-            if (mpm_store != NULL) {
-                sh->init->app_mpms[a->reg->id] = mpm_store->mpm_ctx;
-
-                SCLogDebug("a->reg->PrefilterRegister %p mpm_store->mpm_ctx %p",
-                        a->reg->PrefilterRegister, mpm_store->mpm_ctx);
-                SCLogDebug("a %p a->reg->name %s a->reg->PrefilterRegisterWithListId %p "
-                        "mpm_store->mpm_ctx %p", a, a->reg->name,
-                        a->reg->v2.PrefilterRegisterWithListId, mpm_store->mpm_ctx);
-
-                /* if we have just certain types of negated patterns,
-                 * mpm_ctx can be NULL */
-                if (a->reg->v2.PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
-                    BUG_ON(a->reg->v2.PrefilterRegisterWithListId(de_ctx,
-                                sh, mpm_store->mpm_ctx,
-                                a->reg, a->reg->sm_list) != 0);
-                    SCLogDebug("mpm %s %d set up", a->reg->name, a->reg->sm_list);
-                }
-                else if (a->reg->PrefilterRegister && mpm_store->mpm_ctx) {
-                    BUG_ON(a->reg->PrefilterRegister(de_ctx, sh, mpm_store->mpm_ctx) != 0);
-                    SCLogDebug("mpm %s %d set up", a->reg->name, a->reg->sm_list);
-                }
-            }
-        }
-        a++;
-    }
-
     return 0;
 }
 

src/detect-engine-mpm.h

@@ -32,8 +32,9 @@
 
 #include "stream.h"
 
+void DetectMpmInitializePktMpms(DetectEngineCtx *de_ctx);
+int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx);
 void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx);
-void DetectMpmSetupAppMpms(DetectEngineCtx *de_ctx);
 int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx);
 void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx);
 int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx);
@@ -79,20 +80,6 @@ MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh, enum
  */
 int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx);
 
-/** \brief register an app layer keyword for mpm
- *  \param name keyword name
- *  \param direction SIG_FLAG_TOSERVER or SIG_FLAG_TOCLIENT
- *  \param PrefilterRegister Prefilter api registration function
- *
- *  \note direction must be set to either toserver or toclient.
- *        If both are needed, register the keyword twice.
- *  \deprecated since 5.0.0
- */
-void DetectAppLayerMpmRegister(const char *name,
-        int direction, int priority,
-        int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
-            SigGroupHead *sgh, MpmCtx *mpm_ctx)) __attribute__((deprecated));
-
 /** \brief register an app layer keyword for mpm
  *  \param name buffer name
  *  \param direction SIG_FLAG_TOSERVER or SIG_FLAG_TOCLIENT
@@ -109,7 +96,7 @@ void DetectAppLayerMpmRegister2(const char *name,
         int direction, int priority,
         int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
             SigGroupHead *sgh, MpmCtx *mpm_ctx,
-            const DetectMpmAppLayerRegistery *mpm_reg, int list_id),
+            const DetectBufferMpmRegistery *mpm_reg, int list_id),
         InspectionBufferGetDataPtr GetData,
         AppProto alproto, int tx_min_progress);
 void DetectAppLayerMpmRegisterByParentId(
@@ -117,5 +104,21 @@ void DetectAppLayerMpmRegisterByParentId(
         const int id, const int parent_id,
         DetectEngineTransforms *transforms);
 
+void DetectPktMpmRegister(const char *name,
+        int priority,
+        int (*PrefilterRegister)(DetectEngineCtx *de_ctx,
+            SigGroupHead *sgh, MpmCtx *mpm_ctx,
+            const DetectBufferMpmRegistery *mpm_reg, int list_id),
+        InspectionBufferGetPktDataPtr GetData);
+void DetectPktMpmRegisterByParentId(DetectEngineCtx *de_ctx,
+        const int id, const int parent_id,
+        DetectEngineTransforms *transforms);
+
+
+int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx,
+         SigGroupHead *sgh, MpmCtx *mpm_ctx,
+         const DetectBufferMpmRegistery *mpm_reg, int list_id);
+
+
 #endif /* __DETECT_ENGINE_MPM_H__ */
 

src/detect-engine-prefilter.c

@@ -610,22 +610,91 @@ static void PrefilterGenericMpmFree(void *ptr)
 
 int PrefilterGenericMpmRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
     PrefilterMpmCtx *pectx = SCCalloc(1, sizeof(*pectx));
     if (pectx == NULL)
         return -1;
     pectx->list_id = list_id;
-    pectx->GetData = mpm_reg->v2.GetData;
+    pectx->GetData = mpm_reg->app_v2.GetData;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpm,
-        mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+        mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
         pectx, PrefilterGenericMpmFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
     }
     return r;
 }
+
+/* generic mpm for pkt engines */
+
+typedef struct PrefilterMpmPktCtx {
+    int list_id;
+    InspectionBufferGetPktDataPtr GetData;
+    const MpmCtx *mpm_ctx;
+    const DetectEngineTransforms *transforms;
+} PrefilterMpmPktCtx;
+
+/** \brief Generic Mpm prefilter callback
+ *
+ *  \param det_ctx detection engine thread ctx
+ *  \param p packet to inspect
+ *  \param f flow to inspect
+ *  \param txv tx to inspect
+ *  \param pectx inspection context
+ */
+static void PrefilterMpmPkt(DetectEngineThreadCtx *det_ctx,
+        Packet *p, const void *pectx)
+{
+    SCEnter();
+
+    const PrefilterMpmPktCtx *ctx = (const PrefilterMpmPktCtx *)pectx;
+    const MpmCtx *mpm_ctx = ctx->mpm_ctx;
+    SCLogDebug("running on list %d", ctx->list_id);
+
+    InspectionBuffer *buffer = ctx->GetData(det_ctx, ctx->transforms,
+            p, ctx->list_id);
+    if (buffer == NULL)
+        return;
+
+    const uint32_t data_len = buffer->inspect_len;
+    const uint8_t *data = buffer->inspect;
+
+    SCLogDebug("mpm'ing buffer:");
+    //PrintRawDataFp(stdout, data, data_len);
+
+    if (data != NULL && data_len >= mpm_ctx->minlen) {
+        (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
+                &det_ctx->mtcu, &det_ctx->pmq, data, data_len);
+    }
+}
+
+static void PrefilterMpmPktFree(void *ptr)
+{
+    SCFree(ptr);
+}
+
+int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx,
+        SigGroupHead *sgh, MpmCtx *mpm_ctx,
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
+{
+    SCEnter();
+    PrefilterMpmPktCtx *pectx = SCCalloc(1, sizeof(*pectx));
+    if (pectx == NULL)
+        return -1;
+    pectx->list_id = list_id;
+    pectx->GetData = mpm_reg->pkt_v1.GetData;
+    pectx->mpm_ctx = mpm_ctx;
+    pectx->transforms = &mpm_reg->transforms;
+
+    int r = PrefilterAppendEngine(de_ctx, sgh, PrefilterMpmPkt,
+        pectx, PrefilterMpmPktFree, mpm_reg->pname);
+    if (r != 0) {
+        SCFree(pectx);
+    }
+    return r;
+}

src/detect-engine-prefilter.h

@@ -74,6 +74,11 @@ void PrefilterDeinit(DetectEngineCtx *de_ctx);
 
 int PrefilterGenericMpmRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+        const DetectBufferMpmRegistery *mpm_reg, int list_id);
+
+int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx,
+        SigGroupHead *sgh, MpmCtx *mpm_ctx,
+        const DetectBufferMpmRegistery *mpm_reg, int list_id);
+
 
 #endif

src/detect-engine-siggroup.c

@@ -64,6 +64,9 @@ void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid)
     if (sghid->app_mpms != NULL) {
         SCFree(sghid->app_mpms);
     }
+    if (sghid->pkt_mpms != NULL) {
+        SCFree(sghid->pkt_mpms);
+    }
 
     PrefilterFreeEnginesList(sghid->tx_engines);
     PrefilterFreeEnginesList(sghid->pkt_engines);

src/detect-engine.c

@@ -99,6 +99,7 @@ static uint32_t DetectEngineTentantGetIdFromVlanId(const void *ctx, const Packet
 static uint32_t DetectEngineTentantGetIdFromPcap(const void *ctx, const Packet *p);
 
 static DetectEngineAppInspectionEngine *g_app_inspect_engines = NULL;
+static DetectEnginePktInspectionEngine *g_pkt_inspect_engines = NULL;
 
 SCEnumCharMap det_ctx_event_table[ ] = {
 #ifdef UNITTESTS
@@ -121,6 +122,48 @@ SCEnumCharMap det_ctx_event_table[ ] = {
     { NULL,                         -1 },
 };
 
+/** \brief register inspect engine at start up time
+ *
+ *  \note errors are fatal */
+void DetectPktInspectEngineRegister(const char *name,
+        InspectionBufferGetPktDataPtr GetPktData,
+        InspectionBufferPktInspectFunc Callback)
+{
+    DetectBufferTypeRegister(name);
+    const int sm_list = DetectBufferTypeGetByName(name);
+    if (sm_list == -1) {
+        FatalError(SC_ERR_INITIALIZATION,
+            "failed to register inspect engine %s", name);
+    }
+
+    if ((sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
+        (Callback == NULL))
+    {
+        SCLogError(SC_ERR_INVALID_ARGUMENTS, "Invalid arguments");
+        BUG_ON(1);
+    }
+
+    DetectEnginePktInspectionEngine *new_engine = SCCalloc(1, sizeof(*new_engine));
+    if (unlikely(new_engine == NULL)) {
+        FatalError(SC_ERR_INITIALIZATION,
+            "failed to register inspect engine %s: %s", name, strerror(errno));
+    }
+    new_engine->sm_list = sm_list;
+    new_engine->v1.Callback = Callback;
+    new_engine->v1.GetData = GetPktData;
+
+    if (g_pkt_inspect_engines == NULL) {
+        g_pkt_inspect_engines = new_engine;
+    } else {
+        DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
+        while (t->next != NULL) {
+            t = t->next;
+        }
+
+        t->next = new_engine;
+    }
+}
+
 /** \brief register inspect engine at start up time
  *
  *  \note errors are fatal */
@@ -302,6 +345,66 @@ static void DetectAppLayerInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_c
     }
 }
 
+/* copy an inspect engine with transforms to a new list id. */
+static void DetectPktInspectEngineCopy(
+        DetectEngineCtx *de_ctx,
+        int sm_list, int new_list,
+        const DetectEngineTransforms *transforms)
+{
+    const DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
+    while (t) {
+        if (t->sm_list == sm_list) {
+            DetectEnginePktInspectionEngine *new_engine = SCCalloc(1, sizeof(DetectEnginePktInspectionEngine));
+            if (unlikely(new_engine == NULL)) {
+                exit(EXIT_FAILURE);
+            }
+            new_engine->sm_list = new_list;         /* use new list id */
+            new_engine->v1 = t->v1;
+            new_engine->v1.transforms = transforms; /* assign transforms */
+
+            if (de_ctx->pkt_inspect_engines == NULL) {
+                de_ctx->pkt_inspect_engines = new_engine;
+            } else {
+                DetectEnginePktInspectionEngine *list = de_ctx->pkt_inspect_engines;
+                while (list->next != NULL) {
+                    list = list->next;
+                }
+
+                list->next = new_engine;
+            }
+        }
+        t = t->next;
+    }
+}
+
+/* copy inspect engines from global registrations to de_ctx list */
+static void DetectPktInspectEngineCopyListToDetectCtx(DetectEngineCtx *de_ctx)
+{
+    const DetectEnginePktInspectionEngine *t = g_pkt_inspect_engines;
+    while (t) {
+        SCLogDebug("engine %p", t);
+        DetectEnginePktInspectionEngine *new_engine = SCCalloc(1, sizeof(DetectEnginePktInspectionEngine));
+        if (unlikely(new_engine == NULL)) {
+            exit(EXIT_FAILURE);
+        }
+        new_engine->sm_list = t->sm_list;
+        new_engine->v1 = t->v1;
+
+        if (de_ctx->pkt_inspect_engines == NULL) {
+            de_ctx->pkt_inspect_engines = new_engine;
+        } else {
+            DetectEnginePktInspectionEngine *list = de_ctx->pkt_inspect_engines;
+            while (list->next != NULL) {
+                list = list->next;
+            }
+
+            list->next = new_engine;
+        }
+
+        t = t->next;
+    }
+}
+
 /** \internal
  *  \brief append the stream inspection
  *
@@ -375,6 +478,47 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
         SCLogDebug("ptrs[%d] is set", i);
     }
 
+    /* set up pkt inspect engines */
+    const DetectEnginePktInspectionEngine *e = de_ctx->pkt_inspect_engines;
+    while (e != NULL) {
+        SCLogDebug("e %p sm_list %u nlists %u ptrs[] %p", e, e->sm_list, nlists, e->sm_list < nlists ? ptrs[e->sm_list] : NULL);
+        if (e->sm_list < nlists && ptrs[e->sm_list] != NULL) {
+            bool prepend = false;
+
+            DetectEnginePktInspectionEngine *new_engine = SCCalloc(1, sizeof(DetectEnginePktInspectionEngine));
+            if (unlikely(new_engine == NULL)) {
+                exit(EXIT_FAILURE);
+            }
+            if (mpm_list == e->sm_list) {
+                SCLogDebug("%s is mpm", DetectBufferTypeGetNameById(de_ctx, e->sm_list));
+                prepend = true;
+                new_engine->mpm = true;
+            }
+
+            new_engine->sm_list = e->sm_list;
+            new_engine->smd = ptrs[new_engine->sm_list];
+            new_engine->v1 = e->v1;
+            SCLogDebug("sm_list %d new_engine->v1 %p/%p/%p",
+                    new_engine->sm_list, new_engine->v1.Callback,
+                    new_engine->v1.GetData, new_engine->v1.transforms);
+
+            if (s->pkt_inspect == NULL) {
+                s->pkt_inspect = new_engine;
+            } else if (prepend) {
+                new_engine->next = s->pkt_inspect;
+                s->pkt_inspect = new_engine;
+            } else {
+                DetectEnginePktInspectionEngine *a = s->pkt_inspect;
+                while (a->next != NULL) {
+                    a = a->next;
+                }
+                new_engine->next = a->next;
+                a->next = new_engine;
+            }
+        }
+        e = e->next;
+    }
+
     bool head_is_mpm = false;
     uint32_t last_id = DE_STATE_FLAG_BASE;
     const DetectEngineAppInspectionEngine *t = de_ctx->app_inspect_engines;
@@ -518,12 +662,18 @@ void DetectEngineAppInspectionEngineSignatureFree(Signature *s)
 
     DetectEngineAppInspectionEngine *ie = s->app_inspect;
     while (ie) {
-        nlists = MAX(ie->sm_list, nlists);
+        nlists = MAX(ie->sm_list + 1, nlists);
         ie = ie->next;
     }
-    if (nlists == 0)
+    DetectEnginePktInspectionEngine *e = s->pkt_inspect;
+    while (e) {
+        nlists = MAX(e->sm_list + 1, nlists);
+        e = e->next;
+    }
+    if (nlists == 0) {
+        BUG_ON(s->pkt_inspect);
         return;
-    nlists++;
+    }
 
     SigMatchData *ptrs[nlists];
     memset(&ptrs, 0, (nlists * sizeof(SigMatchData *)));
@@ -537,10 +687,16 @@ void DetectEngineAppInspectionEngineSignatureFree(Signature *s)
         SCFree(ie);
         ie = next;
     }
+    e = s->pkt_inspect;
+    while (e) {
+        DetectEnginePktInspectionEngine *next = e->next;
+        ptrs[e->sm_list] = e->smd;
+        SCFree(e);
+        e = next;
+    }
 
     /* free the smds */
-    int i;
+    for (int i = 0; i < nlists; i++)
-    for (i = 0; i < nlists; i++)
     {
         if (ptrs[i] == NULL)
             continue;
@@ -1022,6 +1178,8 @@ static void DetectBufferTypeSetupDetectEngine(DetectEngineCtx *de_ctx)
     PrefilterInit(de_ctx);
     DetectMpmInitializeAppMpms(de_ctx);
     DetectAppLayerInspectEngineCopyListToDetectCtx(de_ctx);
+    DetectMpmInitializePktMpms(de_ctx);
+    DetectPktInspectEngineCopyListToDetectCtx(de_ctx);
 }
 
 static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx)
@@ -1038,12 +1196,24 @@ static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx)
             SCFree(ilist);
             ilist = next;
         }
-        DetectMpmAppLayerRegistery *mlist = de_ctx->app_mpms_list;
+        DetectBufferMpmRegistery *mlist = de_ctx->app_mpms_list;
         while (mlist) {
-            DetectMpmAppLayerRegistery *next = mlist->next;
+            DetectBufferMpmRegistery *next = mlist->next;
             SCFree(mlist);
             mlist = next;
         }
+        DetectEnginePktInspectionEngine *plist = de_ctx->pkt_inspect_engines;
+        while (plist) {
+            DetectEnginePktInspectionEngine *next = plist->next;
+            SCFree(plist);
+            plist = next;
+        }
+        DetectBufferMpmRegistery *pmlist = de_ctx->pkt_mpms_list;
+        while (pmlist) {
+            DetectBufferMpmRegistery *next = pmlist->next;
+            SCFree(pmlist);
+            pmlist = next;
+        }
         PrefilterDeinit(de_ctx);
     }
 }
@@ -1068,6 +1238,8 @@ int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id,
         return -1;
     }
 
+    SCLogDebug("base_map %s", base_map->string);
+
     DetectEngineTransforms t;
     memset(&t, 0, sizeof(t));
     for (int i = 0; i < transform_cnt; i++) {
@@ -1092,10 +1264,16 @@ int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id,
     map->parent_id = base_map->id;
     map->transforms = t;
     map->mpm = base_map->mpm;
+    map->packet = base_map->packet;
     map->SetupCallback = base_map->SetupCallback;
     map->ValidateCallback = base_map->ValidateCallback;
-    DetectAppLayerMpmRegisterByParentId(de_ctx,
+    if (map->packet) {
-            map->id, map->parent_id, &map->transforms);
+        DetectPktMpmRegisterByParentId(de_ctx,
+                map->id, map->parent_id, &map->transforms);
+    } else {
+        DetectAppLayerMpmRegisterByParentId(de_ctx,
+                map->id, map->parent_id, &map->transforms);
+    }
 
     BUG_ON(HashListTableAdd(de_ctx->buffer_type_hash, (void *)map, 0) != 0);
     SCLogDebug("buffer %s registered with id %d, parent %d", map->string, map->id, map->parent_id);
@@ -1108,25 +1286,29 @@ int DetectBufferTypeGetByIdTransforms(DetectEngineCtx *de_ctx, const int id,
         de_ctx->buffer_type_map[map->id] = map;
         de_ctx->buffer_type_map_elements = map->id + 1;
 
-        DetectAppLayerInspectEngineCopy(de_ctx, map->parent_id, map->id,
+        if (map->packet) {
-                &map->transforms);
+            DetectPktInspectEngineCopy(de_ctx, map->parent_id, map->id,
+                    &map->transforms);
+        } else {
+            DetectAppLayerInspectEngineCopy(de_ctx, map->parent_id, map->id,
+                    &map->transforms);
+        }
     }
     return map->id;
 }
 
 /* returns false if no match, true if match */
-static bool DetectEngineInspectRulePacketMatches(
+static int DetectEngineInspectRulePacketMatches(
-    ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
+    DetectEngineThreadCtx *det_ctx,
-    const Signature *s, const SigMatchData *sm_data,
+    const DetectEnginePktInspectionEngine *engine,
-    Flow *f, Packet *p,
+    const Signature *s,
-    uint8_t *alert_flags)
+    Packet *p, uint8_t *_alert_flags)
 {
     SCEnter();
-    BUG_ON(sm_data != s->sm_arrays[DETECT_SM_LIST_MATCH]);
 
     /* run the packet match functions */
     KEYWORD_PROFILING_SET_LIST(det_ctx, DETECT_SM_LIST_MATCH);
-    const SigMatchData *smd = sm_data;
+    const SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_MATCH];
 
     SCLogDebug("running match functions, sm %p", smd);
     while (1) {
@@ -1146,15 +1328,13 @@ static bool DetectEngineInspectRulePacketMatches(
     return true;
 }
 
-static bool DetectEngineInspectRulePayloadMatches(
+static int DetectEngineInspectRulePayloadMatches(
-     ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
+     DetectEngineThreadCtx *det_ctx,
-     const Signature *s, const SigMatchData *sm_data,
+     const DetectEnginePktInspectionEngine *engine,
-     Flow *f, Packet *p,
+     const Signature *s, Packet *p, uint8_t *alert_flags)
-     uint8_t *alert_flags)
 {
     SCEnter();
 
-    BUG_ON(alert_flags == NULL);
     DetectEngineCtx *de_ctx = det_ctx->de_ctx;
 
     KEYWORD_PROFILING_SET_LIST(det_ctx, DETECT_SM_LIST_PMATCH);
@@ -1163,7 +1343,7 @@ static bool DetectEngineInspectRulePayloadMatches(
     if (s->flags & SIG_FLAG_REQUIRE_STREAM) {
         int pmatch = 0;
         if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
-            pmatch = DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, f, p);
+            pmatch = DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, p->flow, p);
             if (pmatch) {
                 det_ctx->flags |= DETECT_ENGINE_THREAD_CTX_STREAM_CONTENT_MATCH;
                 /* Tell the engine that this reassembled stream can drop the
@@ -1183,12 +1363,12 @@ static bool DetectEngineInspectRulePayloadMatches(
             if (!(s->flags & SIG_FLAG_REQUIRE_PACKET) && (p->flags & PKT_STREAM_ADD)) {
                 return false;
             }
-            if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, f, p) != 1) {
+            if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, p->flow, p) != 1) {
                 return false;
             }
         }
     } else {
-        if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, f, p) != 1) {
+        if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, p->flow, p) != 1) {
             return false;
         }
     }
@@ -1201,27 +1381,32 @@ bool DetectEnginePktInspectionRun(ThreadVars *tv,
         uint8_t *alert_flags)
 {
     SCEnter();
+
     for (DetectEnginePktInspectionEngine *e = s->pkt_inspect; e != NULL; e = e->next) {
-        if (e->Callback(tv, det_ctx, s, e->sm_data, f, p, alert_flags) == false) {
+        if (e->v1.Callback(det_ctx, e, s, p, alert_flags) == false) {
             SCLogDebug("sid %u: e %p Callback returned false", s->id, e);
             return false;
         }
         SCLogDebug("sid %u: e %p Callback returned true", s->id, e);
     }
+
     SCLogDebug("sid %u: returning true", s->id);
     return true;
 }
 
+/**
+ * \param data pointer to SigMatchData. Allowed to be NULL.
+ */
 static int DetectEnginePktInspectionAppend(Signature *s,
-        DetectEnginePktInspectionFunc Callback,
+        InspectionBufferPktInspectFunc Callback,
         SigMatchData *data)
 {
     DetectEnginePktInspectionEngine *e = SCCalloc(1, sizeof(*e));
     if (e == NULL)
         return -1;
 
-    e->Callback = Callback;
+    e->v1.Callback = Callback;
-    e->sm_data = data;
+    e->smd = data;
 
     if (s->pkt_inspect == NULL) {
         s->pkt_inspect = e;
@@ -1232,7 +1417,6 @@ static int DetectEnginePktInspectionAppend(Signature *s,
         }
         a->next = e;
     }
-
     return 0;
 }
 
@@ -1241,23 +1425,14 @@ int DetectEnginePktInspectionSetup(Signature *s)
     /* only handle PMATCH here if we're not an app inspect rule */
     if (s->sm_arrays[DETECT_SM_LIST_PMATCH] && (s->init_data->init_flags & SIG_FLAG_INIT_STATE_MATCH) == 0) {
         if (DetectEnginePktInspectionAppend(s, DetectEngineInspectRulePayloadMatches,
-                s->sm_arrays[DETECT_SM_LIST_PMATCH]) < 0)
+                NULL) < 0)
             return -1;
         SCLogDebug("sid %u: DetectEngineInspectRulePayloadMatches appended", s->id);
     }
 
-    if (s->sm_arrays[DETECT_SM_LIST_L4HDR] &&
-            (s->proto.proto[IPPROTO_TCP / 8] & (1 << (IPPROTO_TCP % 8))))
-    {
-        if (DetectEnginePktInspectionAppend(s, DetectEngineInspectRuleTcpHeaderMatches,
-                s->sm_arrays[DETECT_SM_LIST_L4HDR]) < 0)
-            return -1;
-        SCLogDebug("sid %u: DetectEngineInspectRuleTcpHeaderMatches appended", s->id);
-    }
-
     if (s->sm_arrays[DETECT_SM_LIST_MATCH]) {
         if (DetectEnginePktInspectionAppend(s, DetectEngineInspectRulePacketMatches,
-                s->sm_arrays[DETECT_SM_LIST_MATCH]) < 0)
+                NULL) < 0)
             return -1;
         SCLogDebug("sid %u: DetectEngineInspectRulePacketMatches appended", s->id);
     }
@@ -1438,6 +1613,65 @@ int DetectEngineInspectBufferGeneric(
     }
 }
 
+/**
+ * \brief Do the content inspection & validation for a signature
+ *
+ * \param de_ctx Detection engine context
+ * \param det_ctx Detection engine thread context
+ * \param s Signature to inspect
+ * \param p Packet
+ *
+ * \retval 0 no match.
+ * \retval 1 match.
+ */
+int DetectEngineInspectPktBufferGeneric(
+        DetectEngineThreadCtx *det_ctx,
+        const DetectEnginePktInspectionEngine *engine,
+        const Signature *s, Packet *p, uint8_t *_alert_flags)
+{
+    const int list_id = engine->sm_list;
+    SCLogDebug("running inspect on %d", list_id);
+
+    SCLogDebug("list %d transforms %p",
+            engine->sm_list, engine->v1.transforms);
+
+    /* if prefilter didn't already run, we need to consider transformations */
+    const DetectEngineTransforms *transforms = NULL;
+    if (!engine->mpm) {
+        transforms = engine->v1.transforms;
+    }
+
+    const InspectionBuffer *buffer = engine->v1.GetData(det_ctx, transforms, p,
+            list_id);
+    if (unlikely(buffer == NULL)) {
+        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    }
+
+    const uint32_t data_len = buffer->inspect_len;
+    const uint8_t *data = buffer->inspect;
+    const uint64_t offset = 0;
+
+    uint8_t ci_flags = DETECT_CI_FLAGS_START|DETECT_CI_FLAGS_END;
+    ci_flags |= buffer->flags;
+
+    det_ctx->discontinue_matching = 0;
+    det_ctx->buffer_offset = 0;
+    det_ctx->inspection_recursion_counter = 0;
+
+    /* Inspect all the uricontents fetched on each
+     * transaction at the app layer */
+    int r = DetectEngineContentInspection(det_ctx->de_ctx, det_ctx,
+                                          s, engine->smd,
+                                          p, p->flow,
+                                          (uint8_t *)data, data_len, offset, ci_flags,
+                                          DETECT_ENGINE_CONTENT_INSPECTION_MODE_HEADER);
+    if (r == 1) {
+        return DETECT_ENGINE_INSPECT_SIG_MATCH;
+    } else {
+        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    }
+}
+
 
 /* nudge capture loops to wake up */
 static void BreakCapture(void)
@@ -1858,8 +2092,6 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     SCSigSignatureOrderingModuleCleanup(de_ctx);
     ThresholdContextDestroy(de_ctx);
     SigCleanSignatures(de_ctx);
-    SCFree(de_ctx->app_mpms);
-    de_ctx->app_mpms = NULL;
     if (de_ctx->sig_array)
         SCFree(de_ctx->sig_array);
 
@@ -3933,8 +4165,6 @@ const char *DetectSigmatchListEnumToString(enum DetectSigmatchListEnum type)
             return "packet";
         case DETECT_SM_LIST_PMATCH:
             return "packet/stream payload";
-        case DETECT_SM_LIST_L4HDR:
-            return "layer 4 header";
 
         case DETECT_SM_LIST_TMATCH:
             return "tag";

src/detect-engine.h

@@ -124,6 +124,11 @@ int DetectEngineInspectBufferGeneric(
         const Signature *s,
         Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
 
+int DetectEngineInspectPktBufferGeneric(
+        DetectEngineThreadCtx *det_ctx,
+        const DetectEnginePktInspectionEngine *engine,
+        const Signature *s, Packet *p, uint8_t *alert_flags);
+
 /**
  * \brief Registers an app inspection engine.
  *
@@ -142,6 +147,10 @@ void DetectAppLayerInspectEngineRegister2(const char *name,
         InspectEngineFuncPtr2 Callback2,
         InspectionBufferGetDataPtr GetData);
 
+void DetectPktInspectEngineRegister(const char *name,
+        InspectionBufferGetPktDataPtr GetPktData,
+        InspectionBufferPktInspectFunc Callback);
+
 int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature *s);
 void DetectEngineAppInspectionEngineSignatureFree(Signature *s);
 

src/detect-fast-pattern.c

@@ -64,7 +64,7 @@ int FastPatternSupportEnabledForSigMatchList(const DetectEngineCtx *de_ctx,
     if (sm_fp_support_smlist_list == NULL)
         return 0;
 
-    if (list_id == DETECT_SM_LIST_PMATCH || list_id == DETECT_SM_LIST_L4HDR)
+    if (list_id == DETECT_SM_LIST_PMATCH)
         return 1;
 
     return DetectBufferTypeSupportsMpmGetById(de_ctx, list_id);

src/detect-file-data.c

@@ -73,7 +73,7 @@ static int DetectEngineInspectFiledata(
         Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
 int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+        const DetectBufferMpmRegistery *mpm_reg, int list_id);
 
 /**
  * \brief Registration function for keyword: file_data
@@ -517,17 +517,17 @@ static void PrefilterMpmFiledataFree(void *ptr)
 
 int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx));
     if (pectx == NULL)
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFiledata,
-            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+            mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
             pectx, PrefilterMpmFiledataFree, mpm_reg->pname);
 }
 

src/detect-filemagic.c

@@ -90,7 +90,7 @@ static int g_file_magic_buffer_id = 0;
 
 static int PrefilterMpmFilemagicRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+        const DetectBufferMpmRegistery *mpm_reg, int list_id);
 static int DetectEngineInspectFilemagic(
         DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
         const DetectEngineAppInspectionEngine *engine,
@@ -626,17 +626,17 @@ static void PrefilterMpmFilemagicFree(void *ptr)
 
 static int PrefilterMpmFilemagicRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     PrefilterMpmFilemagic *pectx = SCCalloc(1, sizeof(*pectx));
     if (pectx == NULL)
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFilemagic,
-            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+            mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
             pectx, PrefilterMpmFilemagicFree, mpm_reg->pname);
 }
 #ifdef UNITTESTS /* UNITTESTS */

src/detect-filename.c

@@ -65,7 +65,7 @@ static int g_file_name_buffer_id = 0;
 
 static int PrefilterMpmFilenameRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+        const DetectBufferMpmRegistery *mpm_reg, int list_id);
 static int DetectEngineInspectFilename(
         DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
         const DetectEngineAppInspectionEngine *engine,
@@ -472,17 +472,17 @@ static void PrefilterMpmFilenameFree(void *ptr)
 
 static int PrefilterMpmFilenameRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     PrefilterMpmFilename *pectx = SCCalloc(1, sizeof(*pectx));
     if (pectx == NULL)
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFilename,
-            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+            mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
             pectx, PrefilterMpmFilenameFree, mpm_reg->pname);
 }
 

src/detect-http-header-names.c

@@ -212,7 +212,7 @@ static void PrefilterMpmHttpHeaderFree(void *ptr)
 
 static int PrefilterTxHttpRequestHeaderNamesRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
 
@@ -221,10 +221,10 @@ static int PrefilterTxHttpRequestHeaderNamesRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHttpRequestHeaderNames,
-            mpm_reg->v2.alproto, HTP_REQUEST_HEADERS,
+            mpm_reg->app_v2.alproto, HTP_REQUEST_HEADERS,
             pectx, PrefilterMpmHttpHeaderFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
@@ -285,7 +285,7 @@ static void PrefilterTxHttpResponseHeaderNames(DetectEngineThreadCtx *det_ctx,
 
 static int PrefilterTxHttpResponseHeaderNamesRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
 
@@ -294,10 +294,10 @@ static int PrefilterTxHttpResponseHeaderNamesRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHttpResponseHeaderNames,
-            mpm_reg->v2.alproto, HTP_RESPONSE_HEADERS,
+            mpm_reg->app_v2.alproto, HTP_RESPONSE_HEADERS,
             pectx, PrefilterMpmHttpHeaderFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);

src/detect-http-header.c

@@ -285,7 +285,7 @@ static void PrefilterMpmHttpHeaderFree(void *ptr)
 
 static int PrefilterMpmHttpHeaderRequestRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
 
@@ -295,10 +295,10 @@ static int PrefilterMpmHttpHeaderRequestRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpmHttpHeader,
-            mpm_reg->v2.alproto, HTP_REQUEST_HEADERS,
+            mpm_reg->app_v2.alproto, HTP_REQUEST_HEADERS,
             pectx, PrefilterMpmHttpHeaderFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
@@ -311,10 +311,10 @@ static int PrefilterMpmHttpHeaderRequestRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpmHttpTrailer,
-            mpm_reg->v2.alproto, HTP_REQUEST_TRAILER,
+            mpm_reg->app_v2.alproto, HTP_REQUEST_TRAILER,
             pectx, PrefilterMpmHttpHeaderFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
@@ -324,7 +324,7 @@ static int PrefilterMpmHttpHeaderRequestRegister(DetectEngineCtx *de_ctx,
 
 static int PrefilterMpmHttpHeaderResponseRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
 
@@ -334,10 +334,10 @@ static int PrefilterMpmHttpHeaderResponseRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpmHttpHeader,
-            mpm_reg->v2.alproto, HTP_RESPONSE_HEADERS,
+            mpm_reg->app_v2.alproto, HTP_RESPONSE_HEADERS,
             pectx, PrefilterMpmHttpHeaderFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
@@ -350,10 +350,10 @@ static int PrefilterMpmHttpHeaderResponseRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpmHttpTrailer,
-            mpm_reg->v2.alproto, HTP_RESPONSE_TRAILER,
+            mpm_reg->app_v2.alproto, HTP_RESPONSE_TRAILER,
             pectx, PrefilterMpmHttpHeaderFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);

src/detect-http-raw-header.c

@@ -65,10 +65,10 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
 
 static int PrefilterMpmHttpHeaderRawRequestRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+        const DetectBufferMpmRegistery *mpm_reg, int list_id);
 static int PrefilterMpmHttpHeaderRawResponseRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+        const DetectBufferMpmRegistery *mpm_reg, int list_id);
 
 /**
  * \brief Registers the keyword handlers for the "http_raw_header" keyword.
@@ -265,7 +265,7 @@ static void PrefilterMpmHttpHeaderRawFree(void *ptr)
 
 static int PrefilterMpmHttpHeaderRawRequestRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
 
@@ -275,10 +275,10 @@ static int PrefilterMpmHttpHeaderRawRequestRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpmHttpHeaderRaw,
-            mpm_reg->v2.alproto, HTP_REQUEST_HEADERS+1,
+            mpm_reg->app_v2.alproto, HTP_REQUEST_HEADERS+1,
             pectx, PrefilterMpmHttpHeaderRawFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
@@ -291,10 +291,10 @@ static int PrefilterMpmHttpHeaderRawRequestRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpmHttpTrailerRaw,
-            mpm_reg->v2.alproto, HTP_REQUEST_TRAILER+1,
+            mpm_reg->app_v2.alproto, HTP_REQUEST_TRAILER+1,
             pectx, PrefilterMpmHttpHeaderRawFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
@@ -304,7 +304,7 @@ static int PrefilterMpmHttpHeaderRawRequestRegister(DetectEngineCtx *de_ctx,
 
 static int PrefilterMpmHttpHeaderRawResponseRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
 
@@ -314,10 +314,10 @@ static int PrefilterMpmHttpHeaderRawResponseRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpmHttpHeaderRaw,
-            mpm_reg->v2.alproto, HTP_RESPONSE_HEADERS,
+            mpm_reg->app_v2.alproto, HTP_RESPONSE_HEADERS,
             pectx, PrefilterMpmHttpHeaderRawFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
@@ -330,10 +330,10 @@ static int PrefilterMpmHttpHeaderRawResponseRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterMpmHttpTrailerRaw,
-            mpm_reg->v2.alproto, HTP_RESPONSE_TRAILER,
+            mpm_reg->app_v2.alproto, HTP_RESPONSE_TRAILER,
             pectx, PrefilterMpmHttpHeaderRawFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);

src/detect-http-start.c

@@ -213,7 +213,7 @@ static void PrefilterMpmHttpHeaderFree(void *ptr)
 
 static int PrefilterTxHttpRequestStartRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
 
@@ -222,10 +222,10 @@ static int PrefilterTxHttpRequestStartRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHttpRequestStart,
-            mpm_reg->v2.alproto, HTP_REQUEST_HEADERS,
+            mpm_reg->app_v2.alproto, HTP_REQUEST_HEADERS,
             pectx, PrefilterMpmHttpHeaderFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);
@@ -282,7 +282,7 @@ static void PrefilterTxHttpResponseStart(DetectEngineThreadCtx *det_ctx,
 
 static int PrefilterTxHttpResponseStartRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     SCEnter();
 
@@ -291,10 +291,10 @@ static int PrefilterTxHttpResponseStartRegister(DetectEngineCtx *de_ctx,
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     int r = PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxHttpResponseStart,
-            mpm_reg->v2.alproto, HTP_RESPONSE_HEADERS,
+            mpm_reg->app_v2.alproto, HTP_RESPONSE_HEADERS,
             pectx, PrefilterMpmHttpHeaderFree, mpm_reg->pname);
     if (r != 0) {
         SCFree(pectx);

src/detect-krb5-cname.c

@@ -176,17 +176,17 @@ static void PrefilterMpmKrb5NameFree(void *ptr)
 
 static int PrefilterMpmKrb5CNameRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     PrefilterMpmKrb5Name *pectx = SCCalloc(1, sizeof(*pectx));
     if (pectx == NULL)
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxKrb5CName,
-            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+            mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
             pectx, PrefilterMpmKrb5NameFree, mpm_reg->name);
 }
 

src/detect-krb5-sname.c

@@ -176,17 +176,17 @@ static void PrefilterMpmKrb5NameFree(void *ptr)
 
 static int PrefilterMpmKrb5SNameRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     PrefilterMpmKrb5Name *pectx = SCCalloc(1, sizeof(*pectx));
     if (pectx == NULL)
         return -1;
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxKrb5SName,
-            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+            mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
             pectx, PrefilterMpmKrb5NameFree, mpm_reg->name);
 }
 

src/detect-parse.c

@@ -1369,15 +1369,6 @@ void SigFree(Signature *s)
 
     DetectEngineAppInspectionEngineSignatureFree(s);
 
-    if (s->pkt_inspect) {
-        DetectEnginePktInspectionEngine *e = s->pkt_inspect;
-        while (e) {
-            DetectEnginePktInspectionEngine *next = e->next;
-            SCFree(e);
-            e = next;
-        }
-    }
-
     SCFree(s);
 }
 

src/detect-tcphdr.c

@@ -27,6 +27,7 @@
 #include "detect.h"
 #include "detect-parse.h"
 #include "detect-engine.h"
+#include "detect-engine-mpm.h"
 #include "detect-engine-prefilter.h"
 #include "detect-engine-content-inspection.h"
 #include "detect-fast-pattern.h"
@@ -38,10 +39,14 @@ static int DetectTcphdrSetup (DetectEngineCtx *, Signature *, const char *);
 void DetectTcphdrRegisterTests (void);
 #endif
 
+static int g_tcphdr_buffer_id = 0;
+
+static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms, Packet *p, const int list_id);
+
 /**
  * \brief Registration function for tcphdr: keyword
  */
-
 void DetectTcphdrRegister(void)
 {
     sigmatch_table[DETECT_TCPHDR].name = "tcp.hdr";
@@ -52,7 +57,17 @@ void DetectTcphdrRegister(void)
 #ifdef UNITTESTS
     sigmatch_table[DETECT_TCPHDR].RegisterTests = DetectTcphdrRegisterTests;
 #endif
-    SupportFastPatternForSigMatchList(DETECT_SM_LIST_L4HDR, 2);
+
+    g_tcphdr_buffer_id = DetectBufferTypeRegister("tcp.hdr");
+    BUG_ON(g_tcphdr_buffer_id < 0);
+
+    DetectBufferTypeSupportsPacket("tcp.hdr");
+
+    DetectPktMpmRegister("tcp.hdr", 2, PrefilterGenericMpmPktRegister, GetData);
+
+    DetectPktInspectEngineRegister("tcp.hdr", GetData,
+            DetectEngineInspectPktBufferGeneric);
+
     return;
 }
 
@@ -73,94 +88,37 @@ static int DetectTcphdrSetup (DetectEngineCtx *de_ctx, Signature *s, const char
 
     s->flags |= SIG_FLAG_REQUIRE_PACKET;
 
-    if (DetectBufferSetActiveList(s, DETECT_SM_LIST_L4HDR) < 0)
+    if (DetectBufferSetActiveList(s, g_tcphdr_buffer_id) < 0)
         return -1;
 
     return 0;
 }
 
-static void PrefilterTcpHeader(DetectEngineThreadCtx *det_ctx,
+static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
-        Packet *p, const void *pectx)
+        const DetectEngineTransforms *transforms, Packet *p, const int list_id)
 {
     SCEnter();
 
-    uint32_t hlen = TCP_GET_HLEN(p);
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
-    if (((uint8_t *)p->tcph + (ptrdiff_t)hlen) >
+    if (buffer->inspect == NULL) {
-            ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)))
+        uint32_t hlen = TCP_GET_HLEN(p);
-    {
+        if (((uint8_t *)p->tcph + (ptrdiff_t)hlen) >
-        SCLogDebug("data out of range: %p > %p",
+                ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)))
-                ((uint8_t *)p->tcph + (ptrdiff_t)hlen),
+        {
-                ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)));
+            SCLogDebug("data out of range: %p > %p",
-        SCReturn;
+                    ((uint8_t *)p->tcph + (ptrdiff_t)hlen),
+                    ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)));
+            return NULL;
+        }
+
+        const uint32_t data_len = hlen;
+        const uint8_t *data = (const uint8_t *)p->tcph;
+
+        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferApplyTransforms(buffer, transforms);
     }
 
-    const MpmCtx *mpm_ctx = (MpmCtx *)pectx;
+    return buffer;
-    if (hlen < mpm_ctx->minlen)
-        SCReturn;
-
-    (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
-            &det_ctx->mtc, &det_ctx->pmq,
-            (uint8_t *)p->tcph, hlen);
-}
-
-int PrefilterTcpHeaderRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx)
-{
-    return PrefilterAppendEngine(de_ctx, sgh,
-            PrefilterTcpHeader, mpm_ctx, NULL, "tcp.hdr");
-}
-
-/**
- *  \brief Do the content inspection & validation for a signature
- *
- *  \param det_ctx Detection engine thread context
- *  \param s Signature to inspect
- *  \param p Packet
- *
- *  \retval false no match
- *  \retval true match
- */
-bool DetectEngineInspectRuleTcpHeaderMatches(
-     ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
-     const Signature *s, const SigMatchData *sm_data,
-     Flow *f, Packet *p,
-     uint8_t *alert_flags)
-{
-    SCEnter();
-
-    BUG_ON(sm_data == NULL);
-    BUG_ON(sm_data != s->sm_arrays[DETECT_SM_LIST_L4HDR]);
-
-    if (!(PKT_IS_TCP(p))) {
-        SCReturnInt(false);
-    }
-    uint32_t hlen = TCP_GET_HLEN(p);
-    if (((uint8_t *)p->tcph + (ptrdiff_t)hlen) >
-            ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)))
-    {
-        SCLogDebug("data out of range: %p > %p",
-                ((uint8_t *)p->tcph + (ptrdiff_t)hlen),
-                ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)));
-        SCReturnInt(false);
-    }
-
-#ifdef DEBUG
-    det_ctx->payload_persig_cnt++;
-    det_ctx->payload_persig_size += hlen;
-#endif
-    det_ctx->buffer_offset = 0;
-    det_ctx->discontinue_matching = 0;
-    det_ctx->inspection_recursion_counter = 0;
-    det_ctx->replist = NULL;
-
-    int r = DetectEngineContentInspection(det_ctx->de_ctx, det_ctx,
-            s, sm_data,
-            p, NULL, (uint8_t *)p->tcph, hlen, 0,
-            DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_HEADER);
-    if (r == 1) {
-        SCReturnInt(true);
-    }
-    SCReturnInt(false);
 }
 
 #ifdef UNITTESTS

src/detect-tcphdr.h

@@ -24,13 +24,6 @@
 #ifndef _DETECT_TCPHDR_H
 #define _DETECT_TCPHDR_H
 
-int PrefilterTcpHeaderRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx);
-bool DetectEngineInspectRuleTcpHeaderMatches(
-     ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
-     const Signature *s, const SigMatchData *sm_data,
-     Flow *f, Packet *p,
-     uint8_t *alert_flags);
 void DetectTcphdrRegister(void);
 
 #endif	/* _DETECT_TCPHDR_H */

src/detect-tls-certs.c

@@ -66,7 +66,7 @@ static int DetectEngineInspectTlsCerts(DetectEngineCtx *de_ctx,
 	uint64_t tx_id);
 static int PrefilterMpmTlsCertsRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+        const DetectBufferMpmRegistery *mpm_reg, int list_id);
 
 static int g_tls_certs_buffer_id = 0;
 
@@ -240,7 +240,7 @@ static void PrefilterMpmTlsCertsFree(void *ptr)
 
 static int PrefilterMpmTlsCertsRegister(DetectEngineCtx *de_ctx,
         SigGroupHead *sgh, MpmCtx *mpm_ctx,
-        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+        const DetectBufferMpmRegistery *mpm_reg, int list_id)
 {
     PrefilterMpmTlsCerts *pectx = SCCalloc(1, sizeof(*pectx));
     if (pectx == NULL)
@@ -248,10 +248,10 @@ static int PrefilterMpmTlsCertsRegister(DetectEngineCtx *de_ctx,
 
     pectx->list_id = list_id;
     pectx->mpm_ctx = mpm_ctx;
-    pectx->transforms = &mpm_reg->v2.transforms;
+    pectx->transforms = &mpm_reg->transforms;
 
     return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxTlsCerts,
-            mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+            mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress,
             pectx, PrefilterMpmTlsCertsFree, mpm_reg->name);
 }
 

src/detect-udphdr.c

@@ -27,6 +27,7 @@
 #include "detect.h"
 #include "detect-parse.h"
 #include "detect-engine.h"
+#include "detect-engine-mpm.h"
 #include "detect-engine-prefilter.h"
 #include "detect-engine-content-inspection.h"
 #include "detect-fast-pattern.h"
@@ -38,8 +39,12 @@ static int DetectUdphdrSetup (DetectEngineCtx *, Signature *, const char *);
 void DetectUdphdrRegisterTests (void);
 #endif
 
+static int g_udphdr_buffer_id = 0;
+
+static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms, Packet *p, const int list_id);
 /**
- * \brief Registration function for tcphdr: keyword
+ * \brief Registration function for udphdr: keyword
  */
 
 void DetectUdphdrRegister(void)
@@ -52,7 +57,16 @@ void DetectUdphdrRegister(void)
 #ifdef UNITTESTS
     sigmatch_table[DETECT_UDPHDR].RegisterTests = DetectUdphdrRegisterTests;
 #endif
-    SupportFastPatternForSigMatchList(DETECT_SM_LIST_L4HDR, 2);
+
+    g_udphdr_buffer_id = DetectBufferTypeRegister("udp.hdr");
+    BUG_ON(g_udphdr_buffer_id < 0);
+
+    DetectBufferTypeSupportsPacket("udp.hdr");
+
+    DetectPktMpmRegister("udp.hdr", 2, PrefilterGenericMpmPktRegister, GetData);
+
+    DetectPktInspectEngineRegister("udp.hdr", GetData,
+            DetectEngineInspectPktBufferGeneric);
     return;
 }
 
@@ -73,92 +87,36 @@ static int DetectUdphdrSetup (DetectEngineCtx *de_ctx, Signature *s, const char
 
     s->flags |= SIG_FLAG_REQUIRE_PACKET;
 
-    if (DetectBufferSetActiveList(s, DETECT_SM_LIST_L4HDR) < 0)
+    if (DetectBufferSetActiveList(s, g_udphdr_buffer_id) < 0)
         return -1;
 
     return 0;
 }
 
-static void PrefilterUdpHeader(DetectEngineThreadCtx *det_ctx,
+static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
-        Packet *p, const void *pectx)
+        const DetectEngineTransforms *transforms, Packet *p, const int list_id)
-{
-    SCEnter();
-
-    if (((uint8_t *)p->udph + (ptrdiff_t)UDP_HEADER_LEN) >
-            ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)))
-    {
-        SCLogDebug("data out of range: %p > %p",
-                ((uint8_t *)p->udph + (ptrdiff_t)UDP_HEADER_LEN),
-                ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)));
-        SCReturn;
-    }
-
-    const MpmCtx *mpm_ctx = (MpmCtx *)pectx;
-    if (UDP_HEADER_LEN < mpm_ctx->minlen)
-        SCReturn;
-
-    (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
-            &det_ctx->mtc, &det_ctx->pmq,
-            (uint8_t *)p->udph, UDP_HEADER_LEN);
-}
-
-int PrefilterUdpHeaderRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx)
-{
-    return PrefilterAppendEngine(de_ctx, sgh,
-            PrefilterUdpHeader, mpm_ctx, NULL, "udp.hdr");
-}
-
-/**
- *  \brief Do the content inspection & validation for a signature
- *
- *  \param det_ctx Detection engine thread context
- *  \param s Signature to inspect
- *  \param p Packet
- *
- *  \retval false no match
- *  \retval true match
- */
-bool DetectEngineInspectRuleUdpHeaderMatches(
-     ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
-     const Signature *s, const SigMatchData *sm_data,
-     Flow *f, Packet *p,
-     uint8_t *alert_flags)
 {
     SCEnter();
 
-    BUG_ON(sm_data == NULL);
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
-    BUG_ON(sm_data != s->sm_arrays[DETECT_SM_LIST_L4HDR]);
+    if (buffer->inspect == NULL) {
-
+        if (((uint8_t *)p->udph + (ptrdiff_t)UDP_HEADER_LEN) >
-    if (!(PKT_IS_UDP(p))) {
+                ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)))
-        SCReturnInt(false);
+        {
-    }
+            SCLogDebug("data out of range: %p > %p",
-    if (((uint8_t *)p->udph + (ptrdiff_t)UDP_HEADER_LEN) >
+                    ((uint8_t *)p->udph + (ptrdiff_t)UDP_HEADER_LEN),
-            ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)))
+                    ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)));
-    {
+            return NULL;
-        SCLogDebug("data out of range: %p > %p",
+        }
-                ((uint8_t *)p->udph + (ptrdiff_t)UDP_HEADER_LEN),
+
-                ((uint8_t *)GET_PKT_DATA(p) + (ptrdiff_t)GET_PKT_LEN(p)));
+        const uint32_t data_len = UDP_HEADER_LEN;
-        SCReturnInt(false);
+        const uint8_t *data = (const uint8_t *)p->udph;
+
+        InspectionBufferSetup(buffer, data, data_len);
+        InspectionBufferApplyTransforms(buffer, transforms);
     }
 
-#ifdef DEBUG
+    return buffer;
-    det_ctx->payload_persig_cnt++;
-    det_ctx->payload_persig_size += UDP_HEADER_LEN;
-#endif
-    det_ctx->buffer_offset = 0;
-    det_ctx->discontinue_matching = 0;
-    det_ctx->inspection_recursion_counter = 0;
-    det_ctx->replist = NULL;
-
-    int r = DetectEngineContentInspection(det_ctx->de_ctx, det_ctx,
-            s, sm_data,
-            p, NULL, (uint8_t *)p->udph, UDP_HEADER_LEN, 0,
-            DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_HEADER);
-    if (r == 1) {
-        SCReturnInt(true);
-    }
-    SCReturnInt(false);
 }
 
 #ifdef UNITTESTS

src/detect-udphdr.h

@@ -24,13 +24,6 @@
 #ifndef _DETECT_UDPHDR_H
 #define _DETECT_UDPHDR_H
 
-int PrefilterUdpHeaderRegister(DetectEngineCtx *de_ctx,
-        SigGroupHead *sgh, MpmCtx *mpm_ctx);
-bool DetectEngineInspectRuleUdpHeaderMatches(
-     ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
-     const Signature *s, const SigMatchData *sm_data,
-     Flow *f, Packet *p,
-     uint8_t *alert_flags);
 void DetectUdphdrRegister(void);
 
 #endif	/* _DETECT_UDPHDR_H */

src/detect.h

@@ -91,7 +91,6 @@ struct SCSigSignatureWrapper_;
 enum DetectSigmatchListEnum {
     DETECT_SM_LIST_MATCH = 0,
     DETECT_SM_LIST_PMATCH,
-    DETECT_SM_LIST_L4HDR,
 
     /* base64_data keyword uses some hardcoded logic so consider
      * built-in
@@ -439,20 +438,33 @@ typedef struct DetectBufferType_ {
     DetectEngineTransforms transforms;
 } DetectBufferType;
 
-/* Function pointer for the pkt inspect engines. Goal for this
+struct DetectEnginePktInspectionEngine;
- * function will be to process the complete list of conditions
+
- * passed to it through 'sm_data'. */
+/**
-typedef bool (*DetectEnginePktInspectionFunc)(ThreadVars *,
+ *  \param alert_flags[out] for setting PACKET_ALERT_FLAG_*
+ */
+typedef int (*InspectionBufferPktInspectFunc)(
         struct DetectEngineThreadCtx_ *,
+        const struct DetectEnginePktInspectionEngine *engine,
         const struct Signature_ *s,
-        const struct SigMatchData_ *sm_data,
+        Packet *p, uint8_t *alert_flags);
-        Flow *f,
+
-        Packet *p,
+/** callback for getting the buffer we need to prefilter/inspect */
-        uint8_t *alert_flags);
+typedef InspectionBuffer *(*InspectionBufferGetPktDataPtr)(
+        struct DetectEngineThreadCtx_ *det_ctx,
+        const DetectEngineTransforms *transforms,
+        Packet *p, const int list_id);
 
 typedef struct DetectEnginePktInspectionEngine {
-    DetectEnginePktInspectionFunc Callback;
+    SigMatchData *smd;
-    SigMatchData *sm_data;
+    uint16_t mpm:1;
+    uint16_t sm_list:15;
+    struct {
+        InspectionBufferGetPktDataPtr GetData;
+        InspectionBufferPktInspectFunc Callback;
+        /** pointer to the transforms in the 'DetectBuffer entry for this list */
+        const DetectEngineTransforms *transforms;
+    } v1;
     struct DetectEnginePktInspectionEngine *next;
 } DetectEnginePktInspectionEngine;
 
@@ -586,39 +598,48 @@ typedef struct Signature_ {
     struct Signature_ *next;
 } Signature;
 
+enum DetectBufferMpmType {
+    DETECT_BUFFER_MPM_TYPE_PKT,
+    DETECT_BUFFER_MPM_TYPE_APP,
+    /* must be last */
+    DETECT_BUFFER_MPM_TYPE_SIZE,
+};
+
 /** \brief one time registration of keywords at start up */
-typedef struct DetectMpmAppLayerRegistery_ {
+typedef struct DetectBufferMpmRegistery_ {
     const char *name;
     char pname[32];             /**< name used in profiling */
     int direction;              /**< SIG_FLAG_TOSERVER or SIG_FLAG_TOCLIENT */
     int sm_list;
-
-    int (*PrefilterRegister)(struct DetectEngineCtx_ *de_ctx,
-            struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx);
-
     int priority;
+    int id;                     /**< index into this array and result arrays */
+    enum DetectBufferMpmType type;
+    int sgh_mpm_context;
 
-    struct {
+    int (*PrefilterRegisterWithListId)(struct DetectEngineCtx_ *de_ctx,
-        int (*PrefilterRegisterWithListId)(struct DetectEngineCtx_ *de_ctx,
+            struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx,
-                struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx,
+            const struct DetectBufferMpmRegistery_ *mpm_reg, int list_id);
-                const struct DetectMpmAppLayerRegistery_ *mpm_reg, int list_id);
+    DetectEngineTransforms transforms;
-        InspectionBufferGetDataPtr GetData;
-        AppProto alproto;
-        int tx_min_progress;
-        DetectEngineTransforms transforms;
-    } v2;
 
-    int id;                     /**< index into this array and result arrays */
+    union {
+        /* app-layer matching: use if type == DETECT_BUFFER_MPM_TYPE_APP */
+        struct {
+            InspectionBufferGetDataPtr GetData;
+            AppProto alproto;
+            int tx_min_progress;
+        } app_v2;
 
-    struct DetectMpmAppLayerRegistery_ *next;
+        /* pkt matching: use if type == DETECT_BUFFER_MPM_TYPE_PKT */
-} DetectMpmAppLayerRegistery;
+        struct {
+            int (*PrefilterRegisterWithListId)(struct DetectEngineCtx_ *de_ctx,
+                    struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx,
+                    const struct DetectBufferMpmRegistery_ *mpm_reg, int list_id);
+            InspectionBufferGetPktDataPtr GetData;
+        } pkt_v1;
+    };
 
-/** \brief structure for storing per detect engine mpm keyword settings
+    struct DetectBufferMpmRegistery_ *next;
- */
+} DetectBufferMpmRegistery;
-typedef struct DetectMpmAppLayerKeyword_ {
-    const DetectMpmAppLayerRegistery *reg;
-    int32_t sgh_mpm_context;    /**< mpm factory id */
-} DetectMpmAppLayerKeyword;
 
 typedef struct DetectReplaceList_ {
     struct DetectContentData_ *cd;
@@ -829,7 +850,6 @@ typedef struct DetectEngineCtx_ {
     int32_t sgh_mpm_context_proto_udp_packet;
     int32_t sgh_mpm_context_proto_other_packet;
     int32_t sgh_mpm_context_stream;
-    int32_t sgh_mpm_context_l4_header;
 
     /* the max local id used amongst all sigs */
     int32_t byte_extract_max_local_id;
@@ -907,17 +927,15 @@ typedef struct DetectEngineCtx_ {
     /* list with app inspect engines. Both the start-time registered ones and
      * the rule-time registered ones. */
     DetectEngineAppInspectionEngine *app_inspect_engines;
-    DetectMpmAppLayerRegistery *app_mpms_list;
+    DetectBufferMpmRegistery *app_mpms_list;
     uint32_t app_mpms_list_cnt;
+    DetectEnginePktInspectionEngine *pkt_inspect_engines;
+    DetectBufferMpmRegistery *pkt_mpms_list;
+    uint32_t pkt_mpms_list_cnt;
 
     uint32_t prefilter_id;
     HashListTable *prefilter_hash_table;
 
-    /** table with mpms and their registration function
-     *  \todo we only need this at init, so perhaps this
-     *        can move to a DetectEngineCtx 'init' struct */
-    DetectMpmAppLayerKeyword *app_mpms;
-
     /** time of last ruleset reload */
     struct timeval last_reload;
 
@@ -1225,8 +1243,6 @@ enum MpmBuiltinBuffers {
     MPMB_UDP_TS,
     MPMB_UDP_TC,
     MPMB_OTHERIP,
-    MPMB_L4HDR_TS,
-    MPMB_L4HDR_TC,
     MPMB_MAX,
 };
 
@@ -1307,6 +1323,7 @@ typedef struct SigGroupHeadInitData_ {
     int whitelist;          /**< try to make this group a unique one */
 
     MpmCtx **app_mpms;
+    MpmCtx **pkt_mpms;
 
     PrefilterEngineList *pkt_engines;
     PrefilterEngineList *payload_engines;