detect: move non-pf rules into special prefilter engines

Instead of having a per detection engine list of rule that couldn't be
prefiltered, put those into special "prefilter" engines.

For packet and frame rules this doesn't change much, it just removes
some hard coded logic from the detect engine.

For the packet non-prefilter rules in the "non-prefilter" special prefilter
engine, add additional filtering for the packet variant. It can prefilter on
alproto, dsize and dest port.

The frame non-prefilter rules are added to a single engine, that per
rule checks the alproto and the type.

For app-layer, there is an engine per progress value, per app-layer
protocol and per direction. This hooks app-layer non-prefilter rules
into the app inspect logic at the correct "progress" hook.

e.g. a rule like
        dns.query; bsize:1;

Negated MPM rules will also fall into this category:
        dns.query; content:!"abc";

Are part of a special "generic list" app engine for dns, at the
same progress hook as `dns.query`.

This all results in a lot fewer checks:

previous:

  --------------------------------------------------------------------------
  Date: 1/29/2025 -- 10:22:25. Sorted by: number of checks.
  --------------------------------------------------------------------------
   Num      Rule         Gid      Rev      Ticks        %      Checks   Matches  Max Ticks   Avg Ticks   Avg Match   Avg No Match
  -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- --------------
  1        20           1        0        181919672    11.85  588808   221      60454       308.96      2691.46     308.07
  2        50           1        0        223455914    14.56  453104   418      61634       493.17      3902.59     490.02
  3        60           1        0        185990683    12.12  453104   418      60950       410.48      1795.40     409.20
  4        51           1        0        192436011    12.54  427028   6084     61223       450.64      2749.12     417.42
  5        61           1        0        180401533    11.75  427028   6084     61093       422.46      2177.04     397.10
  6        70           1        0        153899099    10.03  369836   0        61282       416.13      0.00        416.13
  7        71           1        0        123389405    8.04   369836   12833    44921       333.63      2430.23     258.27
  8        41           1        0        63889876     4.16   155824   12568    39138       410.01      1981.97     272.10
  9        40           1        0        64149724     4.18   155818   210      39792       411.70      4349.57     406.38
  10       10           1        0        70848850     4.62   65558    0        39544       1080.70     0.00        1080.70
  11       11           1        0        94743878     6.17   65558    32214    60547       1445.19     2616.14     313.92

this commit:

  --------------------------------------------------------------------------
  Date: 1/29/2025 -- 10:15:46. Sorted by: number of checks.
  --------------------------------------------------------------------------
   Num      Rule         Gid      Rev      Ticks        %      Checks   Matches  Max Ticks   Avg Ticks   Avg Match   Avg No Match
  -------- ------------ -------- -------- ------------ ------ -------- -------- ----------- ----------- ----------- --------------
  1        50           1        0        138776766    19.23  95920    418      167584      1446.80     3953.11     1435.83
  2        60           1        0        97988084     13.58  95920    418      182817      1021.56     1953.63     1017.48
  3        51           1        0        105318318    14.60  69838    6084     65649       1508.04     2873.38     1377.74
  4        61           1        0        89571260     12.41  69838    6084     164632      1282.56     2208.41     1194.20
  5        11           1        0        91132809     12.63  32779    32214    373569      2780.22     2785.58     2474.45
  6        10           1        0        66095303     9.16   32779    0        56704       2016.39     0.00        2016.39
  7        70           1        0        48107573     6.67   12928    0        42832       3721.19     0.00        3721.19
  8        71           1        0        32308792     4.48   12928    12833    39565       2499.13     2510.05     1025.09
  9        41           1        0        25546837     3.54   12886    12470    41479       1982.53     1980.84     2033.05
  10       40           1        0        26069992     3.61   12886    210      38495       2023.13     4330.05     1984.91
  11       20           1        0        639025       0.09   221      221      14750       2891.52     2891.52     0.00

src/app-layer-frames.h

@@ -26,6 +26,8 @@
 
 #include "rust.h"
 
+/** special value for matching any type */
+#define FRAME_ANY_TYPE 62
 /** max 63 to fit the 64 bit per protocol space */
 #define FRAME_STREAM_TYPE 63
 

src/decode.h

@@ -95,6 +95,9 @@ enum PktSrcEnum {
 
 #include "util-validate.h"
 
+/* for now a uint8_t is enough -- here in decode as it's part of the packet */
+#define SignatureMask uint8_t
+
 /* forward declarations */
 struct DetectionEngineThreadCtx_;
 typedef struct AppLayerThreadCtx_ AppLayerThreadCtx;
@@ -509,6 +512,7 @@ typedef struct Packet_
     /* coccinelle: Packet:flowflags:FLOW_PKT_ */
 
     uint8_t app_update_direction; // enum StreamUpdateDir
+    SignatureMask sig_mask;
 
     /* Pkt Flags */
     uint32_t flags;

src/detect-engine-build.c

@@ -1983,8 +1983,6 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx)
 
         PrefilterSetupRuleGroup(de_ctx, sgh);
 
-        SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh);
-
         sgh->id = idx;
         cnt++;
     }
@@ -1995,7 +1993,7 @@ int SigPrepareStage4(DetectEngineCtx *de_ctx)
     if (de_ctx->decoder_event_sgh != NULL) {
         /* no need to set filestore count here as that would make a
          * signature not decode event only. */
-        SigGroupHeadBuildNonPrefilterArray(de_ctx, de_ctx->decoder_event_sgh);
+        PrefilterSetupRuleGroup(de_ctx, de_ctx->decoder_event_sgh);
     }
 
     int dump_grouping = 0;

src/detect-engine-frame.c

@@ -76,8 +76,9 @@ void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead
     SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt);
     PrefilterEngine *engine = sgh->frame_engines;
     do {
-        BUG_ON(engine->alproto == ALPROTO_UNKNOWN);
-        if (engine->alproto == alproto && engine->ctx.frame_type == frame->type) {
+        if ((engine->alproto == alproto || engine->alproto == ALPROTO_UNKNOWN) &&
+                (engine->ctx.frame_type == frame->type ||
+                        engine->ctx.frame_type == FRAME_ANY_TYPE)) {
             SCLogDebug("frame %p engine %p", frame, engine);
             PREFILTER_PROFILING_START(det_ctx);
             engine->cb.PrefilterFrame(det_ctx, engine->pectx, p, frames, frame);

src/detect-engine-prefilter.c

@@ -51,12 +51,14 @@
 #include "detect-engine-prefilter.h"
 #include "detect-engine-mpm.h"
 #include "detect-engine-frame.h"
+#include "detect-engine-uint.h"
 
 #include "app-layer-parser.h"
 #include "app-layer-htp.h"
 
 #include "util-profiling.h"
 #include "util-validate.h"
+#include "util-hash-string.h"
 
 static int PrefilterStoreGetId(DetectEngineCtx *de_ctx,
         const char *name, void (*FreeFunc)(void *));
@@ -490,7 +492,536 @@ static int PrefilterSetupRuleGroupSortHelper(const void *a, const void *b)
     }
 }
 
-void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
+/** prefilter engine data for the non-prefilter engine for the prefilter API */
+struct PrefilterNonPFDataSig {
+    uint32_t sid : 30;
+    uint32_t type : 2; /**< type for `value` field below: 0:alproto 1:dport 2:dsize */
+    uint16_t value;
+    /* since we have 2 more bytes available due to padding, we can add some additional
+     * filters here. */
+    union {
+        struct {
+            SignatureMask sig_mask;
+        } pkt;
+        struct {
+            /* filter for frame type */
+            uint8_t type;
+        } frame;
+        struct {
+            uint8_t foo; // TODO unused
+        } app;
+    };
+};
+
+struct PrefilterNonPFData {
+    uint32_t size;
+    struct PrefilterNonPFDataSig array[];
+};
+
+struct PrefilterNonPFDataTx {
+    uint32_t size;
+    uint32_t array[];
+};
+
+/** \internal
+ *  \brief wrapper for use in APIs */
+static void PrefilterNonPFDataFree(void *data)
+{
+    SCFree(data);
+}
+
+static void PrefilterTxNonPF(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, Flow *f,
+        void *tx, const uint64_t tx_id, const AppLayerTxData *tx_data, const uint8_t flags)
+{
+    const struct PrefilterNonPFDataTx *data = (const struct PrefilterNonPFDataTx *)pectx;
+    SCLogDebug("adding %u sids", data->size);
+    PrefilterAddSids(&det_ctx->pmq, data->array, data->size);
+}
+
+#ifdef NONPF_PKT_STATS
+static thread_local uint64_t prefilter_pkt_nonpf_called = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_mask_fail = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_alproto_fail = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_dsize_fail = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_dport_fail = 0;
+static thread_local uint64_t prefilter_pkt_nonpf_sids = 0;
+#define NONPF_PKT_STATS_INCR(s) (s)++
+#else
+#define NONPF_PKT_STATS_INCR(s)
+#endif
+
+void PrefilterPktNonPFStatsDump(void)
+{
+#ifdef NONPF_PKT_STATS
+    SCLogDebug("prefilter non-pf: called:%" PRIu64 ", mask_fail:%" PRIu64 ", alproto fail:%" PRIu64
+               ", dport fail:%" PRIu64 ", dsize fail:%" PRIu64 ", sids:%" PRIu64
+               ", avg sids:%" PRIu64,
+            prefilter_pkt_nonpf_called, prefilter_pkt_nonpf_mask_fail,
+            prefilter_pkt_nonpf_alproto_fail, prefilter_pkt_nonpf_dport_fail,
+            prefilter_pkt_nonpf_dsize_fail, prefilter_pkt_nonpf_sids,
+            prefilter_pkt_nonpf_called ? prefilter_pkt_nonpf_sids / prefilter_pkt_nonpf_called : 0);
+#endif
+}
+
+static void PrefilterPktNonPF(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx)
+{
+    const uint16_t alproto = p->flow ? p->flow->alproto : ALPROTO_UNKNOWN;
+    const SignatureMask mask = p->sig_mask;
+    const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx;
+    SCLogDebug("adding %u sids", data->size);
+    NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_called);
+    for (uint32_t i = 0; i < data->size; i++) {
+        const struct PrefilterNonPFDataSig *ds = &data->array[i];
+        const SignatureMask rule_mask = ds->pkt.sig_mask;
+        if ((rule_mask & mask) == rule_mask) {
+            switch (ds->type) {
+                case 0:
+                    if (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto)) {
+                        const uint32_t sid = ds->sid;
+                        PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids);
+                    } else {
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_alproto_fail);
+                    }
+                    break;
+                case 1:
+                    if (ds->value == p->dp) {
+                        const uint32_t sid = ds->sid;
+                        PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids);
+                    } else {
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_dport_fail);
+                    }
+                    break;
+                case 2:
+                    if (ds->value == p->payload_len) {
+                        const uint32_t sid = ds->sid;
+                        PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_sids);
+                    } else {
+                        NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_dsize_fail);
+                    }
+                    break;
+            }
+        } else {
+            NONPF_PKT_STATS_INCR(prefilter_pkt_nonpf_mask_fail);
+        }
+    }
+}
+
+/** \internal
+ *  \brief engine to select the non-prefilter rules for frames
+ *  Checks the alproto and type as well.
+ *  Direction needs no checking as the rule groups are per direction. */
+static void PrefilterFrameNonPF(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p,
+        const Frames *frames, const Frame *frame)
+{
+    DEBUG_VALIDATE_BUG_ON(p->flow == NULL);
+    const uint16_t alproto = p->flow->alproto;
+    const struct PrefilterNonPFData *data = (const struct PrefilterNonPFData *)pectx;
+    SCLogDebug("adding %u sids", data->size);
+    for (uint32_t i = 0; i < data->size; i++) {
+        const struct PrefilterNonPFDataSig *ds = &data->array[i];
+        if (ds->frame.type == frame->type &&
+                (ds->value == ALPROTO_UNKNOWN || AppProtoEquals(ds->value, alproto))) {
+            const uint32_t sid = ds->sid;
+            PrefilterAddSids(&det_ctx->pmq, &sid, 1);
+        }
+    }
+}
+
+/* helper funcs for the non prefilter names hash */
+
+static uint32_t NonPFNamesHash(HashTable *h, void *data, uint16_t _len)
+{
+    const char *str = data;
+    return StringHashDjb2((const uint8_t *)str, (uint16_t)strlen(str)) % h->array_size;
+}
+
+static char NonPFNamesCompare(void *data1, uint16_t _len1, void *data2, uint16_t len2)
+{
+    const char *s1 = data1;
+    const char *s2 = data2;
+    return StringHashCompareFunc(data1, (uint16_t)strlen(s1), data2, (uint16_t)strlen(s2));
+}
+
+static void NonPFNamesFree(void *data)
+{
+    SCFree(data);
+}
+
+/* helper funcs for assembling non-prefilter engines */
+
+struct TxNonPFData {
+    AppProto alproto;
+    int dir;      /**< 0: toserver, 1: toclient */
+    int progress; /**< progress state value to register at */
+    uint32_t sigs_cnt;
+    struct PrefilterNonPFDataSig *sigs;
+    const char *engine_name; /**< pointer to name owned by DetectEngineCtx::non_pf_engine_names */
+};
+
+static uint32_t TxNonPFHash(HashListTable *h, void *data, uint16_t _len)
+{
+    struct TxNonPFData *d = data;
+    return (d->alproto + d->progress + d->dir) % h->array_size;
+}
+
+static char TxNonPFCompare(void *data1, uint16_t _len1, void *data2, uint16_t len2)
+{
+    struct TxNonPFData *d1 = data1;
+    struct TxNonPFData *d2 = data2;
+    return d1->alproto == d2->alproto && d1->progress == d2->progress && d1->dir == d2->dir;
+}
+
+static void TxNonPFFree(void *data)
+{
+    struct TxNonPFData *d = data;
+    SCFree(d->sigs);
+    SCFree(d);
+}
+
+static int TxNonPFAddSig(DetectEngineCtx *de_ctx, HashListTable *tx_engines_hash,
+        const AppProto alproto, const int dir, const int16_t progress, const char *name,
+        const Signature *s)
+{
+    const uint32_t max_sids = DetectEngineGetMaxSigId(de_ctx);
+
+    struct TxNonPFData lookup = {
+        .alproto = alproto,
+        .dir = dir,
+        .progress = progress,
+        .sigs_cnt = 0,
+        .sigs = NULL,
+        .engine_name = NULL,
+    };
+    struct TxNonPFData *e = HashListTableLookup(tx_engines_hash, &lookup, 0);
+    if (e != NULL) {
+        bool found = false;
+        // avoid adding same sid multiple times
+        for (uint32_t y = 0; y < e->sigs_cnt; y++) {
+            if (e->sigs[y].sid == s->num) {
+                found = true;
+                break;
+            }
+        }
+        if (!found) {
+            BUG_ON(e->sigs_cnt == max_sids);
+            e->sigs[e->sigs_cnt].sid = s->num;
+            e->sigs[e->sigs_cnt].value = alproto;
+            e->sigs_cnt++;
+        }
+        return 0;
+    }
+
+    struct TxNonPFData *add = SCCalloc(1, sizeof(*add));
+    if (add == NULL) {
+        return -1;
+    }
+    add->dir = dir;
+    add->alproto = alproto;
+    add->progress = progress;
+    add->sigs = SCCalloc(max_sids, sizeof(struct PrefilterNonPFDataSig));
+    if (add->sigs == NULL) {
+        SCFree(add);
+        return -1;
+    }
+    add->sigs_cnt = 0;
+    add->sigs[add->sigs_cnt].sid = s->num;
+    add->sigs[add->sigs_cnt].value = alproto;
+    add->sigs_cnt++;
+
+    char engine_name[128];
+    snprintf(engine_name, sizeof(engine_name), "%s:%s:non_pf:%s", AppProtoToString(alproto), name,
+            dir == 0 ? "toserver" : "toclient");
+    char *engine_name_heap = SCStrdup(engine_name);
+    if (engine_name_heap == NULL) {
+        SCFree(add->sigs);
+        SCFree(add);
+        return -1;
+    }
+    int result = HashTableAdd(
+            de_ctx->non_pf_engine_names, engine_name_heap, (uint16_t)strlen(engine_name_heap));
+    if (result != 0) {
+        SCFree(add->sigs);
+        SCFree(add);
+        return -1;
+    }
+
+    add->engine_name = engine_name_heap;
+    SCLogDebug("engine_name_heap %s", engine_name_heap);
+
+    int ret = HashListTableAdd(tx_engines_hash, add, 0);
+    if (ret != 0) {
+        SCFree(add->sigs);
+        SCFree(add);
+        return -1;
+    }
+
+    return 0;
+}
+
+/** \internal
+ *  \brief setup non-prefilter rules in special "non-prefilter" engines that are registered in the
+ * prefilter logic.
+ *
+ *  \retval 0 ok
+ *  \retval -1 error
+ */
+static int SetupNonPrefilter(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
+{
+    const uint32_t max_sids = DetectEngineGetMaxSigId(de_ctx);
+    SCLogDebug("max_sids %u", max_sids);
+    struct PrefilterNonPFDataSig *pkt_non_pf_array = SCCalloc(max_sids, sizeof(*pkt_non_pf_array));
+    if (pkt_non_pf_array == NULL) {
+        return -1;
+    }
+    uint32_t pkt_non_pf_array_size = 0;
+    struct PrefilterNonPFDataSig *frame_non_pf_array =
+            SCCalloc(max_sids, sizeof(*frame_non_pf_array));
+    if (frame_non_pf_array == NULL) {
+        SCFree(pkt_non_pf_array);
+        return -1;
+    }
+    uint32_t frame_non_pf_array_size = 0;
+
+    HashListTable *tx_engines_hash =
+            HashListTableInit(256, TxNonPFHash, TxNonPFCompare, TxNonPFFree);
+    if (tx_engines_hash == NULL) {
+        SCFree(pkt_non_pf_array);
+        SCFree(frame_non_pf_array);
+        return -1;
+    }
+
+    if (de_ctx->non_pf_engine_names == NULL) {
+        de_ctx->non_pf_engine_names =
+                HashTableInit(512, NonPFNamesHash, NonPFNamesCompare, NonPFNamesFree);
+        if (de_ctx->non_pf_engine_names == NULL) {
+            SCFree(pkt_non_pf_array);
+            SCFree(frame_non_pf_array);
+            HashListTableFree(tx_engines_hash);
+            return -1;
+        }
+    }
+
+    SignatureMask pkt_mask = 0;
+    bool pkt_mask_init = false;
+#ifdef NONPF_PKT_STATS
+    uint32_t nonpf_pkt_alproto = 0;
+    uint32_t nonpf_pkt_dsize = 0;
+    uint32_t nonpf_pkt_dport = 0;
+#endif
+    const int app_events_list_id = DetectBufferTypeGetByName("app-layer-events");
+    SCLogDebug("app_events_list_id %d", app_events_list_id);
+    for (uint32_t sig = 0; sig < sgh->init->sig_cnt; sig++) {
+        Signature *s = sgh->init->match_array[sig];
+        if (s == NULL)
+            continue;
+        SCLogDebug("checking sid %u for non-prefilter", s->id);
+        if (s->init_data->mpm_sm != NULL && (s->flags & SIG_FLAG_MPM_NEG) == 0)
+            continue;
+        if (s->init_data->prefilter_sm != NULL)
+            continue;
+        if ((s->flags & (SIG_FLAG_PREFILTER | SIG_FLAG_MPM_NEG)) == SIG_FLAG_PREFILTER)
+            continue;
+        SCLogDebug("setting up sid %u for non-prefilter", s->id);
+
+        uint8_t frame_type = 0; /**< only a single type per rule */
+        bool tx_non_pf = false;
+        bool frame_non_pf = false;
+        bool pkt_non_pf = false;
+        for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
+            const int list_id = s->init_data->buffers[x].id;
+            const DetectBufferType *buf = DetectEngineBufferTypeGetById(de_ctx, list_id);
+            /* for now, exclude app-layer-events, as they are not tied to a specific
+             * progress value like other keywords. */
+            SCLogDebug("list_id %d buf %p", list_id, buf);
+            if (list_id == app_events_list_id)
+                continue;
+            if (buf->packet) {
+                SCLogDebug("packet buf");
+                /* packet is handled below */
+                pkt_non_pf = true;
+            } else if (buf->frame) {
+                for (DetectEngineFrameInspectionEngine *f = de_ctx->frame_inspect_engines;
+                        f != NULL; f = f->next) {
+                    if (!((((s->flags & SIG_FLAG_TOSERVER) != 0 && f->dir == 0) ||
+                                  ((s->flags & SIG_FLAG_TOCLIENT) != 0 && f->dir == 1)) &&
+                                list_id == (int)f->sm_list &&
+                                AppProtoEquals(s->alproto, f->alproto)))
+                        continue;
+
+                    SCLogDebug("frame '%s' type %u", buf->name, f->type);
+                    frame_type = f->type;
+                    frame_non_pf = true;
+
+                    frame_non_pf_array[frame_non_pf_array_size].sid = s->num;
+                    frame_non_pf_array[frame_non_pf_array_size].value = s->alproto;
+                    frame_non_pf_array[frame_non_pf_array_size].frame.type = frame_type;
+                    frame_non_pf_array_size++;
+                    break;
+                }
+
+            } else {
+                SCLogDebug("x %u list_id %d", x, list_id);
+                for (DetectEngineAppInspectionEngine *app = de_ctx->app_inspect_engines;
+                        app != NULL; app = app->next) {
+                    SCLogDebug("app %p proto %s list_d %d sig dir %0x", app,
+                            AppProtoToString(app->alproto), app->sm_list,
+                            s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT));
+
+                    /* skip if:
+                     * - not in our dir
+                     * - not our list
+                     * - app proto mismatch. Both sig and app can have proto or unknown */
+                    if (!((((s->flags & SIG_FLAG_TOSERVER) != 0 && app->dir == 0) ||
+                                  ((s->flags & SIG_FLAG_TOCLIENT) != 0 && app->dir == 1)) &&
+                                list_id == (int)app->sm_list &&
+                                (s->alproto == ALPROTO_UNKNOWN || app->alproto == ALPROTO_UNKNOWN ||
+                                        AppProtoEquals(s->alproto, app->alproto))))
+                        continue;
+
+                    if (TxNonPFAddSig(de_ctx, tx_engines_hash, app->alproto, app->dir,
+                                app->progress, buf->name, s) != 0) {
+                        goto error;
+                    }
+                    tx_non_pf = true;
+                }
+            }
+        }
+        /* mark as prefiltered as the sig is now part of a engine */
+        // s->flags |= SIG_FLAG_PREFILTER;
+        //  TODO doesn't work for sigs that are in multiple sgh's
+
+        /* default to pkt if there was no tx or frame match */
+        if (!(tx_non_pf || frame_non_pf)) {
+            if (!pkt_non_pf) {
+                SCLogDebug("not frame, not tx, so pkt");
+            }
+            pkt_non_pf = true;
+        }
+
+        SCLogDebug("setting up sid %u for non-prefilter: %s", s->id,
+                tx_non_pf ? "tx engine" : (frame_non_pf ? "frame engine" : "pkt engine"));
+
+        if (pkt_non_pf) {
+            /* for pkt non prefilter, we have some space in the structure,
+             * so we can squeeze another filter */
+            uint8_t type;
+            uint16_t value;
+            if ((s->flags & SIG_FLAG_DSIZE) && s->dsize_mode == DETECT_UINT_EQ) {
+                SCLogDebug("dsize extra match");
+                type = 2;
+                value = s->dsize_low;
+#ifdef NONPF_PKT_STATS
+                nonpf_pkt_dsize++;
+#endif
+            } else if (s->dp != NULL && s->dp->next == NULL && s->dp->port == s->dp->port2) {
+                type = 1;
+                value = s->dp->port;
+#ifdef NONPF_PKT_STATS
+                nonpf_pkt_dport++;
+#endif
+            } else {
+                type = 0;
+                value = s->alproto;
+#ifdef NONPF_PKT_STATS
+                nonpf_pkt_alproto++;
+#endif
+            }
+
+            pkt_non_pf_array[pkt_non_pf_array_size].sid = s->num;
+            pkt_non_pf_array[pkt_non_pf_array_size].value = value;
+            pkt_non_pf_array[pkt_non_pf_array_size].type = type;
+            pkt_non_pf_array[pkt_non_pf_array_size].pkt.sig_mask = s->mask;
+            pkt_non_pf_array_size++;
+
+            if (pkt_mask_init) {
+                pkt_mask &= s->mask;
+            } else {
+                pkt_mask = s->mask;
+                pkt_mask_init = true;
+            }
+        }
+    }
+
+    /* for each unique sig set, add an engine */
+    for (HashListTableBucket *b = HashListTableGetListHead(tx_engines_hash); b != NULL;
+            b = HashListTableGetListNext(b)) {
+        struct TxNonPFData *t = HashListTableGetListData(b);
+        SCLogDebug("%s engine for %s hook %d has %u non-pf sigs",
+                t->dir == 0 ? "toserver" : "toclient", AppProtoToString(t->alproto), t->progress,
+                t->sigs_cnt);
+
+        if (((sgh->init->direction & SIG_FLAG_TOSERVER) && t->dir == 1) ||
+                ((sgh->init->direction & SIG_FLAG_TOCLIENT) && t->dir == 0)) {
+            SCLogDebug("skipped");
+            continue;
+        }
+
+        struct PrefilterNonPFDataTx *data =
+                SCCalloc(1, sizeof(*data) + t->sigs_cnt * sizeof(data->array[0]));
+        if (data == NULL)
+            goto error;
+        data->size = t->sigs_cnt;
+        for (uint32_t i = 0; i < t->sigs_cnt; i++) {
+            data->array[i] = t->sigs[i].sid;
+        }
+        if (PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxNonPF, t->alproto, t->progress,
+                    (void *)data, PrefilterNonPFDataFree, t->engine_name) < 0) {
+            SCFree(data);
+            goto error;
+        }
+    }
+    HashListTableFree(tx_engines_hash);
+    tx_engines_hash = NULL;
+
+    if (pkt_non_pf_array_size) {
+        struct PrefilterNonPFData *data =
+                SCCalloc(1, sizeof(*data) + pkt_non_pf_array_size * sizeof(data->array[0]));
+        if (data == NULL)
+            goto error;
+        data->size = pkt_non_pf_array_size;
+        memcpy((uint8_t *)&data->array, pkt_non_pf_array,
+                pkt_non_pf_array_size * sizeof(data->array[0]));
+        if (PrefilterAppendEngine(de_ctx, sgh, PrefilterPktNonPF, pkt_mask, (void *)data,
+                    PrefilterNonPFDataFree, "packet:non_pf") < 0) {
+            SCFree(data);
+            goto error;
+        }
+    }
+    if (frame_non_pf_array_size) {
+        SCLogDebug("%u frame non-pf sigs", frame_non_pf_array_size);
+        struct PrefilterNonPFData *data =
+                SCCalloc(1, sizeof(*data) + frame_non_pf_array_size * sizeof(data->array[0]));
+        if (data == NULL)
+            goto error;
+        data->size = frame_non_pf_array_size;
+        memcpy((uint8_t *)&data->array, frame_non_pf_array,
+                frame_non_pf_array_size * sizeof(data->array[0]));
+        if (PrefilterAppendFrameEngine(de_ctx, sgh, PrefilterFrameNonPF, ALPROTO_UNKNOWN,
+                    FRAME_ANY_TYPE, (void *)data, PrefilterNonPFDataFree, "frame:non_pf") < 0) {
+            SCFree(data);
+            goto error;
+        }
+    }
+
+    SCFree(pkt_non_pf_array);
+    pkt_non_pf_array = NULL;
+    SCFree(frame_non_pf_array);
+    frame_non_pf_array = NULL;
+    return 0;
+
+error:
+    if (tx_engines_hash) {
+        HashListTableFree(tx_engines_hash);
+    }
+    SCFree(pkt_non_pf_array);
+    SCFree(frame_non_pf_array);
+    return -1;
+}
+
+int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
 {
     int r = PatternMatchPrepareGroup(de_ctx, sgh);
     if (r != 0) {
@@ -501,16 +1032,17 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
      * all engines, otherwise only those that have been forced by the
      * prefilter keyword. */
     const enum DetectEnginePrefilterSetting setting = de_ctx->prefilter_setting;
-    for (int i = 0; i < DETECT_TBLSIZE; i++)
-    {
+    for (int i = 0; i < DETECT_TBLSIZE; i++) {
         if (sigmatch_table[i].SetupPrefilter != NULL &&
-                (setting == DETECT_PREFILTER_AUTO ||
-                 de_ctx->sm_types_prefilter[i]))
-        {
+                (setting == DETECT_PREFILTER_AUTO || de_ctx->sm_types_prefilter[i])) {
             sigmatch_table[i].SetupPrefilter(de_ctx, sgh);
         }
     }
 
+    if (SetupNonPrefilter(de_ctx, sgh) != 0) {
+        return -1;
+    }
+
     /* we have lists of engines in sgh->init now. Lets setup the
      * match arrays */
     PrefilterEngineList *el;
@@ -521,7 +1053,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->pkt_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->pkt_engines == NULL) {
-            return;
+            return -1;
         }
         memset(sgh->pkt_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -546,7 +1078,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->payload_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->payload_engines == NULL) {
-            return;
+            return -1;
         }
         memset(sgh->payload_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -571,7 +1103,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->tx_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->tx_engines == NULL) {
-            return;
+            return -1;
         }
         memset(sgh->tx_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -647,7 +1179,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->frame_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->frame_engines == NULL) {
-            return;
+            return -1;
         }
         memset(sgh->frame_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -666,6 +1198,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
             e++;
         }
     }
+
     if (sgh->init->post_rule_match_engines != NULL) {
         uint32_t cnt = 0;
         for (el = sgh->init->post_rule_match_engines; el != NULL; el = el->next) {
@@ -673,7 +1206,7 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         sgh->post_rule_match_engines = SCMallocAligned(cnt * sizeof(PrefilterEngine), CLS);
         if (sgh->post_rule_match_engines == NULL) {
-            return;
+            return -1;
         }
         memset(sgh->post_rule_match_engines, 0x00, (cnt * sizeof(PrefilterEngine)));
 
@@ -690,6 +1223,8 @@ void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
         SCLogDebug("sgh %p max local_id %u", sgh, local_id);
     }
+
+    return 0;
 }
 
 /* hash table for assigning a unique id to each engine type. */

src/detect-engine-prefilter.h

@@ -83,7 +83,7 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx,
 
 void PrefilterFreeEnginesList(PrefilterEngineList *list);
 
-void PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
+int PrefilterSetupRuleGroup(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
 void PrefilterCleanupRuleGroup(const DetectEngineCtx *de_ctx, SigGroupHead *sgh);
 
 #ifdef PROFILING
@@ -105,4 +105,6 @@ int PrefilterGenericMpmPktRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, M
 void PostRuleMatchWorkQueueAppend(
         DetectEngineThreadCtx *det_ctx, const Signature *s, const int type, const uint32_t value);
 
+void PrefilterPktNonPFStatsDump(void);
+
 #endif

src/detect-engine-profile.c

@@ -47,7 +47,6 @@ void RulesDumpTxMatchArray(const DetectEngineThreadCtx *det_ctx, const SigGroupH
     SCJbSetUint(js, "rule_group_id", sgh->id);
     SCJbSetUint(js, "rule_cnt", rule_cnt);
     SCJbSetUint(js, "pkt_rule_cnt", pkt_prefilter_cnt);
-    SCJbSetUint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt);
 
     SCJbOpenArray(js, "rules");
     for (uint32_t x = 0; x < rule_cnt; x++) {
@@ -91,7 +90,6 @@ void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx,
     SCJbSetString(js, "inspect_type", "packet");
     SCJbSetUint(js, "rule_group_id", sgh->id);
     SCJbSetUint(js, "rule_cnt", det_ctx->match_array_cnt);
-    SCJbSetUint(js, "non_pf_rule_cnt", det_ctx->non_pf_store_cnt);
 
     SCJbOpenArray(js, "rules");
     for (uint32_t x = 0; x < det_ctx->match_array_cnt; x++) {

src/detect-engine-siggroup.c

@@ -167,18 +167,6 @@ void SigGroupHeadFree(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
 
     SCLogDebug("sgh %p", sgh);
 
-    if (sgh->non_pf_other_store_array != NULL) {
-        SCFree(sgh->non_pf_other_store_array);
-        sgh->non_pf_other_store_array = NULL;
-        sgh->non_pf_other_store_cnt = 0;
-    }
-
-    if (sgh->non_pf_syn_store_array != NULL) {
-        SCFree(sgh->non_pf_syn_store_array);
-        sgh->non_pf_syn_store_array = NULL;
-        sgh->non_pf_syn_store_cnt = 0;
-    }
-
     if (sgh->init != NULL) {
         SigGroupHeadInitDataFree(sgh->init);
         sgh->init = NULL;
@@ -620,82 +608,6 @@ void SigGroupHeadSetupFiles(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     }
 }
 
-/** \brief build an array of rule id's for sigs with no prefilter
- *  Also updated de_ctx::non_pf_store_cnt_max to track the highest cnt
- */
-int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
-{
-    Signature *s = NULL;
-    uint32_t sig = 0;
-    uint32_t non_pf = 0;
-    uint32_t non_pf_syn = 0;
-
-    if (sgh == NULL)
-        return 0;
-
-    BUG_ON(sgh->non_pf_other_store_array != NULL);
-
-    for (sig = 0; sig < sgh->init->sig_cnt; sig++) {
-        s = sgh->init->match_array[sig];
-        if (s == NULL)
-            continue;
-
-        if (!(s->flags & SIG_FLAG_PREFILTER) || (s->flags & SIG_FLAG_MPM_NEG)) {
-            if (!(DetectFlagsSignatureNeedsSynPackets(s))) {
-                non_pf++;
-            }
-            non_pf_syn++;
-        }
-    }
-
-    if (non_pf == 0 && non_pf_syn == 0) {
-        sgh->non_pf_other_store_array = NULL;
-        sgh->non_pf_syn_store_array = NULL;
-        return 0;
-    }
-
-    if (non_pf > 0) {
-        sgh->non_pf_other_store_array = SCCalloc(non_pf, sizeof(SignatureNonPrefilterStore));
-        BUG_ON(sgh->non_pf_other_store_array == NULL);
-    }
-
-    if (non_pf_syn > 0) {
-        sgh->non_pf_syn_store_array = SCCalloc(non_pf_syn, sizeof(SignatureNonPrefilterStore));
-        BUG_ON(sgh->non_pf_syn_store_array == NULL);
-    }
-
-    for (sig = 0; sig < sgh->init->sig_cnt; sig++) {
-        s = sgh->init->match_array[sig];
-        if (s == NULL)
-            continue;
-
-        if (!(s->flags & SIG_FLAG_PREFILTER) || (s->flags & SIG_FLAG_MPM_NEG)) {
-            if (!(DetectFlagsSignatureNeedsSynPackets(s))) {
-                BUG_ON(sgh->non_pf_other_store_cnt >= non_pf);
-                BUG_ON(sgh->non_pf_other_store_array == NULL);
-                sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].id = s->num;
-                sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].mask = s->mask;
-                sgh->non_pf_other_store_array[sgh->non_pf_other_store_cnt].alproto = s->alproto;
-                sgh->non_pf_other_store_cnt++;
-            }
-
-            BUG_ON(sgh->non_pf_syn_store_cnt >= non_pf_syn);
-            BUG_ON(sgh->non_pf_syn_store_array == NULL);
-            sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].id = s->num;
-            sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].mask = s->mask;
-            sgh->non_pf_syn_store_array[sgh->non_pf_syn_store_cnt].alproto = s->alproto;
-            sgh->non_pf_syn_store_cnt++;
-        }
-    }
-
-    /* track highest cnt for any sgh in our de_ctx */
-    uint32_t max = MAX(sgh->non_pf_other_store_cnt, sgh->non_pf_syn_store_cnt);
-    if (max > de_ctx->non_pf_store_cnt_max)
-        de_ctx->non_pf_store_cnt_max = max;
-
-    return 0;
-}
-
 /**
  * \brief Check if a SigGroupHead contains a Signature, whose sid is sent as an
  *        argument.

src/detect-engine.c

@@ -2848,6 +2848,9 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
         SCDetectRequiresStatusFree(de_ctx->requirements);
     }
 
+    if (de_ctx->non_pf_engine_names) {
+        HashTableFree(de_ctx->non_pf_engine_names);
+    }
     SCFree(de_ctx);
     //DetectAddressGroupPrintMemory();
     //DetectSigGroupPrintMemory();
@@ -3400,13 +3403,6 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *
         return TM_ECODE_FAILED;
     }
 
-    /* sized to the max of our sgh settings. A max setting of 0 implies that all
-     * sgh's have: sgh->non_pf_store_cnt == 0 */
-    if (de_ctx->non_pf_store_cnt_max > 0) {
-        det_ctx->non_pf_id_array =  SCCalloc(de_ctx->non_pf_store_cnt_max, sizeof(SigIntId));
-        BUG_ON(det_ctx->non_pf_id_array == NULL);
-    }
-
     /* DeState */
     if (de_ctx->sig_array_len > 0) {
         det_ctx->match_array_len = de_ctx->sig_array_len;
@@ -3659,10 +3655,6 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     if (det_ctx->spm_thread_ctx != NULL) {
         SpmDestroyThreadCtx(det_ctx->spm_thread_ctx);
     }
-
-    if (det_ctx->non_pf_id_array != NULL)
-        SCFree(det_ctx->non_pf_id_array);
-
     if (det_ctx->match_array != NULL)
         SCFree(det_ctx->match_array);
 
@@ -3716,7 +3708,7 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     }
 
     AppLayerDecoderEventsFreeEvents(&det_ctx->decoder_events);
-
+    PrefilterPktNonPFStatsDump();
     SCFree(det_ctx);
 
     ThresholdCacheThreadFree();

src/detect.c

@@ -71,7 +71,6 @@ typedef struct DetectRunScratchpad {
     const uint8_t flow_flags; /* flow/state flags: STREAM_* */
     const bool app_decoder_events;
     const SigGroupHead *sgh;
-    SignatureMask pkt_mask;
 } DetectRunScratchpad;
 
 /* prototypes */
@@ -258,118 +257,18 @@ const SigGroupHead *SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx,
     SCReturnPtr(sgh, "SigGroupHead");
 }
 
-static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
-                                            DetectEngineThreadCtx *det_ctx)
+static inline void DetectPrefilterCopyDeDup(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx)
 {
-    SigIntId mpm, nonmpm;
-    SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array;
-    SigIntId *nonmpm_ptr = det_ctx->non_pf_id_array;
-    uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt;
-    uint32_t n_cnt = det_ctx->non_pf_id_cnt;
-    SigIntId *final_ptr;
-    uint32_t final_cnt;
-    SigIntId id;
-    SigIntId previous_id = (SigIntId)-1;
+    SigIntId *pf_ptr = det_ctx->pmq.rule_id_array;
+    uint32_t final_cnt = det_ctx->pmq.rule_id_array_cnt;
     Signature **sig_array = de_ctx->sig_array;
     Signature **match_array = det_ctx->match_array;
-    Signature *s;
-
-    SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt);
-
-    /* Load first values. */
-    if (likely(m_cnt)) {
-        mpm = *mpm_ptr;
-    } else {
-        /* mpm list is empty */
-        final_ptr = nonmpm_ptr;
-        final_cnt = n_cnt;
-        goto final;
-    }
-    if (likely(n_cnt)) {
-        nonmpm = *nonmpm_ptr;
-    } else {
-        /* non-mpm list is empty. */
-        final_ptr = mpm_ptr;
-        final_cnt = m_cnt;
-        goto final;
-    }
-    while (1) {
-        if (mpm < nonmpm) {
-            /* Take from mpm list */
-            id = mpm;
-
-            s = sig_array[id];
-            /* As the mpm list can contain duplicates, check for that here. */
-            if (likely(id != previous_id)) {
-                *match_array++ = s;
-                previous_id = id;
-            }
-            if (unlikely(--m_cnt == 0)) {
-                /* mpm list is now empty */
-                final_ptr = nonmpm_ptr;
-                final_cnt = n_cnt;
-                goto final;
-             }
-             mpm_ptr++;
-             mpm = *mpm_ptr;
-         } else if (mpm > nonmpm) {
-             id = nonmpm;
-
-             s = sig_array[id];
-             /* As the mpm list can contain duplicates, check for that here. */
-             if (likely(id != previous_id)) {
-                 *match_array++ = s;
-                 previous_id = id;
-             }
-             if (unlikely(--n_cnt == 0)) {
-                 final_ptr = mpm_ptr;
-                 final_cnt = m_cnt;
-                 goto final;
-             }
-             nonmpm_ptr++;
-             nonmpm = *nonmpm_ptr;
-
-        } else { /* implied mpm == nonmpm */
-            /* special case: if on both lists, it's a negated mpm pattern */
-
-            /* mpm list may have dups, so skip past them here */
-            while (--m_cnt != 0) {
-                mpm_ptr++;
-                mpm = *mpm_ptr;
-                if (mpm != nonmpm)
-                    break;
-            }
-            /* if mpm is done, update nonmpm_ptrs and jump to final */
-            if (unlikely(m_cnt == 0)) {
-                n_cnt--;
-
-                /* mpm list is now empty */
-                final_ptr = ++nonmpm_ptr;
-                final_cnt = n_cnt;
-                goto final;
-            }
-            /* otherwise, if nonmpm is done jump to final for mpm
-             * mpm ptrs already updated */
-            if (unlikely(--n_cnt == 0)) {
-                final_ptr = mpm_ptr;
-                final_cnt = m_cnt;
-                goto final;
-            }
-
-            /* not at end of the lists, update nonmpm. Mpm already
-             * updated in while loop above. */
-            nonmpm_ptr++;
-            nonmpm = *nonmpm_ptr;
-        }
-    }
-
- final: /* Only one list remaining. Just walk that list. */
-
+    SigIntId previous_id = (SigIntId)-1;
     while (final_cnt-- > 0) {
-        id = *final_ptr++;
-        s = sig_array[id];
+        SigIntId id = *pf_ptr++;
+        Signature *s = sig_array[id];
 
-        /* As the mpm list can contain duplicates, check for that here. */
+        /* As the prefilter list can contain duplicates, check for that here. */
         if (likely(id != previous_id)) {
             *match_array++ = s;
             previous_id = id;
@@ -377,48 +276,10 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
     }
 
     det_ctx->match_array_cnt = match_array - det_ctx->match_array;
-    DEBUG_VALIDATE_BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_pf_id_cnt) < det_ctx->match_array_cnt);
+    DEBUG_VALIDATE_BUG_ON(det_ctx->pmq.rule_id_array_cnt < det_ctx->match_array_cnt);
     PMQ_RESET(&det_ctx->pmq);
 }
 
-/** \internal
- *  \brief build non-prefilter list based on the rule group list we've set.
- */
-static inline void DetectPrefilterBuildNonPrefilterList(
-        DetectEngineThreadCtx *det_ctx, const SignatureMask mask, const AppProto alproto)
-{
-    for (uint32_t x = 0; x < det_ctx->non_pf_store_cnt; x++) {
-        /* only if the mask matches this rule can possibly match,
-         * so build the non_mpm array only for match candidates */
-        const SignatureMask rule_mask = det_ctx->non_pf_store_ptr[x].mask;
-        const AppProto rule_alproto = det_ctx->non_pf_store_ptr[x].alproto;
-        if ((rule_mask & mask) == rule_mask &&
-                (rule_alproto == 0 || AppProtoEquals(rule_alproto, alproto))) {
-            det_ctx->non_pf_id_array[det_ctx->non_pf_id_cnt++] = det_ctx->non_pf_store_ptr[x].id;
-        }
-    }
-}
-
-/** \internal
- *  \brief select non-mpm list
- *  Based on the packet properties, select the non-mpm list to use
- *  \todo move non_pf_store* into scratchpad */
-static inline void
-DetectPrefilterSetNonPrefilterList(const Packet *p, DetectEngineThreadCtx *det_ctx, DetectRunScratchpad *scratch)
-{
-    if ((p->proto == IPPROTO_TCP) && PacketIsTCP(p) && (PacketGetTCP(p)->th_flags & TH_SYN)) {
-        det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_syn_store_array;
-        det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_syn_store_cnt;
-    } else {
-        det_ctx->non_pf_store_ptr = scratch->sgh->non_pf_other_store_array;
-        det_ctx->non_pf_store_cnt = scratch->sgh->non_pf_other_store_cnt;
-    }
-    SCLogDebug("sgh non_pf ptr %p cnt %u (syn %p/%u, other %p/%u)",
-            det_ctx->non_pf_store_ptr, det_ctx->non_pf_store_cnt,
-            scratch->sgh->non_pf_syn_store_array, scratch->sgh->non_pf_syn_store_cnt,
-            scratch->sgh->non_pf_other_store_array, scratch->sgh->non_pf_other_store_cnt);
-}
-
 /** \internal
  *  \brief update flow's file tracking flags based on the detection engine
  *         A set of flags is prepared that is sent to the File API. The
@@ -670,42 +531,21 @@ static inline void DetectRunPrefilterPkt(
     DetectRunScratchpad *scratch
 )
 {
-    DetectPrefilterSetNonPrefilterList(p, det_ctx, scratch);
-
     /* create our prefilter mask */
-    PacketCreateMask(p, &scratch->pkt_mask, scratch->alproto, scratch->app_decoder_events);
-
-    /* build and prefilter non_pf list against the mask of the packet */
-    PACKET_PROFILING_DETECT_START(p, PROF_DETECT_NONMPMLIST);
-    det_ctx->non_pf_id_cnt = 0;
-    if (likely(det_ctx->non_pf_store_cnt > 0)) {
-        DetectPrefilterBuildNonPrefilterList(det_ctx, scratch->pkt_mask, scratch->alproto);
-    }
-    PACKET_PROFILING_DETECT_END(p, PROF_DETECT_NONMPMLIST);
-
+    PacketCreateMask(p, &p->sig_mask, scratch->alproto, scratch->app_decoder_events);
     /* run the prefilter engines */
-    Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags, scratch->pkt_mask);
+    Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags, p->sig_mask);
     /* create match list if we have non-pf and/or pf */
-    if (det_ctx->non_pf_store_cnt || det_ctx->pmq.rule_id_array_cnt) {
+    if (det_ctx->pmq.rule_id_array_cnt) {
 #ifdef PROFILING
         if (tv) {
             StatsAddUI64(tv, det_ctx->counter_mpm_list, (uint64_t)det_ctx->pmq.rule_id_array_cnt);
         }
 #endif
         PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_SORT2);
-        DetectPrefilterMergeSort(de_ctx, det_ctx);
+        DetectPrefilterCopyDeDup(de_ctx, det_ctx);
         PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_SORT2);
     }
-
-#ifdef PROFILING
-    if (tv) {
-        StatsAddUI64(tv, det_ctx->counter_nonmpm_list,
-                             (uint64_t)det_ctx->non_pf_store_cnt);
-        /* non mpm sigs after mask prefilter */
-        StatsAddUI64(tv, det_ctx->counter_fnonmpm_list,
-                             (uint64_t)det_ctx->non_pf_id_cnt);
-    }
-#endif
 }
 
 /** \internal
@@ -812,8 +652,8 @@ static inline void DetectRulePacketRules(
 
         /* don't run mask check for stateful rules.
          * There we depend on prefilter */
-        if ((s->mask & scratch->pkt_mask) != s->mask) {
-            SCLogDebug("mask mismatch %x & %x != %x", s->mask, scratch->pkt_mask, s->mask);
+        if ((s->mask & p->sig_mask) != s->mask) {
+            SCLogDebug("mask mismatch %x & %x != %x", s->mask, p->sig_mask, s->mask);
             goto next;
         }
 
@@ -1021,7 +861,7 @@ static DetectRunScratchpad DetectRunSetup(
         app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser);
     }
 
-    DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL, 0 };
+    DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, NULL };
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_SETUP);
     return pad;
 }
@@ -1156,11 +996,11 @@ void *DetectGetInnerTx(void *tx_ptr, AppProto alproto, AppProto engine_alproto,
         if (engine_alproto == ALPROTO_DNS) {
             // need to get the dns tx pointer
             tx_ptr = SCDoH2GetDnsTx(tx_ptr, flow_flags);
-        } else if (engine_alproto != ALPROTO_HTTP2) {
+        } else if (engine_alproto != ALPROTO_HTTP2 && engine_alproto != ALPROTO_UNKNOWN) {
             // incompatible engine->alproto with flow alproto
             tx_ptr = NULL;
         }
-    } else if (engine_alproto != alproto) {
+    } else if (engine_alproto != alproto && engine_alproto != ALPROTO_UNKNOWN) {
         // incompatible engine->alproto with flow alproto
         tx_ptr = NULL;
     }

src/detect.h

@@ -316,9 +316,6 @@ typedef struct DetectPort_ {
 // vacancy 1x
 #define SIG_MASK_REQUIRE_ENGINE_EVENT       BIT_U8(7)
 
-/* for now a uint8_t is enough */
-#define SignatureMask uint8_t
-
 #define FILE_SIG_NEED_FILE          0x01
 #define FILE_SIG_NEED_FILENAME      0x02
 #define FILE_SIG_NEED_MAGIC         0x04    /**< need the start of the file */
@@ -891,10 +888,6 @@ typedef struct DetectEngineCtx_ {
 
     uint32_t signum;
 
-    /** Maximum value of all our sgh's non_mpm_store_cnt setting,
-     *  used to alloc det_ctx::non_mpm_id_array */
-    uint32_t non_pf_store_cnt_max;
-
     /* used by the signature ordering module */
     struct SCSigOrderFunc_ *sc_sig_order_funcs;
 
@@ -1079,6 +1072,10 @@ typedef struct DetectEngineCtx_ {
 
     /* number of signatures using filestore, limited as u16 */
     uint16_t filestore_cnt;
+
+    /* name store for non-prefilter engines. Used in profiling but
+     * part of the API, so hash is always used. */
+    HashTable *non_pf_engine_names;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */
@@ -1153,11 +1150,6 @@ typedef struct DetectEngineThreadCtx_ {
     /* the thread to which this detection engine thread belongs */
     ThreadVars *tv;
 
-    /** Array of non-prefiltered sigs that need to be evaluated. Updated
-     *  per packet based on the rule group and traffic properties. */
-    SigIntId *non_pf_id_array;
-    uint32_t non_pf_id_cnt; // size is cnt * sizeof(uint32_t)
-
     uint32_t mt_det_ctxs_cnt;
     struct DetectEngineThreadCtx_ **mt_det_ctxs;
     HashTable *mt_det_ctxs_hash;
@@ -1245,9 +1237,6 @@ typedef struct DetectEngineThreadCtx_ {
     RuleMatchCandidateTx *tx_candidates;
     uint32_t tx_candidates_size;
 
-    SignatureNonPrefilterStore *non_pf_store_ptr;
-    uint32_t non_pf_store_cnt;
-
     MpmThreadCtx mtc; /**< thread ctx for the mpm */
     /* work queue for post-rule matching affecting prefilter */
     PostRuleMatchWorkQueue post_rule_work_queue;
@@ -1528,13 +1517,6 @@ typedef struct SigGroupHead_ {
 
     uint32_t id; /**< unique id used to index sgh_array for stats */
 
-    /* non prefilter list excluding SYN rules */
-    uint32_t non_pf_other_store_cnt;
-    uint32_t non_pf_syn_store_cnt;
-    SignatureNonPrefilterStore *non_pf_other_store_array; // size is non_mpm_store_cnt * sizeof(SignatureNonPrefilterStore)
-    /* non mpm list including SYN rules */
-    SignatureNonPrefilterStore *non_pf_syn_store_array; // size is non_mpm_syn_store_cnt * sizeof(SignatureNonPrefilterStore)
-
     PrefilterEngine *pkt_engines;
     PrefilterEngine *payload_engines;
     PrefilterEngine *tx_engines;

src/packet.c

@@ -97,6 +97,7 @@ void PacketReinit(Packet *p)
     p->recursion_level = 0;
     PACKET_FREE_EXTDATA(p);
     p->app_update_direction = 0;
+    p->sig_mask = 0;
     p->flags = 0;
     p->flowflags = 0;
     p->pkt_src = 0;

src/util-profiling-rulegroups.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2015 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -38,9 +38,6 @@
 typedef struct SCProfileSghData_ {
     uint64_t checks;
 
-    uint64_t non_mpm_generic;
-    uint64_t non_mpm_syn;
-
     uint64_t post_prefilter_sigs_total;
     uint64_t post_prefilter_sigs_max;
 
@@ -133,8 +130,6 @@ static void DoDumpJSON(SCProfileSghDetectCtx *rules_ctx, FILE *fp, const char *n
         if (jsm) {
             json_object_set_new(jsm, "id", json_integer(i));
             json_object_set_new(jsm, "checks", json_integer(d->checks));
-            json_object_set_new(jsm, "non_mpm_generic", json_integer(d->non_mpm_generic));
-            json_object_set_new(jsm, "non_mpm_syn", json_integer(d->non_mpm_syn));
             json_object_set_new(jsm, "avgmpms", json_real(avgmpms));
             json_object_set_new(jsm, "mpm_match_cnt_max", json_integer(d->mpm_match_cnt_max));
             json_object_set_new(jsm, "avgsigs", json_real(avgsigs));
@@ -203,16 +198,8 @@ static void DoDump(SCProfileSghDetectCtx *rules_ctx, FILE *fp, const char *name)
             avgmpms = (double)((double)d->mpm_match_cnt_total / (double)d->checks);
         }
 
-        fprintf(fp,
-            "  %-16u %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15.2f %-15"PRIu64" %-15.2f %-15"PRIu64"\n",
-            i,
-            d->checks,
-            d->non_mpm_generic,
-            d->non_mpm_syn,
-            avgmpms,
-            d->mpm_match_cnt_max,
-            avgsigs,
-            d->post_prefilter_sigs_max);
+        fprintf(fp, "  %-16u %-15" PRIu64 " %-15.2f %-15" PRIu64 " %-15.2f %-15" PRIu64 "\n", i,
+                d->checks, avgmpms, d->mpm_match_cnt_max, avgsigs, d->post_prefilter_sigs_max);
     }
     fprintf(fp,"\n");
 }
@@ -264,12 +251,6 @@ SCProfilingSghUpdateCounter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *
         SCProfileSghData *p = &det_ctx->sgh_perf_data[sgh->id];
         p->checks++;
 
-        if (det_ctx->non_pf_store_cnt > 0) {
-            if (det_ctx->non_pf_store_ptr == sgh->non_pf_syn_store_array)
-                p->non_mpm_syn++;
-            else
-                p->non_mpm_generic++;
-        }
         p->post_prefilter_sigs_total += det_ctx->match_array_cnt;
         if (det_ctx->match_array_cnt > p->post_prefilter_sigs_max)
             p->post_prefilter_sigs_max = det_ctx->match_array_cnt;
@@ -334,8 +315,6 @@ static void SCProfilingSghThreadMerge(DetectEngineCtx *de_ctx, const DetectEngin
     uint32_t i;
     for (i = 0; i < de_ctx->sgh_array_cnt; i++) {
         ADD(checks);
-        ADD(non_mpm_generic);
-        ADD(non_mpm_syn);
         ADD(post_prefilter_sigs_total);
         ADD(mpm_match_cnt_total);