From f8f28c6f6ff58e2325bd51611092f5199ce7ac82 Mon Sep 17 00:00:00 2001
From: Victor Julien <vjulien@oisf.net>
Date: Wed, 1 Jul 2026 20:40:39 +0200
Subject: [PATCH] detect: sync prefilter and app inspect alproto logic

In firewall mode the alproto logic when building prefilter and rule
engines is strict, but was out of sync between prefilter and rule app
inspect engines.

For SIGNATURE_HOOK_TYPE_APP rules the logic is strict, with an exception
for HTTP/2 and DOH2. DOH2 is not a full protocol implementation, but
rather HTTP/2 with a different alproto.

Fixes: d64954a873be ("detect: don't register unrelated inspect engines")
---
 src/app-layer-protos.h        | 18 ++++++++++++++++++
 src/detect-engine-mpm.c       | 12 ++++++++++--
 src/detect-engine-prefilter.c |  7 +++++++
 src/detect-engine.c           |  2 +-
 src/detect-parse.c            |  2 +-
 5 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/src/app-layer-protos.h b/src/app-layer-protos.h
index 6eca4c99a0..21b21359ca 100644
--- a/src/app-layer-protos.h
+++ b/src/app-layer-protos.h
@@ -117,6 +117,24 @@ static inline bool AppProtoEquals(AppProto sigproto, AppProto alproto)
     return false;
 }
 
+// whether a signature AppProto matches a flow (or signature) AppProto
+// only see DOH2/HTTP2 as the same
+static inline bool AppProtoEqualsStrict(AppProto sigproto, AppProto alproto)
+{
+    if (sigproto == alproto) {
+        return true;
+    }
+    switch (sigproto) {
+        case ALPROTO_HTTP2:
+            // a HTTP2 signature matches on either HTTP2 or DOH2 flows
+            return (alproto == ALPROTO_DOH2);
+        case ALPROTO_DOH2:
+            // a DOH2 signature accepts dns, http2 or http generic keywords
+            return (alproto == ALPROTO_HTTP2);
+    }
+    return false;
+}
+
 // whether a signature AppProto matches a flow (or signature) AppProto
 static inline AppProto AppProtoCommon(AppProto sigproto, AppProto alproto)
 {
diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c
index 7803d00537..90291d03ef 100644
--- a/src/detect-engine-mpm.c
+++ b/src/detect-engine-mpm.c
@@ -2216,8 +2216,16 @@ static void PrepareMpms(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             case DETECT_BUFFER_MPM_TYPE_APP: {
                 for (size_t e = 0; e < engines[list].idx; e++) {
                     const AppProto alproto = engines[list].array[e];
-                    if (!(AppProtoEquals(s->alproto, alproto) || s->alproto == 0))
-                        continue;
+                    if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+                        /* SIGNATURE_HOOK_TYPE_APP rules are exact about their protocol */
+                        if (!(AppProtoEqualsStrict(s->alproto, alproto))) {
+                            continue;
+                        }
+                    } else {
+                        /* other rules use the more relax AppProtoEquals logic */
+                        if (!(AppProtoEquals(s->alproto, alproto) || s->alproto == 0))
+                            continue;
+                    }
 
                     DetectBufferInstance lookup = { .list = list, .alproto = alproto };
                     DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c
index 9edb31796c..103e25ea3b 100644
--- a/src/detect-engine-prefilter.c
+++ b/src/detect-engine-prefilter.c
@@ -1038,6 +1038,13 @@ static int SetupNonPrefilter(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
                             AppProtoToString(app->alproto), app->sm_list,
                             s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT));
 
+                    /* extra strict alproto check for SIGNATURE_HOOK_TYPE_APP,
+                     * just like with mpm and per rule app engine. */
+                    if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+                        if (!AppProtoEqualsStrict(s->alproto, app->alproto))
+                            continue;
+                    }
+
                     /* skip if:
                      * - not in our dir
                      * - not our list
diff --git a/src/detect-engine.c b/src/detect-engine.c
index 0a4109c760..591cf855f9 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -739,7 +739,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     } else if (s->alproto != ALPROTO_UNKNOWN) {
         if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
             /* SIGNATURE_HOOK_TYPE_APP rules are exact about their protocol */
-            if (t->alproto != s->alproto) {
+            if (!(AppProtoEqualsStrict(s->alproto, t->alproto))) {
                 return;
             }
         } else {
diff --git a/src/detect-parse.c b/src/detect-parse.c
index 6b7f61b9ea..e8225c98cf 100644
--- a/src/detect-parse.c
+++ b/src/detect-parse.c
@@ -2866,7 +2866,7 @@ static int SigValidateCheckBuffers(
             if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
                 /* only allow rules to use the hook for engines at that
                  * exact progress for now. */
-                if (app->alproto != s->alproto) {
+                if (!(AppProtoEqualsStrict(s->alproto, app->alproto))) {
                     continue;
                 }
             } else {