detect: sync prefilter and app inspect alproto logic

In firewall mode the alproto logic when building prefilter and rule
engines is strict, but was out of sync between prefilter and rule app
inspect engines.

For SIGNATURE_HOOK_TYPE_APP rules the logic is strict, with an exception
for HTTP/2 and DOH2. DOH2 is not a full protocol implementation, but
rather HTTP/2 with a different alproto.

Fixes: d64954a873 ("detect: don't register unrelated inspect engines")

src/app-layer-protos.h

@@ -117,6 +117,24 @@ static inline bool AppProtoEquals(AppProto sigproto, AppProto alproto)
     return false;
 }
 
+// whether a signature AppProto matches a flow (or signature) AppProto
+// only see DOH2/HTTP2 as the same
+static inline bool AppProtoEqualsStrict(AppProto sigproto, AppProto alproto)
+{
+    if (sigproto == alproto) {
+        return true;
+    }
+    switch (sigproto) {
+        case ALPROTO_HTTP2:
+            // a HTTP2 signature matches on either HTTP2 or DOH2 flows
+            return (alproto == ALPROTO_DOH2);
+        case ALPROTO_DOH2:
+            // a DOH2 signature accepts dns, http2 or http generic keywords
+            return (alproto == ALPROTO_HTTP2);
+    }
+    return false;
+}
+
 // whether a signature AppProto matches a flow (or signature) AppProto
 static inline AppProto AppProtoCommon(AppProto sigproto, AppProto alproto)
 {

src/detect-engine-mpm.c

@@ -2216,8 +2216,16 @@ static void PrepareMpms(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             case DETECT_BUFFER_MPM_TYPE_APP: {
                 for (size_t e = 0; e < engines[list].idx; e++) {
                     const AppProto alproto = engines[list].array[e];
-                    if (!(AppProtoEquals(s->alproto, alproto) || s->alproto == 0))
-                        continue;
+                    if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+                        /* SIGNATURE_HOOK_TYPE_APP rules are exact about their protocol */
+                        if (!(AppProtoEqualsStrict(s->alproto, alproto))) {
+                            continue;
+                        }
+                    } else {
+                        /* other rules use the more relax AppProtoEquals logic */
+                        if (!(AppProtoEquals(s->alproto, alproto) || s->alproto == 0))
+                            continue;
+                    }
 
                     DetectBufferInstance lookup = { .list = list, .alproto = alproto };
                     DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);

src/detect-engine-prefilter.c

@@ -1038,6 +1038,13 @@ static int SetupNonPrefilter(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
                             AppProtoToString(app->alproto), app->sm_list,
                             s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT));
 
+                    /* extra strict alproto check for SIGNATURE_HOOK_TYPE_APP,
+                     * just like with mpm and per rule app engine. */
+                    if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+                        if (!AppProtoEqualsStrict(s->alproto, app->alproto))
+                            continue;
+                    }
+
                     /* skip if:
                      * - not in our dir
                      * - not our list

src/detect-engine.c

@@ -739,7 +739,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     } else if (s->alproto != ALPROTO_UNKNOWN) {
         if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
             /* SIGNATURE_HOOK_TYPE_APP rules are exact about their protocol */
-            if (t->alproto != s->alproto) {
+            if (!(AppProtoEqualsStrict(s->alproto, t->alproto))) {
                 return;
             }
         } else {

src/detect-parse.c

@@ -2866,7 +2866,7 @@ static int SigValidateCheckBuffers(
             if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
                 /* only allow rules to use the hook for engines at that
                  * exact progress for now. */
-                if (app->alproto != s->alproto) {
+                if (!(AppProtoEqualsStrict(s->alproto, app->alproto))) {
                     continue;
                 }
             } else {