detect: don't register unrelated inspect engines

For rules that specify an explicit app-layer hook,
e.g. http1:request_headers, don't register inspect engines for
other protocols like HTTP/2. These have their own progress tracking,
so should be excluded from these rules.

src/detect-engine.c

@@ -707,8 +707,19 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
 {
     if (t->alproto == ALPROTO_UNKNOWN) {
         /* special case, inspect engine applies to all protocols */
-    } else if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, t->alproto))
+    } else if (s->alproto != ALPROTO_UNKNOWN) {
+        if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+            /* SIGNATURE_HOOK_TYPE_APP rules are exact about their protocol */
+            if (t->alproto != s->alproto) {
                 return;
+            }
+        } else {
+            /* other rules use the more relax AppProtoEquals logic */
+            if (!AppProtoEquals(s->alproto, t->alproto)) {
+                return;
+            }
+        }
+    }
 
     if (s->flags & SIG_FLAG_TOSERVER && !(s->flags & SIG_FLAG_TOCLIENT)) {
         if (t->dir == 1)

src/detect-parse.c

@@ -2700,8 +2700,21 @@ static int SigValidateCheckBuffers(
 
         const DetectEngineAppInspectionEngine *app = de_ctx->app_inspect_engines;
         for (; app != NULL; app = app->next) {
-            if (app->sm_list == b->id &&
-                    (AppProtoEquals(s->alproto, app->alproto) || s->alproto == 0)) {
+            if (app->sm_list != b->id)
+                continue;
+
+            if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+                /* only allow rules to use the hook for engines at that
+                 * exact progress for now. */
+                if (app->alproto != s->alproto) {
+                    continue;
+                }
+            } else {
+                if (!(AppProtoEquals(s->alproto, app->alproto) || s->alproto == 0)) {
+                    continue;
+                }
+            }
+
             SCLogDebug("engine %s dir %d alproto %d",
                     DetectEngineBufferTypeGetNameById(de_ctx, app->sm_list), app->dir,
                     app->alproto);
@@ -2718,8 +2731,6 @@ static int SigValidateCheckBuffers(
                 bufdir[b->id].tc += (app->dir == 1);
             }
 
-                /* only allow rules to use the hook for engines at that
-                 * exact progress for now. */
             if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
                 if ((s->flags & SIG_FLAG_TOSERVER) && (app->dir == 0) &&
                         app->progress != s->init_data->hook.t.app.app_progress) {
@@ -2734,7 +2745,6 @@ static int SigValidateCheckBuffers(
                 }
             }
         }
-        }
 
         if (!DetectEngineBufferRunValidateCallback(de_ctx, b->id, s, &de_ctx->sigerror)) {
             SCReturnInt(0);