detect: introduce explicit hooks

Generic:
        <app_proto>:request_started and <app_proto>:response_started
        <app_proto>:request_complete and <app_proto>:response_complete

Per protocol, it uses the registered progress (state) values. E.g.

        tls:client_hello_done

A rule ruleset could be:

        pass tls:client_hello_done any any -> any any (tls.sni; content:"www.google.com"; sid:21; alert;)
        drop tls:client_hello_done any any -> any any (sid:22;)

The pass rule is evaluated when the client hello is parsed, and if it
doesn't match the drop rule will be evaluated.

Registers each generic lists as "<alproto>:<progress state>:generic"
(e.g. "tls:client_hello_done:generic").

Ticket: #7485.

src/detect-engine-build.c

@@ -1644,6 +1644,12 @@ void SignatureSetType(DetectEngineCtx *de_ctx, Signature *s)
     BUG_ON(s->type != SIG_TYPE_NOT_SET);
     int iponly = 0;
 
+    if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+        s->type = SIG_TYPE_APP_TX;
+        SCLogDebug("%u: set to app_tx due to hook type app", s->id);
+        SCReturn;
+    }
+
     /* see if the sig is dp only */
     if (SignatureIsPDOnly(de_ctx, s) == 1) {
         s->type = SIG_TYPE_PDONLY;

src/detect-engine-mpm.c

@@ -2556,9 +2556,8 @@ void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature *
 
     const DetectEngineAppInspectionEngine *app = s->app_inspect;
     for (; app != NULL; app = app->next) {
-        DEBUG_VALIDATE_BUG_ON(app->smd == NULL);
         SigMatchData *smd = app->smd;
-        do {
+        while (smd) {
             switch (smd->type) {
                 case DETECT_CONTENT: {
                     const DetectContentData *cd = (const DetectContentData *)smd->ctx;
@@ -2586,7 +2585,7 @@ void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature *
             if (smd->is_last)
                 break;
             smd++;
-        } while (1);
+        }
     }
     const DetectEnginePktInspectionEngine *pkt = s->pkt_inspect;
     for (; pkt != NULL; pkt = pkt->next) {

src/detect-engine-register.c

@@ -506,6 +506,8 @@ void SigTableInit(void)
 
 void SigTableSetup(void)
 {
+    DetectRegisterAppLayerHookLists();
+
     DetectSidRegister();
     DetectPriorityRegister();
     DetectPrefilterRegister();

src/detect-engine.c

@@ -686,7 +686,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     new_engine->sm_list = t->sm_list;
     new_engine->sm_list_base = t->sm_list_base;
     new_engine->smd = smd;
-    new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd);
+    new_engine->match_on_null = smd ? DetectContentInspectionMatchOnAbsentBuffer(smd) : false;
     new_engine->progress = t->progress;
     new_engine->v2 = t->v2;
     SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback,
@@ -752,6 +752,7 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
     const int files_id = DetectBufferTypeGetByName("files");
     bool head_is_mpm = false;
     uint8_t last_id = DE_STATE_FLAG_BASE;
+    SCLogDebug("%u: setup app inspect engines. %u buffers", s->id, s->init_data->buffer_index);
 
     for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
         SigMatchData *smd = SigMatchList2DataArray(s->init_data->buffers[x].head);
@@ -800,6 +801,39 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
         }
     }
 
+    /* handle rules that have an app-layer hook w/o bringing their own app inspect engine,
+     * e.g. `alert dns:request_complete ... (sid:1;)`
+     *
+     * Here we use a minimal stub inspect engine in which we set:
+     * - alproto
+     * - progress
+     * - sm_list/sm_list_base to get the mapping to the hook name
+     * - dir based on sig direction
+     *
+     * The inspect engine has no callback and is thus considered a straight match.
+     */
+    if (s->init_data->buffer_index == 0 && s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+        uint8_t dir = 0;
+        if ((s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) ==
+                (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT))
+            abort();
+        if ((s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) == 0)
+            abort();
+        if (s->flags & SIG_FLAG_TOSERVER)
+            dir = 0;
+        else if (s->flags & SIG_FLAG_TOCLIENT)
+            dir = 1;
+
+        DetectEngineAppInspectionEngine t = {
+            .alproto = s->init_data->hook.t.app.alproto,
+            .progress = (uint16_t)s->init_data->hook.t.app.app_progress,
+            .sm_list = (uint16_t)s->init_data->hook.sm_list,
+            .sm_list_base = (uint16_t)s->init_data->hook.sm_list,
+            .dir = dir,
+        };
+        AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm);
+    }
+
     if ((s->init_data->init_flags & SIG_FLAG_INIT_STATE_MATCH) &&
             s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL)
     {

src/detect-parse.c

@@ -1107,6 +1107,173 @@ error:
     return -1;
 }
 
+static bool IsBuiltIn(const char *n)
+{
+    if (strcmp(n, "request_started") == 0 || strcmp(n, "response_started") == 0) {
+        return true;
+    }
+    if (strcmp(n, "request_complete") == 0 || strcmp(n, "response_complete") == 0) {
+        return true;
+    }
+    return false;
+}
+
+/** \brief register app hooks as generic lists
+ *
+ *  Register each hook in each app protocol as:
+ *  <alproto>:<hook name>:generic
+ *  These lists can be used by lua scripts to hook into.
+ *
+ *  \todo move elsewhere? maybe a detect-engine-hook.c?
+ */
+void DetectRegisterAppLayerHookLists(void)
+{
+    for (AppProto a = ALPROTO_FAILED + 1; a < g_alproto_max; a++) {
+        const char *alproto_name = AppProtoToString(a);
+        if (strcmp(alproto_name, "http") == 0)
+            alproto_name = "http1";
+        SCLogDebug("alproto %u/%s", a, alproto_name);
+
+        const int max_progress_ts =
+                AppLayerParserGetStateProgressCompletionStatus(a, STREAM_TOSERVER);
+        const int max_progress_tc =
+                AppLayerParserGetStateProgressCompletionStatus(a, STREAM_TOCLIENT);
+
+        char ts_tx_started[64];
+        snprintf(ts_tx_started, sizeof(ts_tx_started), "%s:request_started:generic", alproto_name);
+        DetectAppLayerInspectEngineRegister(
+                ts_tx_started, a, SIG_FLAG_TOSERVER, 0, DetectEngineInspectGenericList, NULL);
+        SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, "request_name", ts_tx_started,
+                (uint32_t)strlen(ts_tx_started));
+
+        char tc_tx_started[64];
+        snprintf(tc_tx_started, sizeof(tc_tx_started), "%s:response_started:generic", alproto_name);
+        DetectAppLayerInspectEngineRegister(
+                tc_tx_started, a, SIG_FLAG_TOCLIENT, 0, DetectEngineInspectGenericList, NULL);
+        SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, "response_name", tc_tx_started,
+                (uint32_t)strlen(tc_tx_started));
+
+        char ts_tx_complete[64];
+        snprintf(ts_tx_complete, sizeof(ts_tx_complete), "%s:request_complete:generic",
+                alproto_name);
+        DetectAppLayerInspectEngineRegister(ts_tx_complete, a, SIG_FLAG_TOSERVER, max_progress_ts,
+                DetectEngineInspectGenericList, NULL);
+        SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, "request_name", ts_tx_complete,
+                (uint32_t)strlen(ts_tx_complete));
+
+        char tc_tx_complete[64];
+        snprintf(tc_tx_complete, sizeof(tc_tx_complete), "%s:response_complete:generic",
+                alproto_name);
+        DetectAppLayerInspectEngineRegister(tc_tx_complete, a, SIG_FLAG_TOCLIENT, max_progress_tc,
+                DetectEngineInspectGenericList, NULL);
+        SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, "response_name", tc_tx_complete,
+                (uint32_t)strlen(tc_tx_complete));
+
+        for (int p = 0; p <= max_progress_ts; p++) {
+            const char *name = AppLayerParserGetStateNameById(
+                    IPPROTO_TCP /* TODO no ipproto */, a, p, STREAM_TOSERVER);
+            if (name != NULL && !IsBuiltIn(name)) {
+                char list_name[64];
+                snprintf(list_name, sizeof(list_name), "%s:%s:generic", alproto_name, name);
+                SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, name, list_name,
+                        (uint32_t)strlen(list_name));
+
+                DetectAppLayerInspectEngineRegister(
+                        list_name, a, SIG_FLAG_TOSERVER, p, DetectEngineInspectGenericList, NULL);
+            }
+        }
+        for (int p = 0; p <= max_progress_tc; p++) {
+            const char *name = AppLayerParserGetStateNameById(
+                    IPPROTO_TCP /* TODO no ipproto */, a, p, STREAM_TOCLIENT);
+            if (name != NULL && !IsBuiltIn(name)) {
+                char list_name[64];
+                snprintf(list_name, sizeof(list_name), "%s:%s:generic", alproto_name, name);
+                SCLogDebug("- hook %s:%s list %s (%u)", alproto_name, name, list_name,
+                        (uint32_t)strlen(list_name));
+
+                DetectAppLayerInspectEngineRegister(
+                        list_name, a, SIG_FLAG_TOCLIENT, p, DetectEngineInspectGenericList, NULL);
+            }
+        }
+    }
+}
+
+static const char *SignatureHookTypeToString(enum SignatureHookType t)
+{
+    switch (t) {
+        case SIGNATURE_HOOK_TYPE_NOT_SET:
+            return "not_set";
+        case SIGNATURE_HOOK_TYPE_APP:
+            return "app";
+            // case SIGNATURE_HOOK_TYPE_PKT:
+            //     return "pkt";
+    }
+    return "unknown";
+}
+
+static SignatureHook SetAppHook(const AppProto alproto, int progress)
+{
+    SignatureHook h = {
+        .type = SIGNATURE_HOOK_TYPE_APP,
+        .t.app.alproto = alproto,
+        .t.app.app_progress = progress,
+    };
+    return h;
+}
+
+/**
+ * \param proto_hook string of protocol and hook, e.g. dns:request_complete
+ */
+static int SigParseProtoHookApp(Signature *s, const char *proto_hook, const char *p, const char *h)
+{
+    if (strcmp(h, "request_started") == 0) {
+        s->flags |= SIG_FLAG_TOSERVER;
+        s->init_data->hook =
+                SetAppHook(s->alproto, 0); // state 0 should be the starting state in each protocol.
+    } else if (strcmp(h, "response_started") == 0) {
+        s->flags |= SIG_FLAG_TOCLIENT;
+        s->init_data->hook =
+                SetAppHook(s->alproto, 0); // state 0 should be the starting state in each protocol.
+    } else if (strcmp(h, "request_complete") == 0) {
+        s->flags |= SIG_FLAG_TOSERVER;
+        s->init_data->hook = SetAppHook(s->alproto,
+                AppLayerParserGetStateProgressCompletionStatus(s->alproto, STREAM_TOSERVER));
+    } else if (strcmp(h, "response_complete") == 0) {
+        s->flags |= SIG_FLAG_TOCLIENT;
+        s->init_data->hook = SetAppHook(s->alproto,
+                AppLayerParserGetStateProgressCompletionStatus(s->alproto, STREAM_TOCLIENT));
+    } else {
+        const int progress_ts = AppLayerParserGetStateIdByName(
+                IPPROTO_TCP /* TODO */, s->alproto, h, STREAM_TOSERVER);
+        if (progress_ts >= 0) {
+            s->flags |= SIG_FLAG_TOSERVER;
+            s->init_data->hook = SetAppHook(s->alproto, progress_ts);
+        } else {
+            const int progress_tc = AppLayerParserGetStateIdByName(
+                    IPPROTO_TCP /* TODO */, s->alproto, h, STREAM_TOCLIENT);
+            if (progress_tc < 0) {
+                return -1;
+            }
+            s->flags |= SIG_FLAG_TOCLIENT;
+            s->init_data->hook = SetAppHook(s->alproto, progress_tc);
+        }
+    }
+
+    char generic_hook_name[64];
+    snprintf(generic_hook_name, sizeof(generic_hook_name), "%s:generic", proto_hook);
+    int list = DetectBufferTypeGetByName(generic_hook_name);
+    if (list < 0) {
+        SCLogError("no list registered as %s for hook %s", generic_hook_name, proto_hook);
+        return -1;
+    }
+    s->init_data->hook.sm_list = list;
+
+    SCLogNotice("protocol:%s hook:%s: type:%s alproto:%u hook:%d", p, h,
+            SignatureHookTypeToString(s->init_data->hook.type), s->init_data->hook.t.app.alproto,
+            s->init_data->hook.t.app.app_progress);
+    return 0;
+}
+
 /**
  * \brief Parses the protocol supplied by the Signature.
  *
@@ -1122,15 +1289,41 @@ error:
 static int SigParseProto(Signature *s, const char *protostr)
 {
     SCEnter();
+    if (strlen(protostr) > 32)
+        return -1;
+
+    char proto[33];
+    strlcpy(proto, protostr, 33);
+    const char *p = proto;
+    const char *h = NULL;
 
-    int r = DetectProtoParse(&s->proto, (char *)protostr);
+    bool has_hook = strchr(proto, ':') != NULL;
+    if (has_hook) {
+        char *xsaveptr = NULL;
+        p = strtok_r(proto, ":", &xsaveptr);
+        h = strtok_r(NULL, ":", &xsaveptr);
+        SCLogDebug("p: '%s' h: '%s'", p, h);
+    }
+    if (p == NULL) {
+        SCLogError("invalid protocol specification '%s'", proto);
+        return -1;
+    }
+
+    int r = DetectProtoParse(&s->proto, p);
     if (r < 0) {
-        s->alproto = AppLayerGetProtoByName((char *)protostr);
+        s->alproto = AppLayerGetProtoByName(p);
         /* indicate that the signature is app-layer */
         if (s->alproto != ALPROTO_UNKNOWN) {
             s->flags |= SIG_FLAG_APPLAYER;
 
             AppLayerProtoDetectSupportedIpprotos(s->alproto, s->proto.proto);
+
+            if (h) {
+                if (SigParseProtoHookApp(s, protostr, p, h) < 0) {
+                    SCLogError("protocol \"%s\" does not support hook \"%s\"", p, h);
+                    SCReturnInt(-1);
+                }
+            }
         }
         else {
             SCLogError("protocol \"%s\" cannot be used "
@@ -1138,7 +1331,7 @@ static int SigParseProto(Signature *s, const char *protostr)
                        "is not yet supported OR detection has been disabled for "
                        "protocol through the yaml option "
                        "app-layer.protocols.%s.detection-enabled",
-                    protostr, protostr);
+                    p, p);
             SCReturnInt(-1);
         }
     }
@@ -2186,6 +2379,7 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
                         DetectEngineBufferTypeGetNameById(de_ctx, app->sm_list), app->dir,
                         app->alproto);
                 SCLogDebug("b->id %d nlists %d", b->id, nlists);
+
                 if (b->only_tc) {
                     if (app->dir == 1)
                         tc_excl++;
@@ -2196,6 +2390,22 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
                     bufdir[b->id].ts += (app->dir == 0);
                     bufdir[b->id].tc += (app->dir == 1);
                 }
+
+                /* only allow rules to use the hook for engines at that
+                 * exact progress for now. */
+                if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
+                    if ((s->flags & SIG_FLAG_TOSERVER) && (app->dir == 0) &&
+                            app->progress != s->init_data->hook.t.app.app_progress) {
+                        SCLogError("engine progress value %d doesn't match hook %u", app->progress,
+                                s->init_data->hook.t.app.app_progress);
+                        SCReturnInt(0);
+                    }
+                    if ((s->flags & SIG_FLAG_TOCLIENT) && (app->dir == 1) &&
+                            app->progress != s->init_data->hook.t.app.app_progress) {
+                        SCLogError("engine progress value doesn't match hook");
+                        SCReturnInt(0);
+                    }
+                }
             }
         }
 

src/detect-parse.h

@@ -122,5 +122,6 @@ int SC_Pcre2SubstringGet(pcre2_match_data *match_data, uint32_t number, PCRE2_UC
         PCRE2_SIZE *bufflen);
 
 int DetectSetupDirection(Signature *s, const char *str);
+void DetectRegisterAppLayerHookLists(void);
 
 #endif /* SURICATA_DETECT_PARSE_H */

src/detect.c

@@ -1113,6 +1113,16 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
             if (unlikely(engine->stream && can->stream_stored)) {
                 match = can->stream_result;
                 TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match);
+            } else if (engine->v2.Callback == NULL) {
+                /* TODO is this the cleanest way to support a non-app sig on a app hook? */
+
+                /* we don't have to store a "hook" match, also don't want to keep any state to make
+                 * sure the hook gets invoked again until tx progress progresses. */
+                if (tx->tx_progress <= engine->progress)
+                    return DETECT_ENGINE_INSPECT_SIG_MATCH;
+
+                /* if progress > engine progress, track state to avoid additional matches */
+                match = DETECT_ENGINE_INSPECT_SIG_MATCH;
             } else {
                 KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list);
                 DEBUG_VALIDATE_BUG_ON(engine->v2.Callback == NULL);

src/detect.h

@@ -547,9 +547,32 @@ typedef struct SignatureInitDataBuffer_ {
     SigMatch *tail;
 } SignatureInitDataBuffer;
 
+enum SignatureHookType {
+    SIGNATURE_HOOK_TYPE_NOT_SET,
+    // SIGNATURE_HOOK_TYPE_PKT,
+    SIGNATURE_HOOK_TYPE_APP,
+};
+
+// dns:request_complete should add DetectBufferTypeGetByName("dns:request_complete");
+// TODO to json
+typedef struct SignatureHook_ {
+    enum SignatureHookType type;
+    int sm_list; /**< list id for the hook's generic list. e.g. for dns:request_complete:generic */
+    union {
+        struct {
+            AppProto alproto;
+            /** progress value of the app-layer hook specified in the rule. Sets the app_proto
+             *  specific progress value. */
+            int app_progress;
+        } app;
+    } t;
+} SignatureHook;
+
 #define SIG_ALPROTO_MAX 4
 
 typedef struct SignatureInitData_ {
+    SignatureHook hook;
+
     /** Number of sigmatches. Used for assigning SigMatch::idx */
     uint16_t sm_cnt;