detect/firewall: implement initial state range support

Allow a single rule to accept a hook and the hooks prior to it.

Example:

        accept:flow tls:<client_hello_done ... \
                tls.sni; content:"suricata.io"; endswith;

This will evaluate the SNI at the client_hello_done hook, but will
act as if there is a `accept:hook tls:client_in_progress ...` as well.

Implementation is currently specific to this `<` operator. During
parsing the sig gets flagged for this case. During setup this has 3 main
effects:

1. prefilter is disabled as we need to eval this right at the first
   state (0)
2. for state 0 a non-PF "prefilter" engine is setup to make sure the
   rule is flagged for evaluation
3. In the Signature::app_inspect list a dummy inspect engine is
   registered per state before Signature::app_progress_hook

The matching logic is building on the stateful rule handling. The
stateful rule handling can now tell the inspection loop that a partial
match occured. For this rule type the partial match will act as a match
with action accept:hook.

Next app updates will then use the continue detection logic to continue
the stateful match. When that fully matches, the final actions are
applied, like accept:flow or accept:tx.

Ticket: #8472.
(cherry picked from commit 651afba883)

src/detect-engine-prefilter.c

@@ -936,6 +936,28 @@ static int SetupNonPrefilter(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
             continue; // done for this sig
         }
 
+        /* special case: insert sigs at hook before Signature::app_progress_hook for the HOOK_LTE
+         * case: we need a addition per hook to make sure that the sig is called when needed. For
+         * hook 0 it could have a preceding rule that makes sure this sig isn't triggered, but then
+         * for hook 1 we would need to be called. */
+        if (s->flags & SIG_FLAG_FW_HOOK_LTE) {
+            for (uint8_t state = 0; state < s->app_progress_hook; state++) {
+                SCLogDebug("handle HOOK %u LTE", state);
+                const int dir = (s->flags & SIG_FLAG_TOSERVER) ? 0 : 1;
+                const char *pname = AppLayerParserGetStateNameById(IPPROTO_TCP, // TODO
+                        s->alproto, state, dir == 0 ? STREAM_TOSERVER : STREAM_TOCLIENT);
+                if (pname == NULL)
+                    goto error;
+                const int sm_list = DetectEngineAppHookToSmlist(
+                        s->alproto, state, dir == 0 ? STREAM_TOSERVER : STREAM_TOCLIENT);
+                if (TxNonPFAddSig(de_ctx, tx_engines_hash, s->alproto, dir, (int16_t)state, sm_list,
+                            pname, s) != 0) {
+                    goto error;
+                }
+                tx_non_pf = true;
+            }
+        }
+
         for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
             const int list_id = s->init_data->buffers[x].id;
             const DetectBufferType *buf = DetectEngineBufferTypeGetById(de_ctx, list_id);

src/detect-engine.c

@@ -794,6 +794,32 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     s->init_data->init_flags |= SIG_FLAG_INIT_STATE_MATCH;
 }
 
+/** \brief get the sm_list for a app hook */
+int DetectEngineAppHookToSmlist(const AppProto p, const uint8_t state, const int direction)
+{
+    const char *app_proto = AppProtoToString(p);
+    if (app_proto == NULL) {
+        SCLogError("unknown app_proto %u", p);
+        return -1;
+    }
+    if (strcmp(app_proto, "http") == 0)
+        app_proto = "http1";
+
+    const char *name = AppLayerParserGetStateNameById(
+            IPPROTO_TCP, p, state, direction & (STREAM_TOSERVER | STREAM_TOCLIENT));
+    if (name == NULL)
+        return -1;
+
+    char generic_hook_name[256];
+    snprintf(generic_hook_name, sizeof(generic_hook_name), "%s:%s:generic", app_proto, name);
+    int list = DetectBufferTypeGetByName(generic_hook_name);
+    if (list < 0) {
+        SCLogError("no list registered as %s for %s hook %s", generic_hook_name, app_proto, name);
+        return -1;
+    }
+    return list;
+}
+
 /**
  *  \note for the file inspect engine, the id DE_STATE_ID_FILE_INSPECT
  *        is assigned.
@@ -806,6 +832,41 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
     uint8_t last_id = DE_STATE_FLAG_BASE;
     SCLogDebug("%u: setup app inspect engines. %u buffers", s->id, s->init_data->buffer_index);
 
+    if (s->flags & SIG_FLAG_FW_HOOK_LTE) {
+        SCLogDebug("need an inspect engine per state, range 0-%u", s->app_progress_hook);
+        for (uint8_t state = 0; state < s->app_progress_hook; state++) {
+            uint8_t dir = 0;
+            int direction = 0;
+            BUG_ON((s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) ==
+                    (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT));
+            BUG_ON((s->flags & (SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT)) == 0);
+            if (s->flags & SIG_FLAG_TOSERVER) {
+                direction = STREAM_TOSERVER;
+                dir = 0;
+            } else if (s->flags & SIG_FLAG_TOCLIENT) {
+                direction = STREAM_TOCLIENT;
+                dir = 1;
+            }
+
+            int sm_list =
+                    DetectEngineAppHookToSmlist(s->init_data->hook.t.app.alproto, 0, direction);
+            if (sm_list < 0)
+                return -1;
+
+            DetectEngineAppInspectionEngine t = {
+                .alproto = s->init_data->hook.t.app.alproto,
+                .progress = (uint16_t)state,
+                .sm_list = (uint16_t)sm_list,
+                .sm_list_base = (uint16_t)sm_list,
+                .dir = dir,
+            };
+            AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm);
+            SCLogDebug("sid %u: appended pass-tru engine at hook:%u sm_list:%d for "
+                       "SIG_FLAG_INIT_HOOK_LTE",
+                    s->id, state, sm_list);
+        }
+    }
+
     for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
         SigMatchData *smd = SigMatchList2DataArray(s->init_data->buffers[x].head);
         SCLogDebug("smd %p, id %u", smd, s->init_data->buffers[x].id);

src/detect-engine.h

@@ -209,4 +209,6 @@ bool DetectMd5ValidateCallback(
 
 void DeStateRegisterTests(void);
 
+int DetectEngineAppHookToSmlist(const AppProto p, const uint8_t state, const int direction);
+
 #endif /* SURICATA_DETECT_ENGINE_H */

src/detect-parse.c

@@ -1315,6 +1315,7 @@ static SignatureHook SetAppHook(const AppProto alproto, int progress)
  */
 static int SigParseProtoHookApp(Signature *s, const char *proto_hook, const char *p, const char *h)
 {
+    SCLogDebug("h:'%s'", h);
     if (strcmp(h, "request_started") == 0) {
         s->flags |= SIG_FLAG_TOSERVER;
         s->init_data->hook =
@@ -1349,7 +1350,7 @@ static int SigParseProtoHookApp(Signature *s, const char *proto_hook, const char
     }
 
     char generic_hook_name[64];
-    snprintf(generic_hook_name, sizeof(generic_hook_name), "%s:generic", proto_hook);
+    snprintf(generic_hook_name, sizeof(generic_hook_name), "%s:%s:generic", p, h);
     int list = DetectBufferTypeGetByName(generic_hook_name);
     if (list < 0) {
         SCLogError("no list registered as %s for hook %s", generic_hook_name, proto_hook);
@@ -1410,6 +1411,13 @@ static int SigParseProto(Signature *s, const char *protostr)
             AppLayerProtoDetectSupportedIpprotos(s->alproto, s->proto.proto);
 
             if (h) {
+                /* FW hook LTE mode */
+                SCLogDebug("hook '%s'", h);
+                if (*h == '<') {
+                    h++;
+                    SCLogDebug("hook and prior hooks: '%s'", h);
+                    s->flags |= SIG_FLAG_FW_HOOK_LTE;
+                }
                 if (SigParseProtoHookApp(s, protostr, p, h) < 0) {
                     SCLogError("protocol \"%s\" does not support hook \"%s\"", p, h);
                     SCReturnInt(-1);
@@ -2436,6 +2444,11 @@ static void SigSetupPrefilter(DetectEngineCtx *de_ctx, Signature *s)
     SCLogDebug("s %u: set up prefilter/mpm", s->id);
     DEBUG_VALIDATE_BUG_ON(s->init_data->mpm_sm != NULL);
 
+    if (s->flags & SIG_FLAG_FW_HOOK_LTE) {
+        SCLogDebug("no prefilter for SIG_FLAG_FW_HOOK_LTE sig");
+        SCReturn;
+    }
+
     if (s->init_data->prefilter_sm != NULL) {
         if (s->init_data->prefilter_sm->type == DETECT_CONTENT) {
             RetrieveFPForSig(de_ctx, s);

src/detect.c

@@ -1239,27 +1239,21 @@ void *DetectGetInnerTx(void *tx_ptr, AppProto alproto, AppProto engine_alproto,
  *         If stored_flags is set it means we're continuing
  *         inspection from an earlier run.
  *
- *  \retval bool true sig matched, false didn't match
+ *  \retval  1 sig matched
+ *  \retval  0 partial incomplete match
+ *  \retval -1 failed to match
  */
-static bool DetectRunTxInspectRule(ThreadVars *tv,
-        DetectEngineCtx *de_ctx,
-        DetectEngineThreadCtx *det_ctx,
-        Packet *p,
-        Flow *f,
-        const uint8_t in_flow_flags,   // direction, EOF, etc
-        void *alstate,
-        DetectTransaction *tx,
-        const Signature *s,
-        uint32_t *stored_flags,
-        RuleMatchCandidateTx *can,
-        DetectRunScratchpad *scratch)
+static int DetectRunTxInspectRule(ThreadVars *tv, DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f,
+        const uint8_t in_flow_flags, // direction, EOF, etc
+        void *alstate, DetectTransaction *tx, const Signature *s, uint32_t *stored_flags,
+        RuleMatchCandidateTx *can, DetectRunScratchpad *scratch)
 {
     const uint8_t flow_flags = in_flow_flags;
     const int direction = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
     uint32_t inspect_flags = stored_flags ? *stored_flags : 0;
     int total_matches = 0;
     uint16_t file_no_match = 0;
-    bool retval = false;
     bool mpm_before_progress = false;   // is mpm engine before progress?
     bool mpm_in_progress = false;       // is mpm engine in a buffer we will revisit?
 
@@ -1270,17 +1264,19 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
         TRACE_SID_TXS(s->id, tx, "first inspect, run packet matches");
         if (!DetectRunInspectRuleHeader(p, f, s, s->flags, s->proto.flags)) {
             TRACE_SID_TXS(s->id, tx, "DetectRunInspectRuleHeader() no match");
-            return false;
+            return -1;
         }
         if (!DetectEnginePktInspectionRun(tv, det_ctx, s, f, p, NULL)) {
             TRACE_SID_TXS(s->id, tx, "DetectEnginePktInspectionRun no match");
-            return false;
+            return -1;
         }
         /* stream mpm and negated mpm sigs can end up here with wrong proto */
         if (!(AppProtoEquals(s->alproto, f->alproto) || s->alproto == ALPROTO_UNKNOWN)) {
             TRACE_SID_TXS(s->id, tx, "alproto mismatch");
-            return false;
+            return -1;
         }
+    } else {
+        TRACE_SID_TXS(s->id, tx, "continue, inspect_flags %x", inspect_flags);
     }
 
     const DetectEngineAppInspectionEngine *engine = s->app_inspect;
@@ -1346,8 +1342,9 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
 
                 /* we don't have to store a "hook" match, also don't want to keep any state to make
                  * sure the hook gets invoked again until tx progress progresses. */
-                if (tx->tx_progress <= engine->progress)
-                    return DETECT_ENGINE_INSPECT_SIG_MATCH;
+                if ((s->flags & SIG_FLAG_FW_HOOK_LTE) == 0 && tx->tx_progress <= engine->progress) {
+                    return 1; // DETECT_ENGINE_INSPECT_SIG_MATCH;
+                }
 
                 /* if progress > engine progress, track state to avoid additional matches */
                 match = DETECT_ENGINE_INSPECT_SIG_MATCH;
@@ -1407,10 +1404,11 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
     TRACE_SID_TXS(s->id, tx, "inspect_flags %x, total_matches %u, engine %p",
             inspect_flags, total_matches, engine);
 
+    bool full_match = false;
     if (engine == NULL && total_matches) {
         inspect_flags |= DE_STATE_FLAG_FULL_INSPECT;
         TRACE_SID_TXS(s->id, tx, "MATCH");
-        retval = true;
+        full_match = true;
     }
 
     if (stored_flags) {
@@ -1448,14 +1446,27 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
         } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) {
             TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, "
                     "mpm will revisit it");
+            return -1; /* no match */
         } else if (inspect_flags != 0 || file_no_match != 0) {
             TRACE_SID_TXS(s->id, tx, "storing state: flags %08x", inspect_flags);
             DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s,
                     inspect_flags, flow_flags, file_no_match);
+        } else {
+            if (inspect_flags == 0) {
+                TRACE_SID_TXS(s->id, tx, "no match: inspect_flags %08x", inspect_flags);
+                return -1;
+            }
         }
     }
-
-    return retval;
+    if (full_match) {
+        return 1;
+        /* can't be a partial match if we're at the end state */
+    } else if ((inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) == 0 &&
+               tx->tx_progress < tx->tx_end_state) {
+        return 0;
+    } else {
+        return -1;
+    }
 }
 
 #define NO_TX                                                                                      \
@@ -1755,6 +1766,10 @@ static enum DetectTxFirewallFlowControl DetectRunTxPreCheckFirewallPolicy(
         fw_state->fw_next_progress_missing = false;
         fw_state->fw_last_for_progress = false;
 
+        if (s->flags & SIG_FLAG_FW_HOOK_LTE) {
+            SCLogDebug("SIG_FLAG_FW_HOOK_LTE");
+            return DETECT_TX_FW_FC_OK; // TODO check for other cases
+        }
         /* if our first rule is beyond the starting state, we need to check if
          * there are rules missing for states in between. */
         // TODO detect_progress_orig already is +1?
@@ -2295,6 +2310,7 @@ static void DetectRunTx(ThreadVars *tv,
             RULE_PROFILING_START(p);
             const int r = DetectRunTxInspectRule(tv, de_ctx, det_ctx, p, f, flow_flags,
                     alstate, &tx, s, inspect_flags, can, scratch);
+            SCLogDebug("s %u r %d", s->id, r);
             if (r == 1) {
                 /* match */
                 DetectRunPostMatch(tv, det_ctx, p, s);
@@ -2327,7 +2343,21 @@ static void DetectRunTx(ThreadVars *tv,
                     }
                 }
                 AlertQueueAppend(det_ctx, s, p, tx.tx_id, alert_flags);
-
+            } else if (r == 0) {
+                SCLogDebug("sid %u partial match", s->id);
+                if ((s->flags & SIG_FLAG_FIREWALL) && (s->action & ACTION_ACCEPT)) {
+                    /* partial match always uses ACTION_SCOPE_HOOK. Final action only on the full
+                     * match */
+                    if (last_tx) {
+                        SCLogDebug("need to apply accept to packet");
+                        DetectRunAppendDefaultAccept(det_ctx, p);
+                    }
+                    if (s->action_scope == ACTION_SCOPE_FLOW) {
+                        SCLogDebug("only applying accept:flow on full match, downgrading to "
+                                   "accept:hook");
+                    }
+                    break;
+                }
             } else if (fw_state.fw_last_for_progress && (s->flags & SIG_FLAG_FIREWALL)) {
                 SCLogDebug("%" PRIu64 ": %s default policy for progress %u", p->pcap_cnt,
                         flow_flags & STREAM_TOSERVER ? "toserver" : "toclient",

src/detect.h

@@ -249,7 +249,7 @@ typedef struct DetectPort_ {
 #define SIG_FLAG_APPLAYER               BIT_U32(6) /**< signature applies to app layer instead of packets */
 #define SIG_FLAG_TXBOTHDIR              BIT_U32(7) /**< signature needs tx with both directions to match */
 
-// vacancy
+#define SIG_FLAG_FW_HOOK_LTE BIT_U32(8) /**< Signature::app_progress_hook is to be used as LTE */
 
 #define SIG_FLAG_REQUIRE_PACKET         BIT_U32(9)  /**< signature is requiring packet match */
 #define SIG_FLAG_REQUIRE_STREAM         BIT_U32(10) /**< signature is requiring stream match */