detect/transforms: disqualify variable-key transforms from prefilter

A transform that derives its key material from a runtime byte_extract or
byte_math variable cannot have its buffer precomputed at load time, so the
content of that buffer is unusable as a fast_pattern. Add the engine
plumbing to recognise such transforms and keep them out of prefilter/MPM.

SignatureInitData gains a var_transform flag. RetrieveFPForSig bails out
for any signature that sets it, so the signature falls back to full
inspection rather than contributing an MPM pattern that can't be built
ahead of time.

New helpers support a transform consuming a variable:

- SCSignatureSetVarTransform() raises the disqualification flag.
- SCDetectByteVarResolve() resolves a byte_extract or byte_math variable
  by name to its runtime slot, and for a byte_extract reports its natural
  key width (a byte_math result has none).
- SCDetectEngineThreadCtxGetByteVar() reads a byte_* variable's runtime
  value from the thread context.

The helpers are declared in detect-engine-buffer.h so the bindgen run
picks them up; the Rust bindings are regenerated to match.

Issue: 7847

rust/sys/src/sys.rs

@@ -689,6 +689,21 @@ extern "C" {
         det_ctx: *mut DetectEngineThreadCtx, id: ::std::os::raw::c_int,
     ) -> *mut ::std::os::raw::c_void;
 }
+extern "C" {
+    #[doc = " \\brief Flag the signature as carrying a transform whose key comes from a\n  runtime variable; this disqualifies the signature from prefilter/MPM."]
+    pub fn SCSignatureSetVarTransform(s: *mut Signature);
+}
+extern "C" {
+    #[doc = " \\brief Resolve a byte variable (byte_extract or byte_math) by name for use\n        as transform input.\n\n Returns the runtime slot (local_id) for the named variable. For a\n byte_extract variable, nbytes is the natural key width (the number of bytes\n the keyword reads); a byte_math result is a computed value with no inherent\n width, so nbytes is set to 0 and the caller must supply an explicit width.\n\n \\retval true  variable found; local_id set, nbytes is the natural width or 0\n \\retval false no such byte variable"]
+    pub fn SCDetectByteVarResolve(
+        s: *const Signature, name: *const ::std::os::raw::c_char, local_id: *mut u8,
+        nbytes: *mut u8,
+    ) -> bool;
+}
+extern "C" {
+    #[doc = " \\brief Read a byte_* variable's runtime value from the thread context."]
+    pub fn SCDetectEngineThreadCtxGetByteVar(det_ctx: *const DetectEngineThreadCtx, id: u8) -> u64;
+}
 #[repr(C)]
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub struct InspectionBuffer {

src/detect-byte.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2020 Open Information Security Foundation
+/* Copyright (C) 2020-2026 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -26,6 +26,7 @@
 #include "detect-byte.h"
 #include "detect-byte-extract.h"
 #include "detect-bytemath.h"
+#include "detect-engine-buffer.h"
 
 /**
  * \brief Used to retrieve args from BM.
@@ -54,3 +55,35 @@ bool DetectByteRetrieveSMVar(
     }
     return false;
 }
+
+bool SCDetectByteVarResolve(
+        const Signature *s, const char *name, uint8_t *local_id, uint8_t *nbytes)
+{
+    /* byte_extract carries a natural key width (the number of bytes it reads),
+     * so resolve it directly to get both the runtime slot and that width. */
+    SigMatch *bed_sm = DetectByteExtractRetrieveSMVar(name, s->init_data->list, s);
+    if (bed_sm != NULL) {
+        const SCDetectByteExtractData *bed = (const SCDetectByteExtractData *)bed_sm->ctx;
+        *local_id = bed->local_id;
+        /* In string mode nbytes is the count of input characters parsed, not the
+         * byte width of the resulting value, so it is not a usable key width;
+         * report 0 and let the caller require an explicit width. */
+        *nbytes = (bed->flags & DETECT_BYTE_EXTRACT_FLAG_STRING) ? 0 : bed->nbytes;
+        return true;
+    }
+
+    /* a byte_math result is a computed value with no inherent width, so report
+     * 0 and let the caller require an explicit width. */
+    SigMatch *bmd_sm = DetectByteMathRetrieveSMVar(name, s->init_data->list, s);
+    if (bmd_sm != NULL) {
+        *local_id = ((const DetectByteMathData *)bmd_sm->ctx)->local_id;
+        *nbytes = 0;
+        return true;
+    }
+    return false;
+}
+
+uint64_t SCDetectEngineThreadCtxGetByteVar(const DetectEngineThreadCtx *det_ctx, uint8_t id)
+{
+    return det_ctx->byte_values[id];
+}

src/detect-engine-buffer.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2025 Open Information Security Foundation
+/* Copyright (C) 2025-2026 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -197,3 +197,8 @@ int SCDetectSignatureAddTransform(Signature *s, int transform, void *options)
 
     SCReturnInt(0);
 }
+
+void SCSignatureSetVarTransform(Signature *s)
+{
+    s->init_data->var_transform = true;
+}

src/detect-engine-buffer.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2025 Open Information Security Foundation
+/* Copyright (C) 2025-2026 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -43,4 +43,26 @@ int SCDetectRegisterThreadCtxGlobalFuncs(
         const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *));
 void *SCDetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id);
 
+/** \brief Flag the signature as carrying a transform whose key comes from a
+ *  runtime variable; this disqualifies the signature from prefilter/MPM. */
+void SCSignatureSetVarTransform(Signature *s);
+
+/**
+ * \brief Resolve a byte variable (byte_extract or byte_math) by name for use
+ *        as transform input.
+ *
+ * Returns the runtime slot (local_id) for the named variable. For a
+ * byte_extract variable, nbytes is the natural key width (the number of bytes
+ * the keyword reads); a byte_math result is a computed value with no inherent
+ * width, so nbytes is set to 0 and the caller must supply an explicit width.
+ *
+ * \retval true  variable found; local_id set, nbytes is the natural width or 0
+ * \retval false no such byte variable
+ */
+bool SCDetectByteVarResolve(
+        const Signature *s, const char *name, uint8_t *local_id, uint8_t *nbytes);
+
+/** \brief Read a byte_* variable's runtime value from the thread context. */
+uint64_t SCDetectEngineThreadCtxGetByteVar(const DetectEngineThreadCtx *det_ctx, uint8_t id);
+
 #endif /* SURICATA_DETECT_ENGINE_BUFFER_H */

src/detect-engine-mpm.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2021 Open Information Security Foundation
+/* Copyright (C) 2007-2026 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -1186,6 +1186,15 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s)
     if (s->init_data->mpm_sm != NULL)
         return;
 
+    /* A transform that derives its key from a runtime byte_extract or byte_math
+     * variable cannot have its buffer precomputed at load time, so the content
+     * of such a buffer is unusable as a fast_pattern. Disqualify the whole
+     * signature from prefilter; it falls back to full inspection. */
+    if (s->init_data->var_transform) {
+        SCLogDebug("sid %u uses a variable-key transform; skipping fast_pattern", s->id);
+        return;
+    }
+
     const int nlists = s->init_data->max_content_list_id + 1;
     DEBUG_VALIDATE_BUG_ON(nlists > UINT16_MAX);
     int pos_sm_list[nlists];

src/detect.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2025 Open Information Security Foundation
+/* Copyright (C) 2007-2026 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -676,6 +676,12 @@ typedef struct SignatureInitData_ {
 
     /* Signature is a "firewall" rule. */
     bool firewall_rule;
+
+    /* True if a transform on one of this signature's buffers reads its key
+     * material from a runtime byte_extract or byte_math variable. Such a buffer
+     * cannot be precomputed, so the signature is disqualified from
+     * prefilter/MPM. */
+    bool var_transform;
 } SignatureInitData;
 
 /** \brief Signature container */