detect/xor: support a variable key from a byte_extract or byte_math variable

Extend the xor transform to take its key either as a literal hex string
(xor:"<hex>") or as a byte_extract or byte_math variable
(xor:var <name> [<nbytes>]) resolved at inspection time.

For a variable key, setup resolves the variable to its runtime slot and
marks the signature so it stays out of prefilter. At inspection the
transform reads the value from the thread context and renders the low
<nbytes> bytes of that value big-endian, regardless of the keyword's own
endianness.

The optional <nbytes> sets the key width. For a byte_extract variable read
in the default (non-string) mode it defaults to the read width and may be
omitted. A byte_math result, or a string-mode byte_extract, has no inherent
value width, so the width is required there. The buffer-dedup id carries a
leading discriminant so static and variable keys never share an inspection
buffer.

The key option is now optionally quoted (SIGMATCH_QUOTES_OPTIONAL) so the
unquoted 'var <name>' form is accepted alongside the quoted hex key.

Issue: 7847

rust/src/detect/transforms/varkey.rs

@@ -15,8 +15,15 @@
  * 02110-1301, USA.
  */
 
-//! Shared helpers for transforms that read key bytes from a fixed
-//! position within the inspection buffer at transform time.
+//! Shared helpers for transforms whose key is not a literal in the rule.
+//!
+//! Two key sources are supported:
+//!
+//! * a fixed location within the inspection buffer, read at transform time
+//!   ([`VariableKeyLocation`] / [`variable_key_bytes`]); and
+//! * a `byte_extract` or `byte_math` variable, resolved against the signature
+//!   at setup ([`parse_byte_var`] / [`resolve_byte_var`]) and read from the
+//!   thread context at transform time ([`byte_var_key_bytes`]).
 
 use nom8::{
     character::complete::{digit1, multispace1},
@@ -24,6 +31,11 @@ use nom8::{
     sequence::separated_pair,
     IResult, Parser,
 };
+use std::ffi::CString;
+use suricata_sys::sys::{
+    DetectEngineThreadCtx, SCDetectByteVarResolve, SCDetectEngineThreadCtxGetByteVar,
+    SCSignatureSetVarTransform, Signature,
+};
 
 /// Location in an inspection buffer from which key bytes are read at
 /// transform time.
@@ -76,6 +88,127 @@ pub(super) fn parse_key_location(s: &str) -> Option<VariableKeyLocation> {
     }
 }
 
+/// Parsed `var <name> [<nbytes>]` spec, before the variable is resolved
+/// against the signature.
+#[derive(Debug, PartialEq, Eq)]
+pub(super) struct ByteVarSpec {
+    /// The `byte_extract`/`byte_math` variable name.
+    pub(super) name: String,
+    /// Optional explicit key width in bytes; resolved against the variable's
+    /// natural width when omitted.
+    pub(super) width: Option<u8>,
+}
+
+/// A key read from a `byte_extract`/`byte_math` variable at transform time.
+/// `local_id` is the runtime slot; `nbytes` (1-8) selects how many low bytes
+/// of the variable's value form the key, rendered big-endian.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) struct ByteVarKey {
+    pub(super) local_id: u8,
+    pub(super) nbytes: u8,
+}
+
+/// Parse `<name> [<nbytes>]` for the `var` form (the `var` keyword has already
+/// been stripped by the caller). Logs and returns `None` on malformed input.
+pub(super) fn parse_byte_var(rest: &str) -> Option<ByteVarSpec> {
+    let mut parts = rest.split_whitespace();
+    let name = match parts.next() {
+        Some(n) => n,
+        None => {
+            SCLogError!("transform 'var' requires a variable name");
+            return None;
+        }
+    };
+    // Optional explicit key width (number of bytes of the variable's value to
+    // use). Required for byte_math variables, which have no natural width;
+    // optional for byte_extract, which defaults to its read width.
+    let width = match parts.next() {
+        Some(w) => match w.parse::<u8>() {
+            Ok(n) => Some(n),
+            Err(_) => {
+                SCLogError!("transform 'var' width must be a number");
+                return None;
+            }
+        },
+        None => None,
+    };
+    if parts.next().is_some() {
+        SCLogError!("transform 'var' takes a variable name and optional width");
+        return None;
+    }
+    Some(ByteVarSpec {
+        name: name.to_string(),
+        width,
+    })
+}
+
+/// Resolve a parsed byte-variable spec against signature `s`: look up the
+/// variable, settle the key width, validate it, and mark the signature so it
+/// is kept out of prefilter (the key is only known at inspection time).
+/// Returns the compiled key, or `None` on any error (already logged).
+///
+/// # Safety
+///
+/// `s` must be a valid signature pointer; intended to be called from a
+/// transform's `Setup` callback.
+pub(super) unsafe fn resolve_byte_var(s: *mut Signature, spec: &ByteVarSpec) -> Option<ByteVarKey> {
+    let cname = CString::new(spec.name.as_str()).ok()?;
+    let mut local_id: u8 = 0;
+    let mut natural_nbytes: u8 = 0;
+    if !SCDetectByteVarResolve(s, cname.as_ptr(), &mut local_id, &mut natural_nbytes) {
+        SCLogError!(
+            "transform 'var': unknown byte_extract or byte_math variable '{}'",
+            spec.name
+        );
+        return None;
+    }
+    // Use the explicit width if given, otherwise the variable's natural width.
+    // A byte_math result has no natural width (reported as 0), so an explicit
+    // width is required there.
+    let nbytes = match spec.width {
+        Some(w) => w,
+        None => {
+            if natural_nbytes == 0 {
+                SCLogError!(
+                    "transform 'var': variable '{}' has no inherent width; specify the key width, e.g. 'var <name> <nbytes>'",
+                    spec.name
+                );
+                return None;
+            }
+            natural_nbytes
+        }
+    };
+    if nbytes == 0 || nbytes > 8 {
+        SCLogError!("transform 'var': key width must be 1-8 bytes");
+        return None;
+    }
+    /* The key is only known at inspection time, so the buffer can't be
+     * precomputed: disqualify this signature from prefilter/MPM. */
+    SCSignatureSetVarTransform(s);
+    Some(ByteVarKey { local_id, nbytes })
+}
+
+/// Read the byte-variable key's value from the thread context and render its
+/// low `nbytes` big-endian into `out`, returning the written slice. `out` must
+/// be at least `key.nbytes` (<= 8) bytes long. Returns `None` if `det` is null.
+///
+/// # Safety
+///
+/// `det` must be a valid thread-context pointer or null.
+pub(super) unsafe fn byte_var_key_bytes<'a>(
+    det: *mut DetectEngineThreadCtx, key: &ByteVarKey, out: &'a mut [u8],
+) -> Option<&'a [u8]> {
+    if det.is_null() {
+        return None;
+    }
+    // Render the low `nbytes` bytes of the value big-endian, regardless of the
+    // keyword's own endianness.
+    let value = SCDetectEngineThreadCtxGetByteVar(det, key.local_id);
+    let n = key.nbytes as usize;
+    out[..n].copy_from_slice(&value.to_be_bytes()[8 - n..]);
+    Some(&out[..n])
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -85,6 +218,57 @@ mod tests {
         assert_eq!(strip_keyword_prefix("var 1 0", "var"), Some("1 0"));
     }
 
+    #[test]
+    fn test_parse_byte_var_name_only() {
+        // width is resolved later from the variable
+        assert_eq!(
+            parse_byte_var("xkey"),
+            Some(ByteVarSpec {
+                name: "xkey".to_string(),
+                width: None
+            })
+        );
+        // extra whitespace is tolerated
+        assert_eq!(
+            parse_byte_var("  xkey  "),
+            Some(ByteVarSpec {
+                name: "xkey".to_string(),
+                width: None
+            })
+        );
+    }
+
+    #[test]
+    fn test_parse_byte_var_explicit_width() {
+        assert_eq!(
+            parse_byte_var("xkey 4"),
+            Some(ByteVarSpec {
+                name: "xkey".to_string(),
+                width: Some(4)
+            })
+        );
+    }
+
+    #[test]
+    fn test_parse_byte_var_missing_name() {
+        assert!(parse_byte_var("").is_none());
+        assert!(parse_byte_var("   ").is_none());
+    }
+
+    #[test]
+    fn test_parse_byte_var_bad_width() {
+        // width must be numeric
+        assert!(parse_byte_var("xkey wide").is_none());
+        // width must fit in u8
+        assert!(parse_byte_var("xkey 256").is_none());
+    }
+
+    #[test]
+    fn test_parse_byte_var_trailing_garbage() {
+        // more than a name and a width
+        assert!(parse_byte_var("xkey 1 extra").is_none());
+    }
+
     #[test]
     fn test_strip_keyword_prefix_offset() {
         assert_eq!(

rust/src/detect/transforms/xor.rs

@@ -22,7 +22,8 @@ use suricata_sys::sys::{
 };
 
 use super::varkey::{
-    parse_key_location, strip_keyword_prefix, variable_key_bytes, VariableKeyLocation,
+    byte_var_key_bytes, parse_byte_var, parse_key_location, resolve_byte_var, strip_keyword_prefix,
+    variable_key_bytes, ByteVarKey, ByteVarSpec, VariableKeyLocation,
 };
 use std::ffi::CStr;
 use std::os::raw::{c_char, c_int, c_void};
@@ -36,6 +37,8 @@ enum XorKeySource {
     Static(Vec<u8>),
     /// Key read directly from the inspection buffer at a fixed location.
     Variable(VariableKeyLocation),
+    /// Key read from a byte_extract/byte_math variable at inspection time.
+    ByteVar(ByteVarKey),
 }
 
 #[derive(Debug)]
@@ -47,17 +50,21 @@ struct DetectTransformXorData {
     /// Precomputed identity bytes returned by xor_id. Layout:
     ///   Static:   [0x00, key_bytes..., xor_offset_le4]
     ///   Variable: [0x01, key_offset_lo, key_offset_hi, nbytes, xor_offset_le4]
-    /// The leading discriminant byte ensures static and variable identities
-    /// can never collide. Including xor_offset ensures rules with the same
-    /// key but different decode-start positions get independent buffers.
+    ///   ByteVar:  [0x02, local_id, nbytes, xor_offset_le4]
+    /// The leading discriminant byte ensures the three key kinds can never
+    /// collide. Including xor_offset ensures rules with the same key but
+    /// different decode-start positions get independent buffers.
     id_buf: Vec<u8>,
 }
 
-/// Parsed key specifier — either decoded key bytes or an inline buffer location.
+/// Parsed key specifier — decoded key bytes, an inline buffer location, or the
+/// name of a byte_extract/byte_math variable.
 #[derive(Debug, PartialEq)]
 enum XorKeySpec {
     Hex(Vec<u8>),
     Var(VariableKeyLocation),
+    /// `var <name> [<nbytes>]`, resolved against the signature at setup time.
+    ByteVar(ByteVarSpec),
 }
 
 /// Intermediate parse result.
@@ -76,7 +83,8 @@ fn try_parse_hex_key(s: &str) -> Option<Vec<u8>> {
         .filter(|k| !k.is_empty() && k.len() <= usize::from(u8::MAX))
 }
 
-/// Parse a key specifier — either `extract <nbytes> <offset>` or a hex string.
+/// Parse a key specifier — `extract <nbytes> <offset>`, `var <name> [<nbytes>]`,
+/// or a hex string.
 fn parse_key_part(s: &str) -> Option<XorKeySpec> {
     if let Some(rest) = strip_keyword_prefix(s, "extract") {
         if let Some(loc) = parse_key_location(rest) {
@@ -85,6 +93,9 @@ fn parse_key_part(s: &str) -> Option<XorKeySpec> {
         SCLogError!("XOR transform: 'extract' requires format 'extract <nbytes> <offset>'");
         return None;
     }
+    if let Some(rest) = strip_keyword_prefix(s, "var") {
+        return parse_byte_var(rest).map(XorKeySpec::ByteVar);
+    }
     let s = strip_quotes(s);
     if s.is_empty() {
         SCLogError!("XOR transform: missing hex key");
@@ -102,6 +113,7 @@ fn parse_key_part(s: &str) -> Option<XorKeySpec> {
 /// Parse the xor option string. Accepts:
 ///   - `<hex_key>`
 ///   - `extract <nbytes> <offset>`
+///   - `var <name> [<nbytes>]`
 ///   - `offset <N>,<hex_key>`
 ///   - `offset <N>,extract <nbytes> <offset>`
 fn xor_parse_options(input: &str) -> Option<XorParseResult> {
@@ -148,32 +160,29 @@ fn strip_quotes(s: &str) -> &str {
         .unwrap_or(s)
 }
 
-fn xor_build_ctx(input: &str) -> Option<DetectTransformXorData> {
-    let parsed = xor_parse_options(input)?;
-    let xor_offset = parsed.xor_offset.unwrap_or(0);
-    let key_source = match parsed.key_spec {
-        XorKeySpec::Hex(key) => XorKeySource::Static(key),
-        XorKeySpec::Var(loc) => XorKeySource::Variable(loc),
-    };
-    let id_buf = match &key_source {
+/// Build the transform context (including the buffer-dedup identity) from a
+/// resolved key source and decode offset.
+fn build_xor_data(key_source: XorKeySource, xor_offset: u32) -> DetectTransformXorData {
+    // Each arm builds the discriminant + key-specific prefix; the xor_offset
+    // suffix is invariant across kinds, so append it once after the match.
+    let mut id_buf = match &key_source {
         XorKeySource::Static(key) => {
             let mut buf = vec![0x00];
             buf.extend_from_slice(key);
-            buf.extend_from_slice(&xor_offset.to_le_bytes());
             buf
         }
         XorKeySource::Variable(loc) => {
             let [lo, hi] = loc.offset.to_le_bytes();
-            let mut buf = vec![0x01, lo, hi, loc.nbytes];
-            buf.extend_from_slice(&xor_offset.to_le_bytes());
-            buf
+            vec![0x01, lo, hi, loc.nbytes]
         }
+        XorKeySource::ByteVar(key) => vec![0x02, key.local_id, key.nbytes],
     };
-    Some(DetectTransformXorData {
+    id_buf.extend_from_slice(&xor_offset.to_le_bytes());
+    DetectTransformXorData {
         key_source,
         xor_offset,
         id_buf,
-    })
+    }
 }
 
 unsafe extern "C" fn xor_setup(
@@ -183,10 +192,22 @@ unsafe extern "C" fn xor_setup(
         Ok(s) => s,
         Err(_) => return -1,
     };
-    let ctx = match xor_build_ctx(input) {
-        Some(d) => Box::into_raw(Box::new(d)) as *mut c_void,
+    let parsed = match xor_parse_options(input) {
+        Some(p) => p,
         None => return -1,
     };
+    let xor_offset = parsed.xor_offset.unwrap_or(0);
+    let key_source = match parsed.key_spec {
+        XorKeySpec::Hex(key) => XorKeySource::Static(key),
+        XorKeySpec::Var(loc) => XorKeySource::Variable(loc),
+        XorKeySpec::ByteVar(spec) => match resolve_byte_var(s, &spec) {
+            Some(key) => XorKeySource::ByteVar(key),
+            None => return -1,
+        },
+    };
+
+    let ctx = Box::into_raw(Box::new(build_xor_data(key_source, xor_offset))) as *mut c_void;
+
     let r = SCDetectSignatureAddTransform(s, G_TRANSFORM_XOR_ID, ctx);
     if r != 0 {
         xor_free(de, ctx);
@@ -209,7 +230,7 @@ fn xor_transform_do(input: &[u8], output: &mut [u8], key: &[u8], xor_offset: usi
 }
 
 unsafe extern "C" fn xor_transform(
-    _det: *mut DetectEngineThreadCtx, buffer: *mut InspectionBuffer, ctx: *const c_void,
+    det: *mut DetectEngineThreadCtx, buffer: *mut InspectionBuffer, ctx: *const c_void,
 ) {
     let input = (*buffer).inspect;
     let input_len = (*buffer).inspect_len;
@@ -220,6 +241,7 @@ unsafe extern "C" fn xor_transform(
 
     let output = SCInspectionBufferCheckAndExpand(buffer, input_len);
     if output.is_null() {
+        // allocation failure
         return;
     }
     let output = std::slice::from_raw_parts_mut(output, input_len as usize);
@@ -246,6 +268,10 @@ unsafe extern "C" fn xor_transform(
             }
             None => return,
         },
+        XorKeySource::ByteVar(key) => match byte_var_key_bytes(det, key, &mut var_key) {
+            Some(k) => k,
+            None => return,
+        },
     };
 
     xor_transform_do(input, output, key, xor_offset);
@@ -347,6 +373,36 @@ mod tests {
         assert_eq!(r.xor_offset, Some(4));
     }
 
+    #[test]
+    fn test_parse_byte_var() {
+        // The `var` form is recognized and produces a ByteVarSpec; detailed
+        // spec parsing is covered in the varkey module's tests.
+        let r = xor_parse_options("var xkey").unwrap();
+        assert_eq!(
+            r.key_spec,
+            XorKeySpec::ByteVar(ByteVarSpec {
+                name: "xkey".to_string(),
+                width: None
+            })
+        );
+        assert_eq!(r.xor_offset, None);
+        // explicit width, with a decode offset preceding the variable key
+        let r = xor_parse_options("offset 2,var xkey 4").unwrap();
+        assert_eq!(
+            r.key_spec,
+            XorKeySpec::ByteVar(ByteVarSpec {
+                name: "xkey".to_string(),
+                width: Some(4)
+            })
+        );
+        assert_eq!(r.xor_offset, Some(2));
+        // malformed var specs are rejected (details tested in varkey)
+        assert!(xor_parse_options("var").is_none());
+        assert!(xor_parse_options("var xkey wide").is_none());
+        // no separating whitespace is not a var spec; falls through to hex
+        assert!(xor_parse_options("varxkey").is_none());
+    }
+
     #[test]
     fn test_parse_empty() {
         assert!(xor_parse_options("").is_none());
@@ -418,7 +474,7 @@ mod tests {
         assert_eq!(recovered, input);
     }
 
-    // Build an id_buf for a variable key the same way xor_build_ctx does.
+    // Build an id_buf for a variable (in-buffer) key the same way build_xor_data does.
     fn make_id_buf_var(key_offset: u16, nbytes: u8, xor_offset: u32) -> Vec<u8> {
         let [lo, hi] = key_offset.to_le_bytes();
         let mut buf = vec![0x01, lo, hi, nbytes];
@@ -426,7 +482,7 @@ mod tests {
         buf
     }
 
-    // Build an id_buf for a static key the same way xor_build_ctx does.
+    // Build an id_buf for a static key the same way build_xor_data does.
     fn make_id_buf_static(key: &[u8], xor_offset: u32) -> Vec<u8> {
         let mut buf = vec![0x00];
         buf.extend_from_slice(key);
@@ -437,16 +493,15 @@ mod tests {
     #[test]
     fn test_xor_id_variable_key() {
         // key_offset=0, nbytes=1, xor_offset=0 → [0x01, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00]
-        let id_buf = make_id_buf_var(0, 1, 0);
-        let ctx = Box::new(DetectTransformXorData {
-            key_source: XorKeySource::Variable(VariableKeyLocation {
+        let data = build_xor_data(
+            XorKeySource::Variable(VariableKeyLocation {
                 offset: 0,
                 nbytes: 1,
             }),
-            xor_offset: 0,
-            id_buf,
-        });
-        let ctx_ptr: *const c_void = &*ctx as *const _ as *const c_void;
+            0,
+        );
+        assert_eq!(data.id_buf, make_id_buf_var(0, 1, 0));
+        let ctx_ptr: *const c_void = &data as *const _ as *const c_void;
         let mut data_ptr: *const u8 = std::ptr::null();
         let mut length: u32 = 0;
         unsafe {
@@ -463,38 +518,53 @@ mod tests {
     #[test]
     fn test_xor_id_variable_key_large_offset() {
         // key_offset=300 (0x012c LE = [0x2c, 0x01]), nbytes=4, xor_offset=0
-        let id_buf = make_id_buf_var(300, 4, 0);
-        let ctx = Box::new(DetectTransformXorData {
-            key_source: XorKeySource::Variable(VariableKeyLocation {
+        let data = build_xor_data(
+            XorKeySource::Variable(VariableKeyLocation {
                 offset: 300,
                 nbytes: 4,
             }),
-            xor_offset: 0,
-            id_buf,
-        });
-        let ctx_ptr: *const c_void = &*ctx as *const _ as *const c_void;
-        let mut data_ptr: *const u8 = std::ptr::null();
-        let mut length: u32 = 0;
-        unsafe {
-            xor_id(&mut data_ptr, &mut length, ctx_ptr as *mut c_void);
-            assert!(!data_ptr.is_null());
-            assert_eq!(length, 8);
-            assert_eq!(
-                std::slice::from_raw_parts(data_ptr, 8),
-                &[0x01u8, 0x2c, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00]
-            );
-        }
+            0,
+        );
+        assert_eq!(
+            data.id_buf,
+            &[0x01u8, 0x2c, 0x01, 0x04, 0x00, 0x00, 0x00, 0x00]
+        );
     }
 
     #[test]
     fn test_xor_id_variable_key_nonzero_xor_offset() {
-        // Verify that two rules with the same key location but different xor_offset
+        // Two rules with the same key location but different xor_offset must
         // produce distinct id_bufs and therefore get independent buffers.
         let id_a = make_id_buf_var(0, 1, 0);
         let id_b = make_id_buf_var(0, 1, 5);
         assert_ne!(id_a, id_b);
     }
 
+    #[test]
+    fn test_xor_id_byte_var() {
+        // byte_extract/byte_math key: discriminant 0x02, local_id, nbytes, xor_offset.
+        let data = build_xor_data(
+            XorKeySource::ByteVar(ByteVarKey {
+                local_id: 3,
+                nbytes: 4,
+            }),
+            0,
+        );
+        assert_eq!(data.id_buf, vec![0x02, 0x03, 0x04, 0x00, 0x00, 0x00, 0x00]);
+        // distinct from a static key whose bytes happen to match
+        let stat = build_xor_data(XorKeySource::Static(vec![0x03, 0x04]), 0);
+        assert_ne!(data.id_buf, stat.id_buf);
+        // distinct from an in-buffer variable key with matching numbers
+        let var = build_xor_data(
+            XorKeySource::Variable(VariableKeyLocation {
+                offset: 3,
+                nbytes: 4,
+            }),
+            0,
+        );
+        assert_ne!(data.id_buf, var.id_buf);
+    }
+
     #[test]
     fn test_var_key_transform() {
         // Buffer: [key_byte, encoded[0], encoded[1], ...]
@@ -521,14 +591,10 @@ mod tests {
     fn test_xor_id() {
         // Static key [1,2,3,4,5] with xor_offset=0: id = [0x00, key_bytes..., 0,0,0,0]
         let key = vec![1u8, 2, 3, 4, 5];
-        let id_buf = make_id_buf_static(&key, 0);
-        let ctx = Box::new(DetectTransformXorData {
-            key_source: XorKeySource::Static(key),
-            xor_offset: 0,
-            id_buf,
-        });
+        let data = build_xor_data(XorKeySource::Static(key.clone()), 0);
+        assert_eq!(data.id_buf, make_id_buf_static(&key, 0));
 
-        let ctx_ptr: *const c_void = &*ctx as *const _ as *const c_void;
+        let ctx_ptr: *const c_void = &data as *const _ as *const c_void;
         let mut data_ptr: *const u8 = std::ptr::null();
         let mut length: u32 = 0;