From 2a572b0c50e9bef1a8f5e91cd2611a6dd5f52820 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Wed, 24 Jun 2026 08:40:48 -0400 Subject: [PATCH 01/13] dcerpc: set endian flag after applying transforms dcerpc_tx_get_stub_data set the DCE byte-order flag before SCInspectionBufferSetupAndApplyTransforms. Once a later commit makes buffer setup reset the flags, that order would clear the flag and break byte_test's dce option (dcerpc/dcerpc-dcepayload-15). Set the flag after the call. Issue: 7313 --- rust/src/dcerpc/detect.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/src/dcerpc/detect.rs b/rust/src/dcerpc/detect.rs index 03241f7a87..d43c02882d 100644 --- a/rust/src/dcerpc/detect.rs +++ b/rust/src/dcerpc/detect.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2020 Open Information Security Foundation +/* Copyright (C) 2020-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -426,15 +426,15 @@ unsafe extern "C" fn dcerpc_tx_get_stub_data( tx.stub_data_buffer_tc.len() as u32, ) }; + SCInspectionBufferSetupAndApplyTransforms( + det_ctx, list_id, buffer, data, data_len, transforms, + ); + if tx.endianness > 0 { (*buffer).flags |= DETECT_CI_FLAGS_DCE_LE; } else { (*buffer).flags |= DETECT_CI_FLAGS_DCE_BE; } - - SCInspectionBufferSetupAndApplyTransforms( - det_ctx, list_id, buffer, data, data_len, transforms, - ); } return buffer; } From a2ad1422f403dd8363da26e28a693102c8f20a77 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Mon, 2 Feb 2026 08:52:14 -0500 Subject: [PATCH 02/13] detect/inspect: add inspection buffer error function Add SCInspectionBufferSetError(), which sets a new DETECT_CI_FLAGS_ERROR flag and leaves the buffer's data unchanged, so a keyword after a failed transform can tell the failure from a success. Add SIGMATCH_TRANSFORM_CAN_FAIL to mark transforms that call it. Clear the flags in each buffer setup function, so an error from one use of a buffer does not carry over to the next. Issue: 7313 --- rust/sys/src/sys.rs | 4 ++ src/detect-engine-content-inspection.c | 3 +- src/detect-engine-content-inspection.h | 3 +- src/detect-engine-inspect-buffer.c | 12 +++++- src/detect-engine-inspect-buffer.h | 3 +- src/detect-engine-register.h | 2 + src/tests/detect-engine-content-inspection.c | 40 +++++++++++++++++++- 7 files changed, 62 insertions(+), 5 deletions(-) diff --git a/rust/sys/src/sys.rs b/rust/sys/src/sys.rs index 67097e5edb..32281c0081 100644 --- a/rust/sys/src/sys.rs +++ b/rust/sys/src/sys.rs @@ -26,6 +26,7 @@ pub const SIGMATCH_INFO_BITFLAGS_UINT: u32 = 1048576; pub const SIGMATCH_BAN_FIREWALL_RULE: u32 = 2097152; pub const SIGMATCH_BAN_FIREWALL_MODE: u32 = 4194304; pub const SIGMATCH_BAN_TD_FIREWALL_MODE: u32 = 16777216; +pub const SIGMATCH_TRANSFORM_CAN_FAIL: u32 = 33554432; pub type __intmax_t = ::std::os::raw::c_long; pub type intmax_t = __intmax_t; #[repr(u32)] @@ -744,6 +745,9 @@ extern "C" { extern "C" { pub fn SCInspectionBufferTruncate(buffer: *mut InspectionBuffer, buf_len: u32); } +extern "C" { + pub fn SCInspectionBufferSetError(buffer: *mut InspectionBuffer); +} extern "C" { pub fn SCInspectionBufferGet( det_ctx: *mut DetectEngineThreadCtx, list_id: ::std::os::raw::c_int, diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index 2fe5bcaf3d..e2037a748e 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -787,7 +787,8 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd) { - // we will match on NULL buffers there is one absent + // we will match on NULL buffers if there is one absent keyword + // absent: error_or does NOT match on absent buffers (only on transform errors) bool absent_data = false; while (1) { if (smd->type == DETECT_ABSENT) { diff --git a/src/detect-engine-content-inspection.h b/src/detect-engine-content-inspection.h index 3c12477265..1b983c86e7 100644 --- a/src/detect-engine-content-inspection.h +++ b/src/detect-engine-content-inspection.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2023 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -44,6 +44,7 @@ enum DetectContentInspectionType { // next ones come from rust dcerpc // #define DETECT_CI_FLAGS_DCE_LE BIT_U8(2) /**< DCERPC record in little endian */ // #define DETECT_CI_FLAGS_DCE_BE BIT_U8(3) /**< DCERPC record in big endian */ +#define DETECT_CI_FLAGS_ERROR BIT_U8(4) /**< transformation/decode error occurred */ /** buffer is a single, non-streaming, buffer. Data sent to the content * inspection function contains both start and end of the data. */ diff --git a/src/detect-engine-inspect-buffer.c b/src/detect-engine-inspect-buffer.c index 99fd4de459..e8b768144f 100644 --- a/src/detect-engine-inspect-buffer.c +++ b/src/detect-engine-inspect-buffer.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2025 Open Information Security Foundation +/* Copyright (C) 2025-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -24,6 +24,7 @@ #include "suricata-common.h" #include "detect-engine-inspect-buffer.h" #include "detect.h" +#include "detect-engine-content-inspection.h" #include "util-validate.h" @@ -150,6 +151,7 @@ void InspectionBufferSetupMultiEmpty(InspectionBuffer *buffer) buffer->inspect = NULL; buffer->inspect_len = 0; buffer->len = 0; + buffer->flags = 0; buffer->initialized = true; } @@ -163,6 +165,7 @@ void InspectionBufferSetupMulti(DetectEngineThreadCtx *det_ctx, InspectionBuffer buffer->inspect = buffer->orig = data; buffer->inspect_len = buffer->orig_len = data_len; buffer->len = 0; + buffer->flags = 0; buffer->initialized = true; InspectionBufferApplyTransformsInternal(det_ctx, buffer, transforms); @@ -184,6 +187,7 @@ static inline void InspectionBufferSetupInternal(DetectEngineThreadCtx *det_ctx, buffer->inspect = buffer->orig = data; buffer->inspect_len = buffer->orig_len = data_len; buffer->len = 0; + buffer->flags = 0; buffer->initialized = true; } /** \brief setup the buffer with our initial data */ @@ -248,6 +252,12 @@ void SCInspectionBufferTruncate(InspectionBuffer *buffer, uint32_t buf_len) buffer->initialized = true; } +void SCInspectionBufferSetError(InspectionBuffer *buffer) +{ + buffer->flags |= DETECT_CI_FLAGS_ERROR; + buffer->initialized = true; +} + void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len) { SCInspectionBufferCheckAndExpand(buffer, buf_len); diff --git a/src/detect-engine-inspect-buffer.h b/src/detect-engine-inspect-buffer.h index 0e8c7dd443..ea79be035b 100644 --- a/src/detect-engine-inspect-buffer.h +++ b/src/detect-engine-inspect-buffer.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2025 Open Information Security Foundation +/* Copyright (C) 2025-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -62,6 +62,7 @@ void SCInspectionBufferSetupAndApplyTransforms(DetectEngineThreadCtx *det_ctx, c void InspectionBufferFree(InspectionBuffer *buffer); uint8_t *SCInspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size); void SCInspectionBufferTruncate(InspectionBuffer *buffer, uint32_t buf_len); +void SCInspectionBufferSetError(InspectionBuffer *buffer); void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len); void InspectionBufferApplyTransforms(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer, const DetectEngineTransforms *transforms); diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 8a247f3600..748388b86f 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -362,6 +362,8 @@ extern int DETECT_TBLSIZE_IDX; #define SIGMATCH_BAN_FIREWALL_MODE (1UL << (22)) /** keyword cannot be used in td rules with firewall mode */ #define SIGMATCH_BAN_TD_FIREWALL_MODE (1UL << (24)) +/** transform may flag a failure with SCInspectionBufferSetError */ +#define SIGMATCH_TRANSFORM_CAN_FAIL (1UL << (25)) int SigTableList(const char *keyword); void SigTableCleanup(void); diff --git a/src/tests/detect-engine-content-inspection.c b/src/tests/detect-engine-content-inspection.c index a6504adc9a..fbeeb06826 100644 --- a/src/tests/detect-engine-content-inspection.c +++ b/src/tests/detect-engine-content-inspection.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2017 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -316,6 +316,42 @@ static int DetectEngineContentInspectionTest17(void) TEST_FOOTER; } +/** \test SCInspectionBufferSetError keeps the data, and setting the buffer up + * again clears the error flag */ +static int DetectEngineContentInspectionTest18(void) +{ + const uint8_t data[] = "Zm9vYmFy!!"; + const uint32_t data_len = sizeof(data) - 1; + InspectionBuffer buffer; + memset(&buffer, 0, sizeof(buffer)); +#ifdef DEBUG_VALIDATION + buffer.multi = true; +#endif + + InspectionBufferSetupMulti(NULL, &buffer, NULL, data, data_len); + FAIL_IF(buffer.flags & DETECT_CI_FLAGS_ERROR); + SCInspectionBufferSetError(&buffer); + FAIL_IF_NOT(buffer.flags & DETECT_CI_FLAGS_ERROR); + FAIL_IF_NOT(buffer.inspect == data); + FAIL_IF_NOT(buffer.inspect_len == data_len); + FAIL_IF_NOT(buffer.initialized); + + InspectionBufferSetupMulti(NULL, &buffer, NULL, data, data_len); + FAIL_IF(buffer.flags & DETECT_CI_FLAGS_ERROR); + + SCInspectionBufferSetError(&buffer); + buffer.initialized = false; + InspectionBufferSetupMultiEmpty(&buffer); + FAIL_IF(buffer.flags & DETECT_CI_FLAGS_ERROR); + + /* the error flag must not share a bit with the start/end flags or with + * the DCE_LE/DCE_BE flags that rust/src/dcerpc/detect.rs sets */ + FAIL_IF(DETECT_CI_FLAGS_ERROR & (DETECT_CI_FLAGS_SINGLE | BIT_U8(2) | BIT_U8(3))); + + InspectionBufferFree(&buffer); + PASS; +} + void DetectEngineContentInspectionRegisterTests(void) { UtRegisterTest("DetectEngineContentInspectionTest01", @@ -348,6 +384,8 @@ void DetectEngineContentInspectionRegisterTests(void) DetectEngineContentInspectionTest14); UtRegisterTest("DetectEngineContentInspectionTest17 negative distance", DetectEngineContentInspectionTest17); + UtRegisterTest( + "DetectEngineContentInspectionTest18 error flag", DetectEngineContentInspectionTest18); } #undef TEST_HEADER From de7016b79b0cdcab4c4f82a62bb941fc24279831 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Thu, 8 Oct 2026 08:56:45 -0400 Subject: [PATCH 03/13] ffi/base64: add a safe decode function for Rust SCBase64Decode builds its output slice with the input length, whatever the caller allocated, and the from_base64 transform called it through unsafe after asking for only the decoded length. Move the decoding into base64_decode(), which takes the input and output as slices, and keep SCBase64Decode as a wrapper for C callers. The transform switches to base64_decode in the next commit. Issue: 7114 --- rust/src/ffi/base64.rs | 111 +++++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 21 deletions(-) diff --git a/rust/src/ffi/base64.rs b/rust/src/ffi/base64.rs index 6befb5e3f9..b5fb19dadd 100644 --- a/rust/src/ffi/base64.rs +++ b/rust/src/ffi/base64.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2021-2024 Open Information Security Foundation +/* Copyright (C) 2021-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -80,44 +80,37 @@ pub unsafe extern "C" fn SCBase64DecodeBufferSize(input_len: u32) -> u32 { return get_decoded_buffer_size(input_len); } -/// Base64 decode a buffer. +/// Base64 decode `input` into `output` in the given mode. /// -/// This method exposes the Rust base64 decoder to C and should not be called from -/// Rust code. -/// -/// It allows decoding in the modes described by ``SCBase64Mode`` enum. -#[no_mangle] -pub unsafe extern "C" fn SCBase64Decode( - input: *const u8, len: usize, mode: SCBase64Mode, output: *mut u8, -) -> u32 { - if input.is_null() || len == 0 { +/// Returns the number of bytes written to `output`. `output` must be at least +/// as long as `input`. +pub fn base64_decode(input: &[u8], mode: SCBase64Mode, output: &mut [u8]) -> u32 { + if input.is_empty() { return 0; } - let in_vec = build_slice!(input, len); - let out_vec = std::slice::from_raw_parts_mut(output, len); let mut num_decoded: u32 = 0; let mut decoder = Decoder::new(); match mode { SCBase64Mode::SCBase64ModeRFC2045 => { - if decode_rfc2045(&mut decoder, in_vec, out_vec, &mut num_decoded).is_err() { - debug_validate_bug_on!(num_decoded >= len as u32); + if decode_rfc2045(&mut decoder, input, output, &mut num_decoded).is_err() { + debug_validate_bug_on!(num_decoded >= input.len() as u32); return num_decoded; } } SCBase64Mode::SCBase64ModeRFC4648 => { - if decode_rfc4648(&mut decoder, in_vec, out_vec, &mut num_decoded).is_err() { - debug_validate_bug_on!(num_decoded >= len as u32); + if decode_rfc4648(&mut decoder, input, output, &mut num_decoded).is_err() { + debug_validate_bug_on!(num_decoded >= input.len() as u32); return num_decoded; } } SCBase64Mode::SCBase64ModeStrict => { - if let Ok(decoded_len) = STANDARD.decode_slice(in_vec, out_vec) { + if let Ok(decoded_len) = STANDARD.decode_slice(input, output) { num_decoded = decoded_len as u32; } } SCBase64Mode::SCBase64ModeNoPad => { - if let Ok(decoded_len) = STANDARD_NO_PAD.decode_slice(in_vec, out_vec) { + if let Ok(decoded_len) = STANDARD_NO_PAD.decode_slice(input, output) { num_decoded = decoded_len as u32; } } @@ -125,16 +118,35 @@ pub unsafe extern "C" fn SCBase64Decode( let config = base64::engine::GeneralPurposeConfig::new() .with_decode_padding_mode(base64::engine::DecodePaddingMode::Indifferent); let decoder = base64::engine::GeneralPurpose::new(&base64::alphabet::STANDARD, config); - if let Ok(decoded_len) = decoder.decode_slice(in_vec, out_vec) { + if let Ok(decoded_len) = decoder.decode_slice(input, output) { num_decoded = decoded_len as u32; } } } - debug_validate_bug_on!(num_decoded >= len as u32); + debug_validate_bug_on!(num_decoded >= input.len() as u32); return num_decoded; } +/// Base64 decode a buffer. +/// +/// This method exposes the Rust base64 decoder to C and should not be called from +/// Rust code; use ``base64_decode`` instead. +/// +/// It allows decoding in the modes described by ``SCBase64Mode`` enum. +#[no_mangle] +pub unsafe extern "C" fn SCBase64Decode( + input: *const u8, len: usize, mode: SCBase64Mode, output: *mut u8, +) -> u32 { + if input.is_null() || len == 0 { + return 0; + } + + let in_vec = build_slice!(input, len); + let out_vec = std::slice::from_raw_parts_mut(output, len); + base64_decode(in_vec, mode, out_vec) +} + /// Base64 encode a buffer with a provided mode. /// /// This method exposes the Rust base64 encoder to C and should not be called from @@ -194,3 +206,60 @@ pub unsafe extern "C" fn SCBase64Encode( pub extern "C" fn SCBase64EncodeBufferSize(len: c_ulong) -> c_ulong { (4 * ((len) + 2) / 3) + 1 } + +#[cfg(test)] +mod tests { + use super::*; + + fn decode(input: &[u8], mode: SCBase64Mode) -> Vec { + let mut output = vec![0u8; input.len()]; + let n = base64_decode(input, mode, &mut output) as usize; + output.truncate(n); + output + } + + #[test] + fn test_base64_decode_empty() { + assert!(decode(b"", SCBase64Mode::SCBase64ModeStrict).is_empty()); + assert!(decode(b"", SCBase64Mode::SCBase64ModeRFC2045).is_empty()); + } + + #[test] + fn test_base64_decode_output_same_length_as_input() { + for mode in [ + SCBase64Mode::SCBase64ModeRFC2045, + SCBase64Mode::SCBase64ModeRFC4648, + SCBase64Mode::SCBase64ModeStrict, + SCBase64Mode::SCBase64ModePadOpt, + ] { + assert_eq!(decode(b"Zm9vYmFy", mode), b"foobar"); + assert_eq!(decode(b"Zm9vYmE=", mode), b"fooba"); + } + assert_eq!( + decode(b"Zm9vYmE", SCBase64Mode::SCBase64ModeNoPad), + b"fooba" + ); + } + + #[test] + fn test_base64_decode_invalid_bytes() { + // strict, no-pad and pad-optional reject the whole input + assert!(decode(b"Zm9vYmFy!!YmF6", SCBase64Mode::SCBase64ModeStrict).is_empty()); + assert!(decode(b"Zm9vYmFy!!YmF6", SCBase64Mode::SCBase64ModeNoPad).is_empty()); + assert!(decode(b"Zm9vYmFy!!YmF6", SCBase64Mode::SCBase64ModePadOpt).is_empty()); + // rfc4648 stops at the first invalid byte and keeps what it decoded + assert_eq!( + decode(b"Zm9vYmFy!!YmF6", SCBase64Mode::SCBase64ModeRFC4648), + b"foobar" + ); + // rfc2045 skips invalid bytes + assert_eq!( + decode(b"Zm9vYmFy!!YmF6", SCBase64Mode::SCBase64ModeRFC2045), + b"foobarbaz" + ); + assert_eq!( + decode(b"Zm 9v Ym Fy", SCBase64Mode::SCBase64ModeRFC2045), + b"foobar" + ); + } +} From 9f8dad7245e9dfe232340acc53ee52e96a3d6c27 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Mon, 9 Mar 2026 10:29:14 -0400 Subject: [PATCH 04/13] transform/base64: flag from_base64 decode errors from_base64 gave no signal when offset or bytes ran past the input or nothing decoded, so a later keyword could not tell a failed decode from a successful one. It also skipped decoding when bytes equaled the input left after offset, which the C transform before d8a99e1fdd decoded. When an earlier transform had left the input in the buffer's own memory, a strict decode that failed partway wrote decoded bytes over it: after strip_whitespace, "/Zm9vYmFyYmF6cXV4!!" no longer matched. Mark the transform SIGMATCH_TRANSFORM_CAN_FAIL and flag these failures, and allocation failures, with SCInspectionBufferSetError. Decode when bytes equals the remaining input. In place, decode into a separate buffer and copy the result over only on success. Call base64_decode instead of SCBase64Decode. Issue: 7114 --- rust/src/detect/mod.rs | 3 +- rust/src/detect/transforms/base64.rs | 263 ++++++++++++------- src/tests/detect-engine-content-inspection.c | 52 ++++ 3 files changed, 226 insertions(+), 92 deletions(-) diff --git a/rust/src/detect/mod.rs b/rust/src/detect/mod.rs index 0c57481f0f..2b9633725e 100644 --- a/rust/src/detect/mod.rs +++ b/rust/src/detect/mod.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2022 Open Information Security Foundation +/* Copyright (C) 2022-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -96,6 +96,7 @@ pub use suricata_sys::sys::{ SIGMATCH_INFO_MULTI_UINT, SIGMATCH_INFO_STICKY_BUFFER, SIGMATCH_INFO_UINT16, SIGMATCH_INFO_UINT32, SIGMATCH_INFO_UINT64, SIGMATCH_INFO_UINT8, SIGMATCH_NOOPT, SIGMATCH_OPTIONAL_OPT, SIGMATCH_QUOTES_MANDATORY, SIGMATCH_SUPPORT_FIREWALL, + SIGMATCH_TRANSFORM_CAN_FAIL, }; #[derive(Default)] diff --git a/rust/src/detect/transforms/base64.rs b/rust/src/detect/transforms/base64.rs index d6e2975292..d59a44b5a4 100644 --- a/rust/src/detect/transforms/base64.rs +++ b/rust/src/detect/transforms/base64.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2024 Open Information Security Foundation +/* Copyright (C) 2024-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -19,20 +19,14 @@ use crate::detect::error::RuleParseError; use crate::detect::parser::{parse_var, take_until_whitespace, ResultValue}; -use crate::detect::SIGMATCH_OPTIONAL_OPT; -use crate::ffi::base64::{SCBase64Decode, SCBase64Mode}; -use crate::utils::base64::get_decoded_buffer_size; +use crate::detect::{SIGMATCH_OPTIONAL_OPT, SIGMATCH_TRANSFORM_CAN_FAIL}; +use crate::ffi::base64::{base64_decode, SCBase64Mode}; -#[cfg(test)] -use crate::detect::transforms::base64::tests::{ - SCInspectionBufferCheckAndExpand, SCInspectionBufferTruncate, -}; use suricata_sys::sys::{ DetectEngineCtx, DetectEngineThreadCtx, InspectionBuffer, SCDetectHelperTransformRegister, - SCDetectSignatureAddTransform, SCTransformTableElmt, Signature, + SCDetectSignatureAddTransform, SCInspectionBufferCheckAndExpand, SCInspectionBufferInPlace, + SCInspectionBufferSetError, SCInspectionBufferTruncate, SCTransformTableElmt, Signature, }; -#[cfg(not(test))] -use suricata_sys::sys::{SCInspectionBufferCheckAndExpand, SCInspectionBufferTruncate}; use nom8::bytes::complete::tag; use nom8::character::complete::multispace0; @@ -237,43 +231,122 @@ unsafe extern "C" fn base64_id(data: *mut *const u8, length: *mut u32, ctx: *con *length = std::mem::size_of::() as u32; } +/// Outcome of a base64 decode against an input slice. +/// +/// The pure decode logic is separated from the FFI shim so tests can drive +/// it directly with a `&[u8]` and assert against a plain Rust value, rather +/// than mocking the InspectionBuffer helpers. +#[derive(Debug, PartialEq, Eq)] +enum Base64TransformOutcome { + /// Empty input: nothing to decode. The inspection buffer is left untouched. + Skip, + /// Decoder produced at least one byte. The FFI shim copies these into + /// the inspection buffer scratch and calls `SCInspectionBufferTruncate`. + Decoded(Vec), + /// A non-empty input was not decoded: offset at or past the end, nbytes + /// larger than the input left after offset, or zero bytes decoded. The + /// FFI shim calls `SCInspectionBufferSetError` to signal the failure. + Error, +} + +/// Apply `offset` and `bytes` to the input. `Err` holds the outcome when +/// there is nothing to decode: `Skip` or `Error`. +fn base64_slice<'a>( + input: &'a [u8], ctx: &DetectTransformFromBase64Data, +) -> Result<&'a [u8], Base64TransformOutcome> { + if input.is_empty() { + return Err(Base64TransformOutcome::Skip); + } + let mut slice = input; + if ctx.offset > 0 { + if ctx.offset as usize >= slice.len() { + return Err(Base64TransformOutcome::Error); + } + slice = &slice[ctx.offset as usize..]; + } + if ctx.nbytes > 0 { + if ctx.nbytes as usize > slice.len() { + return Err(Base64TransformOutcome::Error); + } + slice = &slice[..ctx.nbytes as usize]; + } + Ok(slice) +} + +fn base64_transform_pure( + input: &[u8], ctx: &DetectTransformFromBase64Data, +) -> Base64TransformOutcome { + let slice = match base64_slice(input, ctx) { + Ok(slice) => slice, + Err(outcome) => return outcome, + }; + // base64_decode needs an output at least as long as its input. + let mut output = vec![0u8; slice.len()]; + let num_decoded = base64_decode(slice, ctx.mode, &mut output) as usize; + if num_decoded == 0 { + return Base64TransformOutcome::Error; + } + output.truncate(num_decoded); + Base64TransformOutcome::Decoded(output) +} + unsafe extern "C" fn base64_transform( _det: *mut DetectEngineThreadCtx, buffer: *mut InspectionBuffer, ctx: *const c_void, ) { - let input = (*buffer).inspect; + let input_ptr = (*buffer).inspect; let input_len = (*buffer).inspect_len; - if input.is_null() || input_len == 0 { + if input_ptr.is_null() { return; } - let mut input = build_slice!(input, input_len as usize); - + let input = build_slice!(input_ptr, input_len as usize); let ctx = cast_pointer!(ctx, DetectTransformFromBase64Data); - if ctx.offset > 0 { - if ctx.offset >= input_len { - return; + if SCInspectionBufferInPlace(buffer) { + // The input is the scratch buffer, so a decode that fails partway + // would leave decoded bytes over it. Decode into a separate buffer + // and copy the result over only when the decode succeeds. + match base64_transform_pure(input, ctx) { + Base64TransformOutcome::Skip => {} + Base64TransformOutcome::Decoded(decoded) => { + let scratch = SCInspectionBufferCheckAndExpand(buffer, decoded.len() as u32); + if scratch.is_null() { + // allocation failure: the decoded bytes cannot replace the input + SCInspectionBufferSetError(buffer); + return; + } + std::ptr::copy_nonoverlapping(decoded.as_ptr(), scratch, decoded.len()); + SCInspectionBufferTruncate(buffer, decoded.len() as u32); + } + Base64TransformOutcome::Error => { + SCInspectionBufferSetError(buffer); + } } - input = &input[ctx.offset as usize..]; + return; } - if ctx.nbytes > 0 { - if ctx.nbytes as usize >= input.len() { + + let slice = match base64_slice(input, ctx) { + Ok(slice) => slice, + Err(Base64TransformOutcome::Error) => { + SCInspectionBufferSetError(buffer); return; } - input = &input[..ctx.nbytes as usize]; - } - - let output_len = get_decoded_buffer_size(input.len() as u32); - // no realloc, we only can shrink - let output = SCInspectionBufferCheckAndExpand(buffer, output_len); - if output.is_null() { - // allocation failure + Err(_) => return, + }; + // base64_decode needs an output at least as long as its input. + let scratch = SCInspectionBufferCheckAndExpand(buffer, slice.len() as u32); + if scratch.is_null() { + // allocation failure: the decoded bytes cannot replace the input + SCInspectionBufferSetError(buffer); return; } - - let num_decoded = SCBase64Decode(input.as_ptr(), input.len(), ctx.mode, output); - if num_decoded > 0 { - SCInspectionBufferTruncate(buffer, num_decoded); + let output = std::slice::from_raw_parts_mut(scratch, slice.len()); + let num_decoded = base64_decode(slice, ctx.mode, output); + if num_decoded == 0 { + // inspect still points at the input, so it is unchanged + SCInspectionBufferSetError(buffer); + return; } + SCInspectionBufferTruncate(buffer, num_decoded); } #[no_mangle] @@ -283,7 +356,7 @@ pub unsafe extern "C" fn DetectTransformFromBase64DecodeRegister() { desc: b"convert the base64 decode of the buffer\0".as_ptr() as *const libc::c_char, url: b"/rules/transforms.html#from-base64\0".as_ptr() as *const libc::c_char, Setup: Some(base64_setup), - flags: SIGMATCH_OPTIONAL_OPT, + flags: SIGMATCH_OPTIONAL_OPT | SIGMATCH_TRANSFORM_CAN_FAIL, Transform: Some(base64_transform), Free: Some(base64_free), TransformValidate: None, @@ -398,77 +471,85 @@ mod tests { assert!(parse_transform_base64("bytes var, offset 3933, mode rfc4648").is_err()); } - // Test/mock versions to keep tests in rust - #[allow(non_snake_case)] - pub(crate) unsafe fn SCInspectionBufferCheckAndExpand( - buffer: *mut InspectionBuffer, min_size: u32, - ) -> *mut u8 { - assert!(min_size <= (*buffer).inspect_len); - return (*buffer).inspect as *mut u8; - } - - #[allow(non_snake_case)] - pub(crate) unsafe fn SCInspectionBufferTruncate(buffer: *mut InspectionBuffer, buf_len: u32) { - (*buffer).inspect_len = buf_len; - } - - fn test_base64_sample(sig: &str, buf: &[u8], out: &[u8]) { - let mut ibuf: InspectionBuffer = unsafe { std::mem::zeroed() }; - let mut input = Vec::new(); - // we will overwrite it, so do not create it const - input.extend_from_slice(buf); - ibuf.inspect = input.as_ptr(); - ibuf.inspect_len = input.len() as u32; - let (_, mut ctx) = parse_transform_base64(sig).unwrap(); - unsafe { - base64_transform( - std::ptr::null_mut(), - &mut ibuf as *mut InspectionBuffer, - &mut ctx as *mut DetectTransformFromBase64Data as *mut c_void, - ); - } - let ibufi = ibuf.inspect; - let output = unsafe { build_slice!(ibufi, ibuf.inspect_len as usize) }; - assert_eq!(output, out); + /// Parse `sig`, run the pure decode against `buf`, and assert the outcome. + fn assert_outcome(sig: &str, buf: &[u8], expected: Base64TransformOutcome) { + let (_, ctx) = parse_transform_base64(sig).unwrap(); + assert_eq!(base64_transform_pure(buf, &ctx), expected); } #[test] fn test_base64_transform() { - /* Simple success case -- check buffer */ - test_base64_sample("", b"VGhpcyBpcyBTdXJpY2F0YQ==", b"This is Suricata"); - /* Simple success case with RFC2045 -- check buffer */ - test_base64_sample("mode rfc2045", b"Zm 9v Ym Fy", b"foobar"); - /* Decode failure case -- ensure no change to buffer */ - test_base64_sample("mode strict", b"This is Suricata\n", b"This is Suricata\n"); - /* bytes > len so --> no transform */ - test_base64_sample( - "bytes 25", - b"VGhpcyBpcyBTdXJpY2F0YQ==", + use Base64TransformOutcome::*; + + /* Simple success case */ + assert_outcome( + "", b"VGhpcyBpcyBTdXJpY2F0YQ==", + Decoded(b"This is Suricata".to_vec()), ); - /* offset > len so --> no transform */ - test_base64_sample( - "offset 25", - b"VGhpcyBpcyBTdXJpY2F0YQ==", + /* Simple success case with RFC2045 */ + assert_outcome("mode rfc2045", b"Zm 9v Ym Fy", Decoded(b"foobar".to_vec())); + /* Decode failure case -- should signal error */ + assert_outcome("mode strict", b"This is Suricata\n", Error); + /* bytes > len: nothing to decode --> error */ + assert_outcome("bytes 25", b"VGhpcyBpcyBTdXJpY2F0YQ==", Error); + /* offset >= len: nothing to decode --> error */ + assert_outcome("offset 25", b"VGhpcyBpcyBTdXJpY2F0YQ==", Error); + /* partial transform */ + assert_outcome( + "bytes 12", b"VGhpcyBpcyBTdXJpY2F0YQ==", + Decoded(b"This is S".to_vec()), ); - /* partial transform */ - test_base64_sample("bytes 12", b"VGhpcyBpcyBTdXJpY2F0YQ==", b"This is S"); /* transform from non-zero offset */ - test_base64_sample("offset 4", b"VGhpcyBpcyBTdXJpY2F0YQ==", b"s is Suricata"); + assert_outcome( + "offset 4", + b"VGhpcyBpcyBTdXJpY2F0YQ==", + Decoded(b"s is Suricata".to_vec()), + ); /* partial decode */ - test_base64_sample( + assert_outcome( "mode rfc2045, bytes 15", b"SGVs bG8 gV29y bGQ=", - b"Hello Wor", + Decoded(b"Hello Wor".to_vec()), ); - /* input is not base64 encoded */ - test_base64_sample( + /* rfc2045 partial decode: valid prefix decoded, invalid suffix skipped */ + assert_outcome( "mode rfc2045", - b"This is not base64-encoded", - &[ - 78, 24, 172, 138, 201, 232, 181, 182, 172, 123, 174, 30, 157, 202, 29, - ], + b"ABCD!!!!", + Decoded(b"\x00\x10\x83".to_vec()), ); + /* input is not base64 encoded: decoder returns 0 -> Error */ + assert_outcome("mode rfc2045", b"!!!!", Error); + } + + #[test] + fn test_base64_transform_decode_error() { + use Base64TransformOutcome::*; + assert_outcome("mode strict", b"!!!!", Error); + assert_outcome("mode rfc4648", b"!!!!", Error); + } + + /// An empty (non-absent) buffer is neither a decode success nor a decode + /// failure: the pure core returns Skip so the FFI shim leaves the + /// inspection buffer untouched. + #[test] + fn test_base64_transform_empty_buffer_is_skip() { + assert_outcome("", b"", Base64TransformOutcome::Skip); + } + + /// offset at or past the end, and nbytes larger than the input left after + /// offset, leave nothing to decode and are errors. nbytes equal to the + /// input left after offset decodes all of it. + #[test] + fn test_base64_transform_offset_and_nbytes_out_of_range() { + use Base64TransformOutcome::*; + assert_outcome("offset 10", b"YWJj", Error); + assert_outcome("offset 4", b"YWJj", Error); + assert_outcome("bytes 10", b"YWJj", Error); + assert_outcome("bytes 5", b"YWJj", Error); + assert_outcome("bytes 4", b"YWJj", Decoded(b"abc".to_vec())); + assert_outcome("bytes 4, offset 2", b"xxYWJj", Decoded(b"abc".to_vec())); + assert_outcome("bytes 5, offset 2", b"xxYWJj", Error); } } diff --git a/src/tests/detect-engine-content-inspection.c b/src/tests/detect-engine-content-inspection.c index fbeeb06826..e5b20b1e00 100644 --- a/src/tests/detect-engine-content-inspection.c +++ b/src/tests/detect-engine-content-inspection.c @@ -28,6 +28,7 @@ #include "../flow.h" #include "../detect.h" #include "detect-engine-build.h" +#include "../detect-engine.h" extern thread_local uint32_t ut_inspection_recursion_counter; @@ -352,6 +353,55 @@ static int DetectEngineContentInspectionTest18(void) PASS; } +/** \test from_base64 decodes into the scratch buffer when the input is + * elsewhere, and flags an error and keeps the input when it cannot get + * room for the decoded bytes */ +static int DetectEngineContentInspectionTest19(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + Signature *s = + DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(http.uri; from_base64; content:\"foo\"; sid:1;)"); + FAIL_IF_NULL(s); + const DetectBufferType *map = + DetectEngineBufferTypeGetById(de_ctx, s->init_data->buffers[0].id); + FAIL_IF_NULL(map); + FAIL_IF_NOT(map->transforms.cnt == 1); + + const uint8_t data[] = "Zm9vYmFy"; + const uint32_t data_len = sizeof(data) - 1; + InspectionBuffer buffer; + + memset(&buffer, 0, sizeof(buffer)); +#ifdef DEBUG_VALIDATION + buffer.multi = true; +#endif + InspectionBufferSetupMulti(NULL, &buffer, &map->transforms, data, data_len); + FAIL_IF(buffer.flags & DETECT_CI_FLAGS_ERROR); + FAIL_IF_NOT(buffer.inspect == buffer.buf); + FAIL_IF_NOT(buffer.inspect_len == 6); + FAIL_IF_NOT(memcmp(buffer.inspect, "foobar", 6) == 0); + InspectionBufferFree(&buffer); + memset(&buffer, 0, sizeof(buffer)); +#ifdef DEBUG_VALIDATION + buffer.multi = true; +#endif + /* SCInspectionBufferCheckAndExpand returns buf without reallocating when + * size is large enough, so a NULL buf here stands in for a failed + * SCRealloc */ + buffer.size = 4096; + + InspectionBufferSetupMulti(NULL, &buffer, &map->transforms, data, data_len); + FAIL_IF_NOT(buffer.flags & DETECT_CI_FLAGS_ERROR); + FAIL_IF_NOT(buffer.inspect == data); + FAIL_IF_NOT(buffer.inspect_len == data_len); + + DetectEngineCtxFree(de_ctx); + PASS; +} + void DetectEngineContentInspectionRegisterTests(void) { UtRegisterTest("DetectEngineContentInspectionTest01", @@ -386,6 +436,8 @@ void DetectEngineContentInspectionRegisterTests(void) DetectEngineContentInspectionTest17); UtRegisterTest( "DetectEngineContentInspectionTest18 error flag", DetectEngineContentInspectionTest18); + UtRegisterTest("DetectEngineContentInspectionTest19 from_base64 allocation failure", + DetectEngineContentInspectionTest19); } #undef TEST_HEADER From 31afa84f72116925d89e041fa116bde8630022f2 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Thu, 19 Mar 2026 10:07:47 -0400 Subject: [PATCH 05/13] transform/pcrexform: flag match failures pcrexform gave no signal when the regex did not match, when the capture could not be extracted, or when there was no room for the capture. A capture of nothing also left the input in place when the buffer had no room yet, since InspectionBufferCopy then copies nothing. Mark the transform SIGMATCH_TRANSFORM_CAN_FAIL and flag those failures with SCInspectionBufferSetError. Set an empty capture's length to 0 directly, and leave an empty input alone, as from_base64 does. Issue: 7313 --- src/detect-transform-pcrexform.c | 27 ++++- src/tests/detect-engine-content-inspection.c | 100 +++++++++++++++++++ 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/src/detect-transform-pcrexform.c b/src/detect-transform-pcrexform.c index d36f5f7130..f1ba40ebb7 100644 --- a/src/detect-transform-pcrexform.c +++ b/src/detect-transform-pcrexform.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2020 Open Information Security Foundation +/* Copyright (C) 2020-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -71,7 +71,8 @@ void DetectTransformPcrexformRegister(void) #ifdef UNITTESTS sigmatch_table[DETECT_TRANSFORM_PCREXFORM].RegisterTests = DetectTransformPcrexformRegisterTests; #endif - sigmatch_table[DETECT_TRANSFORM_PCREXFORM].flags |= SIGMATCH_QUOTES_MANDATORY; + sigmatch_table[DETECT_TRANSFORM_PCREXFORM].flags |= + SIGMATCH_QUOTES_MANDATORY | SIGMATCH_TRANSFORM_CAN_FAIL; } static void DetectTransformPcrexformFree(DetectEngineCtx *de_ctx, void *ptr) @@ -163,6 +164,11 @@ static void DetectTransformPcrexform( const uint32_t input_len = buffer->inspect_len; const DetectTransformPcrexformData *pxd = options; + /* nothing to extract from an empty buffer; leave it as it is, as + * from_base64 does */ + if (input_len == 0) + return; + pcre2_match_data *match = pcre2_match_data_create_from_pattern(pxd->regex, NULL); int ret = pcre2_match(pxd->regex, (PCRE2_SPTR8)input, input_len, 0, 0, match, pxd->context); @@ -172,9 +178,24 @@ static void DetectTransformPcrexform( ret = pcre2_substring_get_bynumber(match, 1, (PCRE2_UCHAR8 **)&str, &caplen); if (ret >= 0) { - InspectionBufferCopy(buffer, (uint8_t *)str, (uint32_t)caplen); + if (caplen == 0) { + /* an empty capture leaves an empty buffer; InspectionBufferCopy + * would keep the input when the buffer has no room yet */ + buffer->inspect_len = 0; + buffer->initialized = true; + } else if (SCInspectionBufferCheckAndExpand(buffer, (uint32_t)caplen) == NULL) { + /* InspectionBufferCopy does not report a failed expand: it + * skips the copy or copies only what fits */ + SCInspectionBufferSetError(buffer); + } else { + InspectionBufferCopy(buffer, (uint8_t *)str, (uint32_t)caplen); + } pcre2_substring_free((PCRE2_UCHAR8 *)str); + } else { + SCInspectionBufferSetError(buffer); } + } else { + SCInspectionBufferSetError(buffer); } pcre2_match_data_free(match); } diff --git a/src/tests/detect-engine-content-inspection.c b/src/tests/detect-engine-content-inspection.c index e5b20b1e00..4a330f810b 100644 --- a/src/tests/detect-engine-content-inspection.c +++ b/src/tests/detect-engine-content-inspection.c @@ -402,6 +402,100 @@ static int DetectEngineContentInspectionTest19(void) PASS; } +/** \test pcrexform flags an error and keeps the input when it cannot get room + * for the capture */ +static int DetectEngineContentInspectionTest20(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(http.uri; pcrexform:\"^/(.*)$\"; " + "content:\"foo\"; sid:1;)"); + FAIL_IF_NULL(s); + const DetectBufferType *map = + DetectEngineBufferTypeGetById(de_ctx, s->init_data->buffers[0].id); + FAIL_IF_NULL(map); + FAIL_IF_NOT(map->transforms.cnt == 1); + + const uint8_t data[] = "/foobar"; + const uint32_t data_len = sizeof(data) - 1; + InspectionBuffer buffer; + memset(&buffer, 0, sizeof(buffer)); +#ifdef DEBUG_VALIDATION + buffer.multi = true; +#endif + /* as in DetectEngineContentInspectionTest19, a NULL buf with a large + * enough size stands in for a failed SCRealloc */ + buffer.size = 4096; + + InspectionBufferSetupMulti(NULL, &buffer, &map->transforms, data, data_len); + FAIL_IF_NOT(buffer.flags & DETECT_CI_FLAGS_ERROR); + FAIL_IF_NOT(buffer.inspect == data); + FAIL_IF_NOT(buffer.inspect_len == data_len); + + DetectEngineCtxFree(de_ctx); + PASS; +} + +/** \brief set up a buffer with data and run the transforms of the first + * buffer in the rule over it */ +static InspectionBuffer *ContentInspectionTestTransform(DetectEngineCtx *de_ctx, const char *rule, + InspectionBuffer *buffer, const uint8_t *data, uint32_t data_len) +{ + Signature *s = DetectEngineAppendSig(de_ctx, rule); + if (s == NULL) + return NULL; + const DetectBufferType *map = + DetectEngineBufferTypeGetById(de_ctx, s->init_data->buffers[0].id); + if (map == NULL) + return NULL; + memset(buffer, 0, sizeof(*buffer)); +#ifdef DEBUG_VALIDATION + buffer->multi = true; +#endif + InspectionBufferSetupMulti(NULL, buffer, &map->transforms, data, data_len); + return buffer; +} + +/** \test pcrexform with an empty capture leaves an empty buffer and no error */ +static int DetectEngineContentInspectionTest21(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + const uint8_t data[] = "/"; + InspectionBuffer buffer; + FAIL_IF_NULL(ContentInspectionTestTransform(de_ctx, + "alert http any any -> any any (http.uri; pcrexform:\"^/(.*)$\"; " + "content:\"x\"; sid:1;)", + &buffer, data, sizeof(data) - 1)); + FAIL_IF_NOT(buffer.inspect_len == 0); + FAIL_IF(buffer.flags & DETECT_CI_FLAGS_ERROR); + InspectionBufferFree(&buffer); + DetectEngineCtxFree(de_ctx); + PASS; +} + +/** \test pcrexform leaves an empty buffer alone and flags no error */ +static int DetectEngineContentInspectionTest22(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + const uint8_t data[] = ""; + InspectionBuffer buffer; + FAIL_IF_NULL(ContentInspectionTestTransform(de_ctx, + "alert http any any -> any any (http.uri; pcrexform:\"^(x.*)$\"; " + "content:\"x\"; sid:1;)", + &buffer, data, 0)); + FAIL_IF_NOT(buffer.inspect_len == 0); + FAIL_IF(buffer.flags & DETECT_CI_FLAGS_ERROR); + InspectionBufferFree(&buffer); + DetectEngineCtxFree(de_ctx); + PASS; +} + void DetectEngineContentInspectionRegisterTests(void) { UtRegisterTest("DetectEngineContentInspectionTest01", @@ -438,6 +532,12 @@ void DetectEngineContentInspectionRegisterTests(void) "DetectEngineContentInspectionTest18 error flag", DetectEngineContentInspectionTest18); UtRegisterTest("DetectEngineContentInspectionTest19 from_base64 allocation failure", DetectEngineContentInspectionTest19); + UtRegisterTest("DetectEngineContentInspectionTest20 pcrexform allocation failure", + DetectEngineContentInspectionTest20); + UtRegisterTest("DetectEngineContentInspectionTest21 pcrexform empty capture", + DetectEngineContentInspectionTest21); + UtRegisterTest("DetectEngineContentInspectionTest22 pcrexform empty input", + DetectEngineContentInspectionTest22); } #undef TEST_HEADER From 99ab49638d58beff6d66fb639dd50a205e93f7a5 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Mon, 2 Feb 2026 09:20:05 -0500 Subject: [PATCH 06/13] detect/transform_result: add keyword for transform outcome Add transform_result, which matches on the error flag a transform sets on its buffer: must_succeed does not match on an error, must_error matches only on one, and error_or matches on an error, an absent buffer, or the keywords after it. It shares DetectAbsentData with absent, whose bool or_else becomes an enum DetectAbsentMode, and absent rejects the transform_result arguments. The validator requires a transform that can fail before transform_result, rejects absent: or_else on such a buffer, rejects must_error with other keywords and fast_pattern on the buffer, and says when a transform placed after the keyword started a new buffer. The engine analyzer reports a mode string for both keywords in place of the or_else boolean. Issue: 7313 Issue: 7114 --- src/detect-engine-analyzer.c | 10 +- src/detect-engine-content-inspection.c | 46 ++- src/detect-engine-mpm.c | 6 +- src/detect-engine-register.h | 3 +- src/detect-isdataat.c | 453 +++++++++++++++++++++++-- src/detect-isdataat.h | 19 +- src/detect-parse.c | 4 +- 7 files changed, 481 insertions(+), 60 deletions(-) diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c index 44faa77005..9f02e24e97 100644 --- a/src/detect-engine-analyzer.c +++ b/src/detect-engine-analyzer.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2025 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -872,10 +872,12 @@ static void DumpMatches(RuleAnalyzer *ctx, SCJsonBuilder *js, const SigMatchData SCJbClose(js); break; } - case DETECT_ABSENT: { + case DETECT_ABSENT: + case DETECT_TRANSFORM_RESULT: { const DetectAbsentData *dad = (const DetectAbsentData *)smd->ctx; - SCJbOpenObject(js, "absent"); - SCJbSetBool(js, "or_else", dad->or_else); + SCJbOpenObject( + js, smd->type == DETECT_TRANSFORM_RESULT ? "transform_result" : "absent"); + SCJbSetString(js, "mode", DetectAbsentModeStr(dad->mode)); SCJbClose(js); break; } diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index e2037a748e..4c362602bd 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2023 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -391,13 +391,35 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, prev_offset); } while(1); - } else if (smd->type == DETECT_ABSENT) { - const DetectAbsentData *id = (DetectAbsentData *)smd->ctx; - if (!id->or_else) { - // we match only on absent buffer - goto no_match; + } else if (smd->type == DETECT_ABSENT || smd->type == DETECT_TRANSFORM_RESULT) { + const DetectAbsentData *dad = (const DetectAbsentData *)smd->ctx; + switch (dad->mode) { + case DETECT_ABSENT_MUST_ERROR: + if (flags & DETECT_CI_FLAGS_ERROR) { + SCLogDebug("transform_result: must_error: error flag match"); + goto final_match; + } + goto no_match; + case DETECT_ABSENT_ONLY: + goto no_match; + case DETECT_ABSENT_ERROR_OR: + if (flags & DETECT_CI_FLAGS_ERROR) { + SCLogDebug("transform_result: error_or: error flag match"); + goto final_match; + } + /* fall through */ + case DETECT_ABSENT_OR_ELSE: + goto match; + case DETECT_ABSENT_MUST_SUCCEED: + if (flags & DETECT_CI_FLAGS_ERROR) { + SCLogDebug("transform_result: must_succeed: error flag, no match"); + goto no_match; + } + goto match; + default: + DEBUG_VALIDATE_BUG_ON(1); + goto no_match; } - goto match; } else if (smd->type == DETECT_ISDATAAT) { SCLogDebug("inspecting isdataat"); @@ -787,14 +809,20 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd) { - // we will match on NULL buffers if there is one absent keyword - // absent: error_or does NOT match on absent buffers (only on transform errors) + // match on NULL buffers if the buffer carries absent, or + // transform_result: error_or, which matches an absent buffer as + // absent: or_else does. must_succeed and must_error never match one. bool absent_data = false; while (1) { if (smd->type == DETECT_ABSENT) { absent_data = true; break; } + if (smd->type == DETECT_TRANSFORM_RESULT) { + const DetectAbsentData *dad = (const DetectAbsentData *)smd->ctx; + absent_data = dad->mode == DETECT_ABSENT_ERROR_OR; + break; + } if (smd->is_last) { break; } diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 88e807a4f5..ca1ecb4f81 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2021 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -1233,8 +1233,8 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s) } for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) { - // a buffer with absent keyword cannot be used as fast_pattern - if (sm->type == DETECT_ABSENT) + // a buffer with absent or transform_result cannot use fast_pattern + if (sm->type == DETECT_ABSENT || sm->type == DETECT_TRANSFORM_RESULT) break; if (sm->type != DETECT_CONTENT) continue; diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index 748388b86f..6baeb1bea7 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2024 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -102,6 +102,7 @@ enum DetectKeywordId { DETECT_ISDATAAT, DETECT_URILEN, DETECT_ABSENT, + DETECT_TRANSFORM_RESULT, DETECT_ENTROPY, /* end of content inspection */ diff --git a/src/detect-isdataat.c b/src/detect-isdataat.c index bcda4f7b34..9161117eec 100644 --- a/src/detect-isdataat.c +++ b/src/detect-isdataat.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2020 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -68,85 +68,253 @@ static void DetectAbsentFree(DetectEngineCtx *de_ctx, void *ptr) SCFree(ptr); } -static int DetectAbsentSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr) +const char *DetectAbsentModeStr(enum DetectAbsentMode mode) +{ + switch (mode) { + case DETECT_ABSENT_ONLY: + return "only"; + case DETECT_ABSENT_OR_ELSE: + return "or_else"; + case DETECT_ABSENT_ERROR_OR: + return "error_or"; + case DETECT_ABSENT_MUST_ERROR: + return "must_error"; + case DETECT_ABSENT_MUST_SUCCEED: + return "must_succeed"; + } + return "unknown"; +} + +static int DetectAbsentAppend(DetectEngineCtx *de_ctx, Signature *s, enum DetectAbsentMode mode, + uint16_t sm_type, const char *kw) { if (s->init_data->list == DETECT_SM_LIST_NOTSET) { - SCLogError("no buffer for absent keyword"); + SCLogError("no buffer for %s keyword", kw); return -1; } if (DetectBufferGetActiveList(de_ctx, s) == -1) return -1; - bool or_else; - if (optstr == NULL) { - or_else = false; - } else if (strcmp(optstr, "or_else") == 0) { - or_else = true; - } else { - SCLogError("unhandled value for absent keyword: %s", optstr); - return -1; - } if (s->init_data->curbuf == NULL || s->init_data->list != (int)s->init_data->curbuf->id) { - SCLogError("unspected buffer for absent keyword"); + SCLogError("unexpected buffer for %s keyword", kw); return -1; } const DetectBufferType *b = DetectEngineBufferTypeGetById(de_ctx, s->init_data->list); if (!b || b->frame) { - SCLogError("absent does not work with frames"); + SCLogError("%s does not work with frames", kw); return -1; } if (s->init_data->curbuf->tail != NULL) { - SCLogError("absent must come first right after buffer"); + SCLogError("%s must come first right after buffer", kw); return -1; } DetectAbsentData *dad = SCMalloc(sizeof(DetectAbsentData)); if (unlikely(dad == NULL)) return -1; - dad->or_else = or_else; + dad->mode = mode; - if (SCSigMatchAppendSMToList( - de_ctx, s, DETECT_ABSENT, (SigMatchCtx *)dad, s->init_data->list) == NULL) { + if (SCSigMatchAppendSMToList(de_ctx, s, sm_type, (SigMatchCtx *)dad, s->init_data->list) == + NULL) { DetectAbsentFree(de_ctx, dad); return -1; } return 0; } -bool DetectAbsentValidateContentCallback(const Signature *s, const SignatureInitDataBuffer *b) +static int DetectAbsentSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr) { + enum DetectAbsentMode mode = DETECT_ABSENT_ONLY; + + if (optstr == NULL || strcmp(optstr, "or_else") == 0) { + mode = optstr == NULL ? DETECT_ABSENT_ONLY : DETECT_ABSENT_OR_ELSE; + } else if (strcmp(optstr, "error_or") == 0 || strcmp(optstr, "must_error") == 0 || + strcmp(optstr, "must_succeed") == 0) { + SCLogError("absent does not take %s; use `transform_result: %s;` instead", optstr, optstr); + return -1; + } else { + SCLogError("unhandled value for absent keyword: %s. " + "Valid options are 'or_else' or no argument; " + "to match on the result of a transform, use the transform_result keyword " + "(must_succeed, must_error, error_or).", + optstr); + return -1; + } + return DetectAbsentAppend(de_ctx, s, mode, DETECT_ABSENT, "absent"); +} + +static int DetectTransformResultSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr) +{ + if (optstr == NULL) { + SCLogError("transform_result requires one of must_succeed, must_error, error_or"); + return -1; + } + + enum DetectAbsentMode mode; + if (strcmp(optstr, "must_succeed") == 0) { + mode = DETECT_ABSENT_MUST_SUCCEED; + } else if (strcmp(optstr, "must_error") == 0) { + mode = DETECT_ABSENT_MUST_ERROR; + } else if (strcmp(optstr, "error_or") == 0) { + mode = DETECT_ABSENT_ERROR_OR; + } else { + SCLogError("unhandled value for transform_result keyword: %s. " + "Valid options are must_succeed, must_error, error_or.", + optstr); + return -1; + } + return DetectAbsentAppend(de_ctx, s, mode, DETECT_TRANSFORM_RESULT, "transform_result"); +} + +/* id of the buffer before any transforms were applied */ +static int DetectAbsentBaseBufferId(const DetectEngineCtx *de_ctx, const int id) +{ + const DetectBufferType *map = DetectEngineBufferTypeGetById(de_ctx, id); + if (map != NULL && map->transforms.cnt > 0) + return map->parent_id; + return id; +} + +/* A transform after absent or transform_result starts a new buffer, which + * takes the keywords that follow it. Tell whether a later buffer in the + * signature comes from the same base buffer as b. */ +static bool DetectAbsentHasLaterTransformedBuffer( + const DetectEngineCtx *de_ctx, const Signature *s, const SignatureInitDataBuffer *b) +{ + const int base = DetectAbsentBaseBufferId(de_ctx, b->id); + for (uint32_t x = (uint32_t)(b - s->init_data->buffers) + 1; x < s->init_data->buffer_index; + x++) { + const SignatureInitDataBuffer *later = &s->init_data->buffers[x]; + if (later->id == b->id || DetectAbsentBaseBufferId(de_ctx, later->id) != base) + continue; + const DetectBufferType *map = DetectEngineBufferTypeGetById(de_ctx, later->id); + if (map != NULL && map->transforms.cnt > 0) + return true; + } + return false; +} + +bool DetectAbsentValidateContentCallback( + const DetectEngineCtx *de_ctx, const Signature *s, const SignatureInitDataBuffer *b) +{ + const DetectAbsentData *absent = NULL; + int absent_count = 0; bool has_other = false; - bool only_absent = false; - bool has_absent = false; + bool has_fast_pattern = false; + for (const SigMatch *sm = b->head; sm != NULL; sm = sm->next) { - if (sm->type == DETECT_ABSENT) { - has_absent = true; - const DetectAbsentData *dad = (const DetectAbsentData *)sm->ctx; - if (!dad->or_else) { - only_absent = true; - } + if (sm->type == DETECT_ABSENT || sm->type == DETECT_TRANSFORM_RESULT) { + absent_count++; + absent = (const DetectAbsentData *)sm->ctx; } else { has_other = true; if (sm->type == DETECT_CONTENT) { const DetectContentData *cd = (DetectContentData *)sm->ctx; - if (has_absent && (cd->flags & DETECT_CONTENT_FAST_PATTERN)) { - SCLogError("signature can't have absent and fast_pattern on the same buffer"); - return false; - } + if (cd->flags & DETECT_CONTENT_FAST_PATTERN) + has_fast_pattern = true; } } } - if (only_absent && has_other) { - SCLogError("signature can't have a buffer tested absent and tested with other keywords " - "such as content"); + if (absent == NULL) + return true; + + const bool is_transform_result = + (absent->mode == DETECT_ABSENT_ERROR_OR || absent->mode == DETECT_ABSENT_MUST_ERROR || + absent->mode == DETECT_ABSENT_MUST_SUCCEED); + const char *kw = is_transform_result ? "transform_result" : "absent"; + + if (absent_count > 1) { + SCLogError("signature can't have more than one absent/transform_result keyword on the same " + "buffer"); return false; - } else if (has_absent && !only_absent && !has_other) { - SCLogError( - "signature with absent: or_else expects other keywords to test on such as content"); + } + if (has_fast_pattern) { + SCLogError("signature can't have %s and fast_pattern on the same buffer", kw); return false; } + + /* mode vs. presence of other keywords */ + switch (absent->mode) { + case DETECT_ABSENT_ONLY: + if (has_other) { + SCLogError("signature can't have a buffer tested absent and tested with other " + "keywords such as content"); + return false; + } + return true; + case DETECT_ABSENT_MUST_ERROR: + if (has_other) { + SCLogError("transform_result: must_error cannot be combined with other keywords"); + return false; + } + break; + case DETECT_ABSENT_OR_ELSE: + if (!has_other && DetectAbsentHasLaterTransformedBuffer(de_ctx, s, b)) { + SCLogError("absent: or_else must come after the buffer's transforms; a transform " + "after it starts a new buffer"); + return false; + } + if (!has_other) { + SCLogError("signature with absent: or_else expects other keywords to test on such " + "as content"); + return false; + } + break; + case DETECT_ABSENT_ERROR_OR: + case DETECT_ABSENT_MUST_SUCCEED: + if (!has_other && DetectAbsentHasLaterTransformedBuffer(de_ctx, s, b)) { + SCLogError("transform_result: %s must come after the buffer's transforms; a " + "transform after it starts a new buffer", + DetectAbsentModeStr(absent->mode)); + return false; + } + if (!has_other) { + SCLogError("signature with transform_result: %s expects other keywords to test on " + "such as content", + DetectAbsentModeStr(absent->mode)); + return false; + } + break; + } + + /* mode vs. transform capability */ + const DetectBufferType *map = DetectEngineBufferTypeGetById(de_ctx, b->id); + DEBUG_VALIDATE_BUG_ON(map == NULL); + if (map == NULL) + return false; + + bool has_can_fail_transform = false; + for (int i = 0; i < map->transforms.cnt; i++) { + int transform = map->transforms.transforms[i].transform; + if (sigmatch_table[transform].flags & SIGMATCH_TRANSFORM_CAN_FAIL) { + has_can_fail_transform = true; + break; + } + } + + switch (absent->mode) { + case DETECT_ABSENT_ERROR_OR: + case DETECT_ABSENT_MUST_ERROR: + case DETECT_ABSENT_MUST_SUCCEED: + if (!has_can_fail_transform) { + SCLogError("transform_result requires a transform that can fail " + "(e.g. from_base64)"); + return false; + } + break; + case DETECT_ABSENT_OR_ELSE: + if (has_can_fail_transform) { + SCLogError("absent: or_else on a buffer with a transform that can fail; " + "use `transform_result: error_or` instead"); + return false; + } + break; + case DETECT_ABSENT_ONLY: + break; + } + return true; } @@ -181,6 +349,13 @@ void DetectIsdataatRegister(void) sigmatch_table[DETECT_ABSENT].RegisterTests = DetectAbsentRegisterTests; #endif + sigmatch_table[DETECT_TRANSFORM_RESULT].name = "transform_result"; + sigmatch_table[DETECT_TRANSFORM_RESULT].desc = + "match on the result of a preceding transform (must_succeed, must_error, error_or)"; + sigmatch_table[DETECT_TRANSFORM_RESULT].url = "/rules/payload-keywords.html#transform-result"; + sigmatch_table[DETECT_TRANSFORM_RESULT].Setup = DetectTransformResultSetup; + sigmatch_table[DETECT_TRANSFORM_RESULT].Free = DetectAbsentFree; + DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); } @@ -715,8 +890,212 @@ static int DetectAbsentTestParse01(void) PASS; } +/** + * \test error_or and must_error validation + */ +static int DetectAbsentTestParse02(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF(de_ctx == NULL); + de_ctx->flags |= DE_QUIET; + + /* transform_result: error_or without a failing transform should fail */ + Signature *s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"error_or without failing transform\"; http.user_agent; " + "transform_result: error_or; content:\"one\"; sid:1;)"); + FAIL_IF(s != NULL); + + /* transform_result: must_error without a failing transform should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"must_error without failing transform\"; http.user_agent; " + "transform_result: must_error; sid:2;)"); + FAIL_IF(s != NULL); + + /* transform_result: must_error with content on same buffer should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"must_error with content\"; http.uri; " + "from_base64: mode strict; transform_result: must_error; content:\"test\"; sid:3;)"); + FAIL_IF(s != NULL); + + /* absent: or_else on buffer with failing transform should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"or_else with failing transform\"; http.uri; " + "from_base64: mode strict; absent: or_else; content:\"test\"; sid:4;)"); + FAIL_IF(s != NULL); + + /* invalid absent option should fail */ + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"invalid absent option\"; http.user_agent; " + "absent: invalid; sid:5;)"); + FAIL_IF(s != NULL); + + /* absent: or_else without other keywords should fail */ + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"or_else alone\"; http.user_agent; " + "absent: or_else; sid:6;)"); + FAIL_IF(s != NULL); + + /* transform_result: error_or without other keywords should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"error_or alone\"; http.uri; " + "from_base64: mode strict; transform_result: error_or; sid:7;)"); + FAIL_IF(s != NULL); + + /* transform_result: must_succeed without a failing transform should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"must_succeed without failing transform\"; http.user_agent; " + "transform_result: must_succeed; content:\"test\"; sid:8;)"); + FAIL_IF(s != NULL); + + /* transform_result: must_succeed without other keywords should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"must_succeed alone\"; http.uri; " + "from_base64: mode strict; transform_result: must_succeed; sid:9;)"); + FAIL_IF(s != NULL); + + /* absent does not take transform_result's arguments */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"absent with must_succeed\"; http.uri; " + "from_base64: mode strict; absent: must_succeed; content:\"test\"; sid:10;)"); + FAIL_IF(s != NULL); + + DetectEngineCtxFree(de_ctx); + PASS; +} + +/** + * \test Valid absent configurations should succeed + */ +static int DetectAbsentTestParse03(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF(de_ctx == NULL); + de_ctx->flags |= DE_QUIET; + + /* valid: bare absent */ + Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"bare absent\"; http.user_agent; " + "absent; sid:1;)"); + FAIL_IF(s == NULL); + + /* valid: or_else with content */ + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"or_else with content\"; http.user_agent; " + "absent: or_else; content:!\"one\"; sid:2;)"); + FAIL_IF(s == NULL); + + /* valid: error_or with failing transform and content */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"error_or with from_base64\"; http.uri; " + "from_base64: mode strict; transform_result: error_or; content:\"test\"; sid:3;)"); + FAIL_IF(s == NULL); + + /* valid: must_error standalone with failing transform */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"must_error standalone\"; http.uri; " + "from_base64: mode strict; transform_result: must_error; sid:4;)"); + FAIL_IF(s == NULL); + + /* valid: must_error cross-buffer */ + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"must_error cross-buffer\"; http.uri; " + "from_base64: mode strict; transform_result: must_error; " + "http.host; content:\"test\"; sid:5;)"); + FAIL_IF(s == NULL); + + /* valid: must_succeed with failing transform and content */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"must_succeed with from_base64\"; http.uri; " + "from_base64: mode strict; transform_result: must_succeed; content:\"test\"; sid:6;)"); + FAIL_IF(s == NULL); + + DetectEngineCtxFree(de_ctx); + PASS; +} + +/** + * \test Duplicate-absent and fast_pattern rejection in DetectAbsentValidateContentCallback + */ +static int DetectAbsentTestParse04(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF(de_ctx == NULL); + de_ctx->flags |= DE_QUIET; + + /* two bare absents on the same buffer should fail */ + Signature *s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"dup bare absent\"; http.user_agent; " + "absent; absent; sid:1;)"); + FAIL_IF(s != NULL); + + /* bare absent followed by absent: or_else on the same buffer should fail */ + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"dup absent mixed modes\"; http.user_agent; " + "absent; absent: or_else; content:!\"one\"; sid:2;)"); + FAIL_IF(s != NULL); + + /* two absent: or_else on the same buffer should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"dup or_else\"; http.user_agent; " + "absent: or_else; absent: or_else; content:!\"one\"; sid:3;)"); + FAIL_IF(s != NULL); + + /* bare absent + fast_pattern on the same buffer should fail */ + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"bare absent + fast_pattern\"; http.user_agent; " + "absent; content:\"one\"; fast_pattern; sid:4;)"); + FAIL_IF(s != NULL); + + /* absent: or_else + fast_pattern on the same buffer should fail */ + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"or_else + fast_pattern\"; http.user_agent; " + "absent: or_else; content:!\"one\"; fast_pattern; sid:5;)"); + FAIL_IF(s != NULL); + + /* transform_result: error_or + fast_pattern on the same buffer should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"error_or + fast_pattern\"; http.uri; " + "from_base64: mode strict; transform_result: error_or; content:\"x\"; fast_pattern; " + "sid:6;)"); + FAIL_IF(s != NULL); + + /* transform_result: must_succeed + fast_pattern on the same buffer should fail */ + s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"must_succeed + fast_pattern\"; http.uri; " + "from_base64: mode strict; transform_result: must_succeed; content:\"x\"; " + "fast_pattern; sid:7;)"); + FAIL_IF(s != NULL); + + /* fast_pattern appearing before absent on the same buffer should still fail + * (the simplified walk is order-independent for fast_pattern detection) */ + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"fast_pattern before or_else\"; http.user_agent; " + "content:!\"one\"; fast_pattern; absent: or_else; sid:8;)"); + FAIL_IF(s != NULL); + + DetectEngineCtxFree(de_ctx); + PASS; +} + void DetectAbsentRegisterTests(void) { UtRegisterTest("DetectAbsentTestParse01", DetectAbsentTestParse01); + UtRegisterTest("DetectAbsentTestParse02", DetectAbsentTestParse02); + UtRegisterTest("DetectAbsentTestParse03", DetectAbsentTestParse03); + UtRegisterTest("DetectAbsentTestParse04", DetectAbsentTestParse04); } #endif diff --git a/src/detect-isdataat.h b/src/detect-isdataat.h index f1932ff56c..3d5977c9cb 100644 --- a/src/detect-isdataat.h +++ b/src/detect-isdataat.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -34,14 +34,25 @@ typedef struct DetectIsdataatData_ { uint8_t flags; /* isdataat options*/ } DetectIsdataatData; +enum DetectAbsentMode { + DETECT_ABSENT_ONLY, /**< match only on absent buffer */ + DETECT_ABSENT_OR_ELSE, /**< match on absent buffer or fall through to next keywords */ + DETECT_ABSENT_ERROR_OR, /**< match on transform error or fall through to next keywords */ + DETECT_ABSENT_MUST_ERROR, /**< match only on transform error */ + DETECT_ABSENT_MUST_SUCCEED /**< reject on transform error, fall through to next keywords on + success */ +}; + typedef struct DetectAbsentData_ { - /** absent or try to match with other keywords (false means only absent) */ - bool or_else; + enum DetectAbsentMode mode; } DetectAbsentData; /* prototypes */ void DetectIsdataatRegister (void); -bool DetectAbsentValidateContentCallback(const Signature *s, const SignatureInitDataBuffer *); +bool DetectAbsentValidateContentCallback( + const DetectEngineCtx *de_ctx, const Signature *s, const SignatureInitDataBuffer *); + +const char *DetectAbsentModeStr(enum DetectAbsentMode mode); #endif /* SURICATA_DETECT_ISDATAAT_H */ diff --git a/src/detect-parse.c b/src/detect-parse.c index d79642ffd0..d337a1b8ff 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2025 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -3106,7 +3106,7 @@ static int SigValidateCheckBuffers( if (!DetectBsizeValidateContentCallback(s, b)) { SCReturnInt(0); } - if (!DetectAbsentValidateContentCallback(s, b)) { + if (!DetectAbsentValidateContentCallback(de_ctx, s, b)) { SCReturnInt(0); } } From a83d18d3f71d6e071293d293e8532ba51f620c14 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Mon, 2 Feb 2026 10:17:06 -0500 Subject: [PATCH 07/13] detect: pass transform errors on to content inspection The file.name, file.magic and file.data engines built their content inspection flags without buffer->flags, and DetectGetMultiData and the dns.response getter overwrote buffer->flags after the transforms ran, so a transform's error flag never reached content inspection. OR buffer->flags into the flags passed on, and OR the single-buffer bits into buffer->flags instead of assigning them. Issue: 7313 --- src/detect-dns-response.c | 4 ++-- src/detect-engine.c | 4 ++-- src/detect-file-data.c | 3 ++- src/detect-filemagic.c | 8 +++++--- src/detect-filename.c | 8 +++++--- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/detect-dns-response.c b/src/detect-dns-response.c index 961a738e20..6483f1be48 100644 --- a/src/detect-dns-response.c +++ b/src/detect-dns-response.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2025 Open Information Security Foundation +/* Copyright (C) 2025-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -174,7 +174,7 @@ static InspectionBuffer *GetBuffer(DetectEngineThreadCtx *det_ctx, uint8_t flags } InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len); - buffer->flags = DETECT_CI_FLAGS_SINGLE; + buffer->flags |= DETECT_CI_FLAGS_SINGLE; return buffer; } diff --git a/src/detect-engine.c b/src/detect-engine.c index f8ce5d9cf3..c066aa8477 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2022 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -2373,7 +2373,7 @@ InspectionBuffer *DetectGetMultiData(struct DetectEngineThreadCtx_ *det_ctx, return NULL; } InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len); - buffer->flags = DETECT_CI_FLAGS_SINGLE; + buffer->flags |= DETECT_CI_FLAGS_SINGLE; return buffer; } diff --git a/src/detect-file-data.c b/src/detect-file-data.c index 293b826c73..43fbab0a27 100644 --- a/src/detect-file-data.c +++ b/src/detect-file-data.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2022 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -517,6 +517,7 @@ uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadC uint8_t ciflags = file_done ? DETECT_CI_FLAGS_END : 0; if (buffer->inspect_offset == 0) ciflags |= DETECT_CI_FLAGS_START; + ciflags |= buffer->flags; const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, buffer->inspect, buffer->inspect_len, buffer->inspect_offset, ciflags, diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index 336edc9204..806aecb641 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2023 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -330,9 +330,11 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin continue; } + uint8_t ciflags = DETECT_CI_FLAGS_SINGLE; + ciflags |= buffer->flags; const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, - buffer->inspect, buffer->inspect_len, buffer->inspect_offset, - DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE); + buffer->inspect, buffer->inspect_len, buffer->inspect_offset, ciflags, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE); if (match) { return DETECT_ENGINE_INSPECT_SIG_MATCH; } else { diff --git a/src/detect-filename.c b/src/detect-filename.c index ad5484b641..cf22d6d77b 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2022 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -265,9 +265,11 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine continue; } + uint8_t ciflags = DETECT_CI_FLAGS_SINGLE; + ciflags |= buffer->flags; const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, - buffer->inspect, buffer->inspect_len, buffer->inspect_offset, - DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE); + buffer->inspect, buffer->inspect_len, buffer->inspect_offset, ciflags, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE); if (match) { return DETECT_ENGINE_INSPECT_SIG_MATCH; } else { From 133cf410fdb52d7e2095712d39fafd4785358d50 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Tue, 20 Oct 2026 09:50:58 -0400 Subject: [PATCH 08/13] detect: ignore transform errors on partial body windows file.data and http.request_body run the transforms on each window of a large body or file. A strict from_base64 decode of a window cut off a 4-byte boundary fails, so must_error and error_or fired on valid base64: a 200 KB body of unwrapped base64 raised must_error on both buffers. Mark such windows DETECT_CI_FLAGS_PARTIAL (START and END not both set), and have must_error and error_or ignore the error flag there. must_succeed still keeps content from matching the window. Issue: 7313 Issue: 7114 --- src/detect-engine-content-inspection.c | 7 +++++-- src/detect-engine-content-inspection.h | 3 +++ src/detect-file-data.c | 2 ++ src/detect-http-client-body.c | 4 +++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index 4c362602bd..876131e013 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -394,8 +394,11 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, } else if (smd->type == DETECT_ABSENT || smd->type == DETECT_TRANSFORM_RESULT) { const DetectAbsentData *dad = (const DetectAbsentData *)smd->ctx; switch (dad->mode) { + /* An error on part of a body or file may come from where the + * window was cut, so must_error and error_or only count errors on + * a buffer inspected in one piece. must_succeed still blocks. */ case DETECT_ABSENT_MUST_ERROR: - if (flags & DETECT_CI_FLAGS_ERROR) { + if ((flags & DETECT_CI_FLAGS_ERROR) && !(flags & DETECT_CI_FLAGS_PARTIAL)) { SCLogDebug("transform_result: must_error: error flag match"); goto final_match; } @@ -403,7 +406,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, case DETECT_ABSENT_ONLY: goto no_match; case DETECT_ABSENT_ERROR_OR: - if (flags & DETECT_CI_FLAGS_ERROR) { + if ((flags & DETECT_CI_FLAGS_ERROR) && !(flags & DETECT_CI_FLAGS_PARTIAL)) { SCLogDebug("transform_result: error_or: error flag match"); goto final_match; } diff --git a/src/detect-engine-content-inspection.h b/src/detect-engine-content-inspection.h index 1b983c86e7..aa33d26ea1 100644 --- a/src/detect-engine-content-inspection.h +++ b/src/detect-engine-content-inspection.h @@ -45,6 +45,9 @@ enum DetectContentInspectionType { // #define DETECT_CI_FLAGS_DCE_LE BIT_U8(2) /**< DCERPC record in little endian */ // #define DETECT_CI_FLAGS_DCE_BE BIT_U8(3) /**< DCERPC record in big endian */ #define DETECT_CI_FLAGS_ERROR BIT_U8(4) /**< transformation/decode error occurred */ +/** buffer is one window of a body or file that is inspected in several parts, + * so a transform error may come from where the window was cut */ +#define DETECT_CI_FLAGS_PARTIAL BIT_U8(5) /** buffer is a single, non-streaming, buffer. Data sent to the content * inspection function contains both start and end of the data. */ diff --git a/src/detect-file-data.c b/src/detect-file-data.c index 43fbab0a27..7f143ec9aa 100644 --- a/src/detect-file-data.c +++ b/src/detect-file-data.c @@ -517,6 +517,8 @@ uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadC uint8_t ciflags = file_done ? DETECT_CI_FLAGS_END : 0; if (buffer->inspect_offset == 0) ciflags |= DETECT_CI_FLAGS_START; + if ((ciflags & DETECT_CI_FLAGS_SINGLE) != DETECT_CI_FLAGS_SINGLE) + ciflags |= DETECT_CI_FLAGS_PARTIAL; ciflags |= buffer->flags; const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c index 44a03c2208..bb1454068a 100644 --- a/src/detect-http-client-body.c +++ b/src/detect-http-client-body.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2021 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -328,6 +328,8 @@ static uint8_t DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx, uint8_t ci_flags = eof ? DETECT_CI_FLAGS_END : 0; ci_flags |= (offset == 0 ? DETECT_CI_FLAGS_START : 0); + if ((ci_flags & DETECT_CI_FLAGS_SINGLE) != DETECT_CI_FLAGS_SINGLE) + ci_flags |= DETECT_CI_FLAGS_PARTIAL; ci_flags |= buffer->flags; /* Inspect all the uricontents fetched on each From a9d70166ec3aabf8b012850d26371263d28e2d99 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Wed, 7 Oct 2026 10:44:20 -0400 Subject: [PATCH 09/13] detect/mpm: honor absent in every fast_pattern pass RetrieveFPForSig stopped at absent and transform_result only while marking candidate lists; the max_len pass and GetMpmForList walked every instance of the chosen list to its end. Repeated multi-instance buffers with no transforms, or with transforms such as to_lowercase, share a list id, so in "http.request_header; content:"Host"; http.request_header; absent: or_else; content:"";" the or_else content became the fast pattern. Stop at absent and transform_result in all three passes. Issue: 7313 Issue: 7114 --- src/detect-engine-mpm.c | 14 ++++++++++++- src/detect-isdataat.c | 44 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index ca1ecb4f81..fd9a0fd8df 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -1119,10 +1119,20 @@ static void SetMpm(Signature *s, SigMatch *mpm_sm, const int mpm_sm_list) s->init_data->mpm_sm = mpm_sm; } +/* absent and transform_result end the part of a buffer that can supply the + * fast_pattern. Repeated buffers can share a list id, so every pass over a + * list's buffers has to stop at them, not just the first. */ +static inline bool StopsFastPattern(const SigMatch *sm) +{ + return sm->type == DETECT_ABSENT || sm->type == DETECT_TRANSFORM_RESULT; +} + static SigMatch *GetMpmForList(const Signature *s, SigMatch *list, SigMatch *mpm_sm, uint16_t max_len, bool skip_negated_content) { for (SigMatch *sm = list; sm != NULL; sm = sm->next) { + if (StopsFastPattern(sm)) + break; if (sm->type != DETECT_CONTENT) continue; @@ -1234,7 +1244,7 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s) for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) { // a buffer with absent or transform_result cannot use fast_pattern - if (sm->type == DETECT_ABSENT || sm->type == DETECT_TRANSFORM_RESULT) + if (StopsFastPattern(sm)) break; if (sm->type != DETECT_CONTENT) continue; @@ -1346,6 +1356,8 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s) DetectEngineBufferTypeGetNameById(de_ctx, list_id)); for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) { + if (StopsFastPattern(sm)) + break; if (sm->type != DETECT_CONTENT) continue; diff --git a/src/detect-isdataat.c b/src/detect-isdataat.c index 9161117eec..fc11c6b5a6 100644 --- a/src/detect-isdataat.c +++ b/src/detect-isdataat.c @@ -1091,11 +1091,55 @@ static int DetectAbsentTestParse04(void) PASS; } +/** \test fast_pattern is not taken from a buffer with absent when a repeated + * buffer shares its list id */ +static int DetectAbsentTestParse05(void) +{ + const char *sigs[] = { + "alert http any any -> any any (http.request_header; content:\"Host\"; " + "http.request_header; absent: or_else; content:\"a-much-longer-pattern\"; sid:1;)", + "alert http any any -> any any (http.request_header; absent: or_else; " + "content:\"a-much-longer-pattern\"; http.request_header; content:\"Host\"; sid:2;)", + "alert http any any -> any any (http.request_header; to_lowercase; content:\"host\"; " + "http.request_header; to_lowercase; absent: or_else; " + "content:\"a-much-longer-pattern\"; sid:3;)", + }; + + for (size_t i = 0; i < ARRAY_SIZE(sigs); i++) { + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, sigs[i]); + FAIL_IF_NULL(s); + FAIL_IF_NOT(s->init_data->buffer_index == 2); + FAIL_IF_NOT(s->init_data->buffers[0].id == s->init_data->buffers[1].id); + + int mpm_cnt = 0; + for (uint32_t x = 0; x < s->init_data->buffer_index; x++) { + for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) { + if (sm->type != DETECT_CONTENT) + continue; + const DetectContentData *cd = (const DetectContentData *)sm->ctx; + if (cd->flags & DETECT_CONTENT_MPM) { + FAIL_IF_NOT(cd->content_len == 4); + mpm_cnt++; + } + } + } + FAIL_IF_NOT(mpm_cnt == 1); + + DetectEngineCtxFree(de_ctx); + } + PASS; +} + void DetectAbsentRegisterTests(void) { UtRegisterTest("DetectAbsentTestParse01", DetectAbsentTestParse01); UtRegisterTest("DetectAbsentTestParse02", DetectAbsentTestParse02); UtRegisterTest("DetectAbsentTestParse03", DetectAbsentTestParse03); UtRegisterTest("DetectAbsentTestParse04", DetectAbsentTestParse04); + UtRegisterTest("DetectAbsentTestParse05", DetectAbsentTestParse05); } #endif From 15a068258dc45c70dd20ab5c82ceb0875e3af0fc Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Tue, 20 Oct 2026 09:56:41 -0400 Subject: [PATCH 10/13] detect/mpm: use content after must_succeed as fast_pattern transform_result stopped fast_pattern selection, and the validator rejected fast_pattern on its buffer. Content after must_succeed only matches when the transform succeeded, and the prefilter scans the same transformed bytes, so "http.uri; from_base64; transform_result: must_succeed; content:"evil";" lost its only fast pattern and ran on every HTTP transaction. Let content after must_succeed be the fast pattern, and accept fast_pattern there. Issue: 7313 Issue: 7114 --- src/detect-engine-mpm.c | 11 +++++++--- src/detect-isdataat.c | 45 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index fd9a0fd8df..91fc9cbf9f 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -35,6 +35,7 @@ #include "detect-engine.h" #include "detect-engine-siggroup.h" #include "detect-engine-mpm.h" +#include "detect-isdataat.h" #include "detect-engine-iponly.h" #include "detect-parse.h" #include "detect-engine-prefilter.h" @@ -1120,11 +1121,15 @@ static void SetMpm(Signature *s, SigMatch *mpm_sm, const int mpm_sm_list) } /* absent and transform_result end the part of a buffer that can supply the - * fast_pattern. Repeated buffers can share a list id, so every pass over a - * list's buffers has to stop at them, not just the first. */ + * fast_pattern. transform_result: must_succeed does not: content after it + * can only match when the transform succeeded, and the prefilter then scans + * the same transformed bytes. Repeated buffers can share a list id, so every + * pass over a list's buffers has to stop at them, not just the first. */ static inline bool StopsFastPattern(const SigMatch *sm) { - return sm->type == DETECT_ABSENT || sm->type == DETECT_TRANSFORM_RESULT; + if (sm->type == DETECT_TRANSFORM_RESULT) + return ((const DetectAbsentData *)sm->ctx)->mode != DETECT_ABSENT_MUST_SUCCEED; + return sm->type == DETECT_ABSENT; } static SigMatch *GetMpmForList(const Signature *s, SigMatch *list, SigMatch *mpm_sm, diff --git a/src/detect-isdataat.c b/src/detect-isdataat.c index fc11c6b5a6..df7e74d4ee 100644 --- a/src/detect-isdataat.c +++ b/src/detect-isdataat.c @@ -230,7 +230,7 @@ bool DetectAbsentValidateContentCallback( "buffer"); return false; } - if (has_fast_pattern) { + if (has_fast_pattern && absent->mode != DETECT_ABSENT_MUST_SUCCEED) { SCLogError("signature can't have %s and fast_pattern on the same buffer", kw); return false; } @@ -1072,13 +1072,13 @@ static int DetectAbsentTestParse04(void) "sid:6;)"); FAIL_IF(s != NULL); - /* transform_result: must_succeed + fast_pattern on the same buffer should fail */ + /* transform_result: must_succeed allows fast_pattern on the same buffer */ s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " "(msg:\"must_succeed + fast_pattern\"; http.uri; " "from_base64: mode strict; transform_result: must_succeed; content:\"x\"; " "fast_pattern; sid:7;)"); - FAIL_IF(s != NULL); + FAIL_IF_NULL(s); /* fast_pattern appearing before absent on the same buffer should still fail * (the simplified walk is order-independent for fast_pattern detection) */ @@ -1134,6 +1134,44 @@ static int DetectAbsentTestParse05(void) PASS; } +/** \test content after transform_result: must_succeed can be the fast_pattern; + * content after error_or cannot */ +static int DetectAbsentTestParse06(void) +{ + const struct { + const char *sig; + bool mpm; + } tests[] = { + { "alert http any any -> any any (http.uri; from_base64; " + "transform_result: must_succeed; content:\"evil\"; sid:1;)", + true }, + { "alert http any any -> any any (http.uri; from_base64; " + "transform_result: error_or; content:\"evil\"; sid:2;)", + false }, + }; + + for (size_t i = 0; i < ARRAY_SIZE(tests); i++) { + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF_NULL(de_ctx); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, tests[i].sig); + FAIL_IF_NULL(s); + FAIL_IF_NOT(s->init_data->buffer_index == 1); + + bool mpm = false; + for (SigMatch *sm = s->init_data->buffers[0].head; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_CONTENT && + (((const DetectContentData *)sm->ctx)->flags & DETECT_CONTENT_MPM)) + mpm = true; + } + FAIL_IF_NOT(mpm == tests[i].mpm); + + DetectEngineCtxFree(de_ctx); + } + PASS; +} + void DetectAbsentRegisterTests(void) { UtRegisterTest("DetectAbsentTestParse01", DetectAbsentTestParse01); @@ -1141,5 +1179,6 @@ void DetectAbsentRegisterTests(void) UtRegisterTest("DetectAbsentTestParse03", DetectAbsentTestParse03); UtRegisterTest("DetectAbsentTestParse04", DetectAbsentTestParse04); UtRegisterTest("DetectAbsentTestParse05", DetectAbsentTestParse05); + UtRegisterTest("DetectAbsentTestParse06", DetectAbsentTestParse06); } #endif From 47e5ae0f3ce057ed58fcc936a09585b14c6a7567 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Tue, 20 Oct 2026 10:01:26 -0400 Subject: [PATCH 11/13] transform/decompress: flag gunzip and zlib_deflate errors gunzip and zlib_deflate emptied the buffer when the input did not decompress, and gave no signal when they could not get room for the output, so transform_result could not be used with them. Mark both SIGMATCH_TRANSFORM_CAN_FAIL, keep the input and flag the error on either failure. When the input was in the buffer's own memory, put the saved copy back. Issue: 7313 --- rust/src/detect/transforms/decompress.rs | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/rust/src/detect/transforms/decompress.rs b/rust/src/detect/transforms/decompress.rs index 3104a54b00..0c6184148a 100644 --- a/rust/src/detect/transforms/decompress.rs +++ b/rust/src/detect/transforms/decompress.rs @@ -16,12 +16,12 @@ */ use crate::detect::uint::detect_parse_uint_with_unit; -use crate::detect::SIGMATCH_OPTIONAL_OPT; +use crate::detect::{SIGMATCH_OPTIONAL_OPT, SIGMATCH_TRANSFORM_CAN_FAIL}; use flate2::bufread::{GzDecoder, ZlibDecoder}; use suricata_sys::sys::{ DetectEngineCtx, DetectEngineThreadCtx, InspectionBuffer, SCDetectHelperTransformRegister, SCDetectSignatureAddTransform, SCInspectionBufferCheckAndExpand, SCInspectionBufferInPlace, - SCInspectionBufferTruncate, SCTransformTableElmt, Signature, + SCInspectionBufferSetError, SCInspectionBufferTruncate, SCTransformTableElmt, Signature, }; use std::ffi::CStr; @@ -133,7 +133,8 @@ unsafe fn decompress_transform( return; } let tmp; - let slice_input = if SCInspectionBufferInPlace(buffer) { + let in_place = SCInspectionBufferInPlace(buffer); + let slice_input = if in_place { // need a temporary buffer as we cannot do the transfom in place // rather copy the input which should be smaller than the decompressed output // (Transform is tried in place when there are multiple chained transforms) @@ -146,7 +147,8 @@ unsafe fn decompress_transform( let output = SCInspectionBufferCheckAndExpand(buffer, ctx.max_size); if output.is_null() { - // allocation failure + // allocation failure: the buffer and the input are unchanged + SCInspectionBufferSetError(buffer); return; } let buf = std::slice::from_raw_parts_mut(output, ctx.max_size as usize); @@ -154,10 +156,16 @@ unsafe fn decompress_transform( // this succeeds if decompressed data > max_size, but we get nb = max_size if let Some(nb) = decompress_fn(slice_input, buf) { SCInspectionBufferTruncate(buffer, nb); - } else { - // decompression failure - SCInspectionBufferTruncate(buffer, 0); + return; + } + // decompression failure: keep the input and flag the error + if in_place { + // the input was in buf, which the decoder may have written over or + // which the expand may have moved, so put the saved copy back + std::ptr::copy_nonoverlapping(slice_input.as_ptr(), output, slice_input.len()); + SCInspectionBufferTruncate(buffer, slice_input.len() as u32); } + SCInspectionBufferSetError(buffer); } unsafe extern "C" fn gunzip_transform( @@ -216,7 +224,7 @@ pub unsafe extern "C" fn DetectTransformGunzipRegister() { desc: b"modify buffer via gunzip decompression\0".as_ptr() as *const libc::c_char, url: b"/rules/transforms.html#gunzip\0".as_ptr() as *const libc::c_char, Setup: Some(gunzip_setup), - flags: SIGMATCH_OPTIONAL_OPT, + flags: SIGMATCH_OPTIONAL_OPT | SIGMATCH_TRANSFORM_CAN_FAIL, Transform: Some(gunzip_transform), Free: Some(decompress_free), TransformValidate: None, @@ -237,7 +245,7 @@ pub unsafe extern "C" fn DetectTransformZlibDeflateRegister() { desc: b"modify buffer via zlib decompression\0".as_ptr() as *const libc::c_char, url: b"/rules/transforms.html#zlib-deflate\0".as_ptr() as *const libc::c_char, Setup: Some(zlib_deflate_setup), - flags: SIGMATCH_OPTIONAL_OPT, + flags: SIGMATCH_OPTIONAL_OPT | SIGMATCH_TRANSFORM_CAN_FAIL, Transform: Some(zlib_deflate_transform), Free: Some(decompress_free), TransformValidate: None, From e45e570f59eaecd94b2f8b61a3598ccab77d52e5 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Tue, 3 Feb 2026 09:45:47 -0500 Subject: [PATCH 12/13] doc: describe transform_result and the absent options Document transform_result and both forms of absent, the transforms that can signal an error, how must_error and error_or treat bodies and files inspected in parts, and the rule-load constraints. Note the behavior changes in upgrade.rst: absent's mode string in the analyzer, absent: or_else rejected after a transform that can fail, gunzip and zlib_deflate keeping their input on failure, pcrexform's empty capture, and from_base64's bytes handling. Issue: 7313 Issue: 7114 --- doc/userguide/rules/payload-keywords.rst | 132 +++++++++++++++++++++-- doc/userguide/rules/transforms.rst | 76 +++++++++++++ doc/userguide/upgrade.rst | 28 +++++ 3 files changed, 230 insertions(+), 6 deletions(-) diff --git a/doc/userguide/rules/payload-keywords.rst b/doc/userguide/rules/payload-keywords.rst index 583aea39c3..a05ce1ceff 100644 --- a/doc/userguide/rules/payload-keywords.rst +++ b/doc/userguide/rules/payload-keywords.rst @@ -337,26 +337,146 @@ These differences are also discussed in :doc:`differences-from-snort`. A discussion of this difference can be found at https://redmine.openinfosecfoundation.org/issues/8031 +.. _rules-keyword-absent: + absent ------ -The keyword ``absent`` checks that a sticky buffer does not exist. -It can be used without any argument to match only on absent buffer : +The keyword ``absent`` checks that a sticky buffer does not exist. It takes +either no argument (match only on absent buffer) or ``or_else`` (match on +absent buffer or fall through to subsequent keywords). + +To match on the result of a transform (``must_succeed``, ``must_error``, +``error_or``), see :ref:`rules-keyword-transform-result`. + +absent (no argument) +~~~~~~~~~~~~~~~~~~~~ -Example of ``absent`` in a rule: +Match only when the buffer is absent (NULL). No other keyword is allowed +on the same buffer. .. container:: example-rule alert http any any -> any any (msg:"HTTP request without referer"; :example-rule-emphasis:`http.referer; absent;` sid:1; rev:1;) +absent: or_else +~~~~~~~~~~~~~~~ -It can take an argument "or_else" to match on absent buffer or on what comes next such as negated content, for instance : +Match when the buffer is absent (NULL) OR when subsequent keywords match, +for example a negated content. .. container:: example-rule - alert http any any -> any any (msg:"HTTP request without referer"; :example-rule-emphasis:`http.referer; absent: or_else;` content: !"abc"; sid:1; rev:1;) + alert http any any -> any any (msg:"HTTP request without referer"; \ + :example-rule-emphasis:`http.referer; absent: or_else;` content: !"abc"; sid:1; rev:1;) + +``or_else`` cannot be used on a buffer with a transform that can signal +an error (``from_base64``, ``pcrexform``, ``gunzip``, ``zlib_deflate``); +use ``transform_result: error_or`` instead, which also matches an absent +buffer. + +For files (i.e. ``file.data``), absent means there are no files in the +transaction. + +.. _rules-keyword-transform-result: + +transform_result +---------------- + +``transform_result`` matches on the outcome of a preceding transform. +When a transform that can signal an error cannot complete, it sets an +error flag on the inspection buffer and leaves its input unchanged, so +later transforms and keywords see the data the failed transform received. +When the failing transform is the first on the buffer, that is the +original buffer content. ``transform_result`` reacts to that flag. + +The keyword takes one of ``must_succeed``, ``must_error``, or ``error_or`` +and requires a transform that can signal errors on the same buffer: +``from_base64``, ``pcrexform``, ``gunzip`` or ``zlib_deflate``. See +:ref:`transform-error-signaling`. + +transform_result: must_succeed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Do not match when the transform signals an error, so the keywords after +it only see decoded or extracted data. This keeps a content match from +running against the input bytes after a failed ``from_base64``. Content +after ``must_succeed`` can be the fast pattern. It does not match when the +buffer is absent. + +.. container:: example-rule + + alert http any any -> any any (msg:"Detect malware only in decoded base64"; file.data; \ + from_base64; :example-rule-emphasis:`transform_result: must_succeed;` \ + content:"malware"; sid:1; rev:1;) + +transform_result: must_error +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Match only when the transform signals an error. No other keyword is +allowed on the same buffer, but other sticky buffers can follow to combine +error detection with further matching conditions. It does not match when +the buffer is absent. + +.. container:: example-rule + + alert http any any -> any any (msg:"Detect base64 decode failure"; file.data; \ + from_base64; :example-rule-emphasis:`transform_result: must_error;` sid:1; rev:1;) + +Example combining with a second sticky buffer:: + + alert http any any -> any any (msg:"base64 error and bad host"; \ + http.uri; from_base64: mode strict; transform_result: must_error; \ + http.host; content:"suspicious.example.com"; sid:1;) + +transform_result: error_or +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Match when the transform signals an error, when the buffer is absent, or +when the keywords after it match. It takes the place of ``absent: or_else`` +on a buffer with a transform that can signal an error. + +.. container:: example-rule -For files (i.e ``file.data``), absent means there are no files in the transaction. + alert http any any -> any any (msg:"Detect base64 decode error or malicious content"; \ + file.data; from_base64; :example-rule-emphasis:`transform_result: error_or;` \ + content:"malicious"; sid:1; rev:1;) + +Bodies and files inspected in parts +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``file.data``, ``http.response_body`` and ``http.request_body`` inspect a +large body or file in windows and run the transforms on each window. A +window can cut the data where a transform cannot handle it: a strict +``from_base64`` decode fails on a window that does not start on a 4-byte +boundary. ``must_error`` and ``error_or`` therefore only react to an error +when the whole body or file is inspected at once. ``must_succeed`` still +keeps content from matching a window whose transform failed. + +Rule-writing constraints +~~~~~~~~~~~~~~~~~~~~~~~~ + +The following constraints apply to both ``absent`` and +``transform_result`` and are enforced at rule-load time: + +* The keyword must come first on its buffer, after the sticky buffer + keyword and any transforms. A transform placed after the keyword starts + a new buffer; with ``absent: or_else`` or ``transform_result`` the rule + then fails to load. +* Only one ``absent`` or ``transform_result`` is allowed per buffer, and + the two cannot be mixed on the same buffer. +* ``fast_pattern`` cannot be used on a buffer that carries ``absent``, + ``transform_result: must_error`` or ``transform_result: error_or``. +* ``absent`` and ``transform_result`` do not work on frame buffers. +* ``transform_result`` requires a transform that can signal errors on the + same buffer. +* ``absent: or_else`` cannot be used on such a buffer; use + ``transform_result: error_or`` instead. +* ``absent: or_else``, ``transform_result: error_or`` and + ``transform_result: must_succeed`` need at least one other keyword after + them on the buffer. +* ``must_error`` cannot be combined with other keywords on the same + buffer, but other sticky buffers can follow. bsize ----- diff --git a/doc/userguide/rules/transforms.rst b/doc/userguide/rules/transforms.rst index ae16aafed2..6d8bd81950 100644 --- a/doc/userguide/rules/transforms.rst +++ b/doc/userguide/rules/transforms.rst @@ -167,6 +167,48 @@ on. alert http any any -> any any (http_request_line; to_sha256; \ content:"\|54A9 7A8A B09C 1B81 3725 2214 51D3 F997 F015 9DD7 049E E5AD CED3 945A FC79 7401\|"; sid:1;) +.. _transform-error-signaling: + +Error-Signaling Transforms +-------------------------- + +Some transforms can signal an error when they cannot complete their operation. The transform +then sets an error flag on the inspection buffer and leaves its input unchanged, so later +transforms and keywords see the data the failed transform received. When the failing +transform is the first on the buffer, that is the original buffer content. + +The following transforms can signal errors. Each also signals one when it cannot get memory +for its result. + +- :ref:`pcrexform ` signals an error when the regular expression does not match + or its capture group cannot be extracted. An empty buffer is left as it is, and a capture + of nothing leaves an empty buffer; neither is an error. +- :ref:`from_base64 ` signals an error when a non-empty input is not decoded: + ``offset`` is at or past the end of the buffer, ``bytes`` is larger than the input left + after ``offset``, or decoding produces nothing. Invalid bytes in the input are not an error + in ``rfc2045`` mode, which skips them, or in ``rfc4648`` mode when something decodes before + them; in ``strict`` mode they are. +- :ref:`gunzip ` and :ref:`zlib_deflate ` signal an error when the + input does not decompress. + +The ``transform_result`` keyword matches on the transform outcome: + +- ``transform_result: error_or`` matches if the transform signals an error, if the buffer + is absent, or if the keywords after it match. +- ``transform_result: must_error`` matches only when the transform signals an error. No + other keyword is allowed on the same buffer. It does not match when the buffer is absent. +- ``transform_result: must_succeed`` does not match when the transform signals an error, so + the keywords after it only see decoded or extracted data. It does not match when the + buffer is absent. + +On ``file.data``, ``http.response_body`` and ``http.request_body``, a large body or file is +inspected in windows, and ``must_error`` and ``error_or`` only react to an error when the +whole body or file is inspected at once. + +See :ref:`rules-keyword-transform-result` for full documentation and examples of each option. + +.. _pcrexform: + pcrexform --------- @@ -174,6 +216,9 @@ Takes the buffer, applies the required regular expression, and outputs the *firs .. note:: this transform requires a mandatory option string containing a regular expression. +If the regular expression does not match the buffer content, the transform signals an error +and leaves its input unchanged. See :ref:`transform-error-signaling` for how to +detect or guard against transform failures using ``transform_result``. This example alerts if ``http.request_line`` contains ``/dropper.php``: @@ -182,6 +227,20 @@ This example alerts if ``http.request_line`` contains ``/dropper.php``: alert http any any -> any any (msg:"HTTP with pcrexform"; http.request_line; \ pcrexform:"[a-zA-Z]+\s+(.*)\s+HTTP"; content:"/dropper.php"; sid:1;) +This example relies on the input being kept when ``pcrexform`` fails. It reduces a name with +a subdomain to its last two labels (``www.example.com`` becomes ``example.com``); a name with +no subdomain does not match the expression and is kept as it is. Either way the buffer is +exactly ``example.com``:: + + alert dns any any -> any any (msg:"example.com or a subdomain of it"; \ + dns.query; pcrexform:"\.([^\.]+\.[^\.]+)$"; content:"example.com"; \ + startswith; endswith; sid:1;) + +This example alerts if ``pcrexform`` fails to match (e.g., unexpected request line format):: + + alert http any any -> any any (msg:"pcrexform match failure"; http.request_line; \ + pcrexform:"[a-zA-Z]+\s+(.*)\s+HTTP"; transform_result: must_error; sid:2;) + url_decode ---------- @@ -303,6 +362,13 @@ line breaks, and any non base64 alphabet. Mode ``strict`` will fail if an invalid character is found in the encoded bytes. +.. note:: When a non-empty input is not decoded, ``from_base64`` signals an error and leaves + its input unchanged. That happens when ``offset`` is at or past the end + of the buffer, when ``bytes`` is larger than the input left after ``offset``, or when + decoding produces nothing (for example, ``strict`` mode rejecting a character outside the + base64 alphabet). See :ref:`transform-error-signaling` for how to detect or guard + against decode failures using ``transform_result``. + The following examples will alert when the buffer contents match (see the last ``content`` value for the expected strings). @@ -431,6 +497,8 @@ the buffer. return string.upper(sub), bytes end +.. _gunzip: + gunzip ------ @@ -444,6 +512,9 @@ If the decompressed data were to be larger than max-size, the transform will decompress data up to max-size. Value 0 is forbidden for max-size (there is no unlimited value). +If the input is not gzip data, the transform signals an error and leaves its input +unchanged. See :ref:`transform-error-signaling`. + This example alerts if ``http.uri`` contains base64-encoded gzipped value: .. container:: example-rule @@ -454,6 +525,8 @@ This example alerts if ``http.uri`` contains base64-encoded gzipped value: gunzip: max-size 1024; content:"This is compressed then base64-encoded"; startswith; endswith; sid:2; rev:1;) +.. _zlib_deflate: + zlib_deflate ------------ @@ -467,6 +540,9 @@ If the decompressed data were to be larger than max-size, the transform will decompress data up to max-size. Value 0 is forbidden for max-size (there is no unlimited value). +If the input is not zlib data, the transform signals an error and leaves its input +unchanged. See :ref:`transform-error-signaling`. + This example alerts if ``http.uri`` contains base64-encoded zlib-compressed value: .. container:: example-rule diff --git a/doc/userguide/upgrade.rst b/doc/userguide/upgrade.rst index 1673d0db8a..0a566c3eee 100644 --- a/doc/userguide/upgrade.rst +++ b/doc/userguide/upgrade.rst @@ -156,6 +156,26 @@ Keyword Changes See :doc:`rules/ssh-keywords` for the full state and failure semantics. +- ``absent: or_else`` is now rejected on a buffer whose transform can signal + an error (``from_base64``, ``pcrexform``, ``gunzip``, ``zlib_deflate``), + and such a rule fails to load. Use ``transform_result: error_or`` instead: + it matches when the buffer is absent, when the transform fails, or when + the keywords after it match. See :ref:`rules-keyword-transform-result`. + +- ``gunzip`` and ``zlib_deflate`` keep their input when it does not + decompress, and signal an error that ``transform_result`` can match on. + 8.0 left an empty buffer, so ``bsize:0`` after a failed decompression no + longer matches; use ``transform_result: must_error`` instead. + +- ``pcrexform`` leaves an empty buffer when its capture group matches + nothing. 8.0 could leave the original buffer in place in that case. + +- ``from_base64`` decodes the input when ``bytes`` equals the input left after + ``offset``; 8.0 left such a buffer undecoded. When ``offset`` is at or past + the end of the buffer, or ``bytes`` is larger than the input left after + ``offset``, the buffer is still left undecoded and the transform now signals + an error that ``transform_result`` can match on. + Other Changes ~~~~~~~~~~~~~ - ``engine-analysis`` output has been changed for keywords that now use the generic integers framework @@ -165,6 +185,14 @@ Other Changes - tcp.seq - tcp.window +- Engine analysis (``rules.json``): the ``absent`` object emitted per keyword + changed from ``{"or_else": true|false}`` to ``{"mode": "only|or_else"}``. + The new ``transform_result`` keyword emits + ``{"transform_result": {"mode": "error_or|must_error|must_succeed"}}`` + under the same per-keyword object. Consumers reading the boolean ``or_else`` + field should switch to the string ``mode`` field, and recognize both the + ``absent`` and ``transform_result`` objects. + - ``dnp3`` has reduced the maximum number of open transactions from 500 down to 32, the maximum number of points per message from unbounded to 16384, and the maximum number of objects per message From 031cc11679be17f459f973bb3fd7678be23bf8d9 Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Wed, 21 Oct 2026 08:43:58 -0400 Subject: [PATCH 13/13] detect/content-inspection: assert smd is set DetectEngineContentInspectionInternal checked smd for NULL only after the recursion-limit return had read smd->type, and its NULL branch read smd->type too. No caller passes NULL: engines without a keyword list have no callback, empty sticky buffers fail to load, payload engines exist only with payload keywords, and the recursion stops at the last keyword. Drop smd from the NULL check and assert it with DEBUG_VALIDATE_BUG_ON. --- src/detect-engine-content-inspection.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index 876131e013..36e911e1bc 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -110,6 +110,9 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, const enum DetectContentInspectionType inspection_mode) { SCEnter(); + // every caller passes a keyword list, and the recursion below stops at + // its last entry; the profiling calls rely on that to read smd->type + DEBUG_VALIDATE_BUG_ON(smd == NULL); KEYWORD_PROFILING_START; ctx->recursion.count++; @@ -119,7 +122,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, } // we want the ability to match on bsize: 0 - if (smd == NULL || buffer == NULL) { + if (buffer == NULL) { KEYWORD_PROFILING_END(det_ctx, smd->type, 0); SCReturnInt(0); }