transforms: move strip_whitespace to rust

Ticket: 7229
pull/12094/head
Philippe Antoine 2 years ago committed by Victor Julien
parent dd71ef0af2
commit 4985ebc0e0

@ -83,6 +83,7 @@ include = [
"QuicTransaction",
"FtpEvent",
"SCSigTableElmt",
"SCTransformTableElmt",
]
# A list of items to not include in the generated bindings

@ -25,6 +25,7 @@ pub mod parser;
pub mod requires;
pub mod stream_size;
pub mod transform_base64;
pub mod transforms;
pub mod uint;
pub mod uri;
pub mod tojson;

@ -0,0 +1,49 @@
/* Copyright (C) 2024 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
//! Module for transforms
use std::os::raw::{c_char, c_int, c_void};
pub mod strip_whitespace;
#[repr(C)]
#[allow(non_snake_case)]
pub struct SCTransformTableElmt {
pub name: *const c_char,
pub desc: *const c_char,
pub url: *const c_char,
pub flags: u16,
pub Setup: unsafe extern "C" fn(de: *mut c_void, s: *mut c_void, raw: *const c_char) -> c_int,
pub Free: Option<unsafe extern "C" fn(de: *mut c_void, ptr: *mut c_void)>,
pub Transform: unsafe extern "C" fn(inspect_buf: *mut c_void, options: *mut c_void),
pub TransformValidate:
Option<unsafe extern "C" fn(content: *const u8, len: u16, context: *mut c_void) -> bool>,
}
/// cbindgen:ignore
extern "C" {
pub fn DetectSignatureAddTransform(
s: *mut c_void, transform_id: c_int, ctx: *mut c_void,
) -> c_int;
pub fn InspectionBufferPtr(buf: *const c_void) -> *const u8;
pub fn InspectionBufferLength(buf: *const c_void) -> u32;
pub fn InspectionBufferCopy(ibuf: *const c_void, buf: *const u8, buf_len: u32);
pub fn DetectHelperTransformRegister(kw: *const SCTransformTableElmt) -> c_int;
pub fn InspectionBufferCheckAndExpand(ibuf: *const c_void, buf_len: u32) -> *mut u8;
pub fn InspectionBufferTruncate(ibuf: *const c_void, buf_len: u32);
}

@ -0,0 +1,136 @@
/* Copyright (C) 2024 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use super::{
DetectHelperTransformRegister, DetectSignatureAddTransform, InspectionBufferCheckAndExpand,
InspectionBufferLength, InspectionBufferPtr, InspectionBufferTruncate, SCTransformTableElmt,
};
use crate::detect::SIGMATCH_NOOPT;
use std::os::raw::{c_int, c_void};
use std::ptr;
static mut G_TRANSFORM_STRIP_WHITESPACE_ID: c_int = 0;
#[no_mangle]
unsafe extern "C" fn strip_whitespace_setup(
_de: *mut c_void, s: *mut c_void, _raw: *const std::os::raw::c_char,
) -> c_int {
return DetectSignatureAddTransform(s, G_TRANSFORM_STRIP_WHITESPACE_ID, ptr::null_mut());
}
fn strip_whitespace_transform_do(input: &[u8], output: &mut [u8]) -> u32 {
let mut nb = 0;
// seems faster than writing one byte at a time via
// for (i, o) in input.iter().filter(|c| !matches!(*c, b'\t' | b'\n' | b'\x0B' | b'\x0C' | b'\r' | b' ')).zip(output)
for subslice in input.split(|c| matches!(*c, b'\t' | b'\n' | b'\x0B' | b'\x0C' | b'\r' | b' '))
{
output[nb..nb + subslice.len()].copy_from_slice(subslice);
nb += subslice.len();
}
return nb as u32;
}
#[no_mangle]
unsafe extern "C" fn strip_whitespace_transform(buffer: *mut c_void, _ctx: *mut c_void) {
let input = InspectionBufferPtr(buffer);
let input_len = InspectionBufferLength(buffer);
if input.is_null() || input_len == 0 {
return;
}
let input = build_slice!(input, input_len as usize);
let output = InspectionBufferCheckAndExpand(buffer, input_len);
if output.is_null() {
// allocation failure
return;
}
let output = std::slice::from_raw_parts_mut(output, input_len as usize);
let output_len = strip_whitespace_transform_do(input, output);
InspectionBufferTruncate(buffer, output_len);
}
#[no_mangle]
unsafe extern "C" fn strip_whitespace_validate(
content: *const u8, len: u16, _ctx: *mut c_void,
) -> bool {
let input = build_slice!(content, len as usize);
for &c in input {
if matches!(c, b'\t' | b'\n' | b'\x0B' | b'\x0C' | b'\r' | b' ') {
return false;
}
}
return true;
}
#[no_mangle]
pub unsafe extern "C" fn DetectTransformStripWhitespaceRegister() {
let kw = SCTransformTableElmt {
name: b"strip_whitespace\0".as_ptr() as *const libc::c_char,
desc: b"modify buffer to strip whitespace before inspection\0".as_ptr()
as *const libc::c_char,
url: b"/rules/transforms.html#strip-whitespace\0".as_ptr() as *const libc::c_char,
Setup: strip_whitespace_setup,
flags: SIGMATCH_NOOPT,
Transform: strip_whitespace_transform,
Free: None,
TransformValidate: Some(strip_whitespace_validate),
};
unsafe {
G_TRANSFORM_STRIP_WHITESPACE_ID = DetectHelperTransformRegister(&kw);
if G_TRANSFORM_STRIP_WHITESPACE_ID < 0 {
SCLogWarning!("Failed registering transform strip_whitespace");
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_strip_whitespace_transform() {
let buf = b" A B C D ";
let mut out = vec![0; buf.len()];
let exp = b"ABCD";
assert_eq!(
strip_whitespace_transform_do(buf, &mut out),
exp.len() as u32
);
assert_eq!(&out[..exp.len()], exp);
let buf = b"EFGH";
let mut out = vec![0; buf.len()];
let exp = b"EFGH";
assert_eq!(
strip_whitespace_transform_do(buf, &mut out),
exp.len() as u32
);
assert_eq!(&out[..exp.len()], exp);
let buf = b"I \t J";
let mut out = vec![0; buf.len()];
let exp = b"IJ";
assert_eq!(
strip_whitespace_transform_do(buf, &mut out),
exp.len() as u32
);
assert_eq!(&out[..exp.len()], exp);
}
}

@ -315,7 +315,6 @@ noinst_HEADERS = \
detect-transform-sha1.h \
detect-transform-sha256.h \
detect-transform-strip-pseudo-headers.h \
detect-transform-strip-whitespace.h \
detect-transform-urldecode.h \
detect-transform-xor.h \
detect-ttl.h \
@ -884,7 +883,6 @@ libsuricata_c_a_SOURCES = \
detect-transform-sha1.c \
detect-transform-sha256.c \
detect-transform-strip-pseudo-headers.c \
detect-transform-strip-whitespace.c \
detect-transform-urldecode.c \
detect-transform-xor.c \
detect-ttl.c \

@ -121,6 +121,33 @@ int DetectHelperKeywordRegister(const SCSigTableElmt *kw)
return DETECT_TBLSIZE_IDX - 1;
}
int DetectHelperTransformRegister(const SCTransformTableElmt *kw)
{
if (DETECT_TBLSIZE_IDX >= DETECT_TBLSIZE) {
void *tmp = SCRealloc(
sigmatch_table, (DETECT_TBLSIZE + DETECT_TBLSIZE_STEP) * sizeof(SigTableElmt));
if (unlikely(tmp == NULL)) {
return -1;
}
sigmatch_table = tmp;
DETECT_TBLSIZE += DETECT_TBLSIZE_STEP;
}
sigmatch_table[DETECT_TBLSIZE_IDX].name = kw->name;
sigmatch_table[DETECT_TBLSIZE_IDX].desc = kw->desc;
sigmatch_table[DETECT_TBLSIZE_IDX].url = kw->url;
sigmatch_table[DETECT_TBLSIZE_IDX].flags = kw->flags;
sigmatch_table[DETECT_TBLSIZE_IDX].Transform =
(void (*)(InspectionBuffer * buffer, void *options)) kw->Transform;
sigmatch_table[DETECT_TBLSIZE_IDX].TransformValidate = (bool (*)(
const uint8_t *content, uint16_t content_len, void *context))kw->TransformValidate;
sigmatch_table[DETECT_TBLSIZE_IDX].Setup =
(int (*)(DetectEngineCtx * de, Signature * s, const char *raw)) kw->Setup;
sigmatch_table[DETECT_TBLSIZE_IDX].Free = (void (*)(DetectEngineCtx * de, void *ptr)) kw->Free;
DETECT_TBLSIZE_IDX++;
return DETECT_TBLSIZE_IDX - 1;
}
InspectionBuffer *DetectHelperGetMultiData(struct DetectEngineThreadCtx_ *det_ctx,
const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
const int list_id, uint32_t index, MultiGetTxBuffer GetBuf)
@ -144,3 +171,13 @@ InspectionBuffer *DetectHelperGetMultiData(struct DetectEngineThreadCtx_ *det_ct
buffer->flags = DETECT_CI_FLAGS_SINGLE;
return buffer;
}
const uint8_t *InspectionBufferPtr(InspectionBuffer *buf)
{
return buf->inspect;
}
uint32_t InspectionBufferLength(InspectionBuffer *buf)
{
return buf->inspect_len;
}

@ -46,4 +46,8 @@ InspectionBuffer *DetectHelperGetMultiData(struct DetectEngineThreadCtx_ *det_ct
const DetectEngineTransforms *transforms, Flow *f, const uint8_t flow_flags, void *txv,
const int list_id, uint32_t index, MultiGetTxBuffer GetBuf);
int DetectHelperTransformRegister(const SCTransformTableElmt *kw);
const uint8_t *InspectionBufferPtr(InspectionBuffer *buf);
uint32_t InspectionBufferLength(InspectionBuffer *buf);
#endif /* SURICATA_DETECT_ENGINE_HELPER_H */

@ -216,7 +216,6 @@
#include "detect-engine-content-inspection.h"
#include "detect-transform-compress-whitespace.h"
#include "detect-transform-strip-whitespace.h"
#include "detect-transform-strip-pseudo-headers.h"
#include "detect-transform-md5.h"
#include "detect-transform-sha1.h"

@ -1623,10 +1623,10 @@ void InspectionBufferFree(InspectionBuffer *buffer)
* \brief make sure that the buffer has at least 'min_size' bytes
* Expand the buffer if necessary
*/
void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
void *InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
{
if (likely(buffer->size >= min_size))
return;
return buffer->buf;
uint32_t new_size = (buffer->size == 0) ? 4096 : buffer->size;
while (new_size < min_size) {
@ -1637,7 +1637,19 @@ void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
if (ptr != NULL) {
buffer->buf = ptr;
buffer->size = new_size;
} else {
return NULL;
}
return buffer->buf;
}
void InspectionBufferTruncate(InspectionBuffer *buffer, uint32_t buf_len)
{
DEBUG_VALIDATE_BUG_ON(buffer->buf == NULL);
DEBUG_VALIDATE_BUG_ON(buf_len > buffer->size);
buffer->inspect = buffer->buf;
buffer->inspect_len = buf_len;
buffer->initialized = true;
}
void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len)

@ -31,7 +31,8 @@ void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size);
void InspectionBufferSetup(DetectEngineThreadCtx *det_ctx, const int list_id,
InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len);
void InspectionBufferFree(InspectionBuffer *buffer);
void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size);
void *InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size);
void InspectionBufferTruncate(InspectionBuffer *buffer, uint32_t buf_len);
void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len);
void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
const DetectEngineTransforms *transforms);

@ -1,209 +0,0 @@
/* Copyright (C) 2007-2020 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*
* Implements the nocase keyword
*/
#include "suricata-common.h"
#include "detect.h"
#include "detect-engine.h"
#include "detect-engine-prefilter.h"
#include "detect-engine-build.h"
#include "detect-parse.h"
#include "detect-transform-strip-whitespace.h"
#include "util-unittest.h"
#include "util-print.h"
static int DetectTransformStripWhitespaceSetup (DetectEngineCtx *, Signature *, const char *);
#ifdef UNITTESTS
static void DetectTransformStripWhitespaceRegisterTests(void);
#endif
static void TransformStripWhitespace(InspectionBuffer *buffer, void *options);
static bool TransformStripWhitespaceValidate(const uint8_t *content, uint16_t content_len, void *options);
void DetectTransformStripWhitespaceRegister(void)
{
sigmatch_table[DETECT_TRANSFORM_STRIP_WHITESPACE].name = "strip_whitespace";
sigmatch_table[DETECT_TRANSFORM_STRIP_WHITESPACE].desc =
"modify buffer to strip whitespace before inspection";
sigmatch_table[DETECT_TRANSFORM_STRIP_WHITESPACE].url =
"/rules/transforms.html#strip-whitespace";
sigmatch_table[DETECT_TRANSFORM_STRIP_WHITESPACE].Transform =
TransformStripWhitespace;
sigmatch_table[DETECT_TRANSFORM_STRIP_WHITESPACE].TransformValidate =
TransformStripWhitespaceValidate;
sigmatch_table[DETECT_TRANSFORM_STRIP_WHITESPACE].Setup =
DetectTransformStripWhitespaceSetup;
#ifdef UNITTESTS
sigmatch_table[DETECT_TRANSFORM_STRIP_WHITESPACE].RegisterTests =
DetectTransformStripWhitespaceRegisterTests;
#endif
sigmatch_table[DETECT_TRANSFORM_STRIP_WHITESPACE].flags |= SIGMATCH_NOOPT;
}
/**
* \internal
* \brief Apply the nocase keyword to the last pattern match, either content or uricontent
* \param det_ctx detection engine ctx
* \param s signature
* \param nullstr should be null
* \retval 0 ok
* \retval -1 failure
*/
static int DetectTransformStripWhitespaceSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr)
{
SCEnter();
int r = DetectSignatureAddTransform(s, DETECT_TRANSFORM_STRIP_WHITESPACE, NULL);
SCReturnInt(r);
}
/*
* \brief Validate content bytes to see if it's compatible with this transform
* \param content Byte array to check for compatibility
* \param content_len Number of bytes to check
* \param options Ignored
* \retval false If the string contains spaces
* \retval true Otherwise.
*/
static bool TransformStripWhitespaceValidate(const uint8_t *content,
uint16_t content_len, void *options)
{
if (content) {
for (uint32_t i = 0; i < content_len; i++) {
if (isspace(*content++)) {
return false;
}
}
}
return true;
}
static void TransformStripWhitespace(InspectionBuffer *buffer, void *options)
{
const uint8_t *input = buffer->inspect;
const uint32_t input_len = buffer->inspect_len;
if (input_len == 0) {
return;
}
uint8_t output[input_len]; // we can only shrink
uint8_t *oi = output, *os = output;
//PrintRawDataFp(stdout, input, input_len);
for (uint32_t i = 0; i < input_len; i++) {
if (!isspace(*input)) {
*oi++ = *input;
}
input++;
}
uint32_t output_size = oi - os;
//PrintRawDataFp(stdout, output, output_size);
InspectionBufferCopy(buffer, os, output_size);
}
#ifdef UNITTESTS
static int TransformDoubleWhitespace(InspectionBuffer *buffer)
{
const uint8_t *input = buffer->inspect;
const uint32_t input_len = buffer->inspect_len;
uint8_t output[input_len * 2]; // if all chars are whitespace this fits
uint8_t *oi = output, *os = output;
PrintRawDataFp(stdout, input, input_len);
for (uint32_t i = 0; i < input_len; i++) {
if (isspace(*input)) {
*oi++ = *input;
}
*oi++ = *input;
input++;
}
uint32_t output_size = oi - os;
PrintRawDataFp(stdout, output, output_size);
InspectionBufferCopy(buffer, os, output_size);
return 0;
}
static int DetectTransformStripWhitespaceTest01(void)
{
const uint8_t *input = (const uint8_t *)" A B C D ";
uint32_t input_len = strlen((char *)input);
InspectionBuffer buffer;
InspectionBufferInit(&buffer, 8);
InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
TransformStripWhitespace(&buffer, NULL);
PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
InspectionBufferFree(&buffer);
PASS;
}
static int DetectTransformStripWhitespaceTest02(void)
{
const uint8_t *input = (const uint8_t *)" A B C D ";
uint32_t input_len = strlen((char *)input);
InspectionBuffer buffer;
InspectionBufferInit(&buffer, 8);
InspectionBufferSetup(NULL, -1, &buffer, input, input_len);
PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
TransformDoubleWhitespace(&buffer);
PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
TransformDoubleWhitespace(&buffer);
PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
TransformStripWhitespace(&buffer, NULL);
PrintRawDataFp(stdout, buffer.inspect, buffer.inspect_len);
InspectionBufferFree(&buffer);
PASS;
}
static int DetectTransformStripWhitespaceTest03(void)
{
const char rule[] = "alert http any any -> any any (http_request_line; strip_whitespace; content:\"GET/HTTP\"; sid:1;)";
ThreadVars th_v;
DetectEngineThreadCtx *det_ctx = NULL;
memset(&th_v, 0, sizeof(th_v));
DetectEngineCtx *de_ctx = DetectEngineCtxInit();
FAIL_IF_NULL(de_ctx);
Signature *s = DetectEngineAppendSig(de_ctx, rule);
FAIL_IF_NULL(s);
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
DetectEngineCtxFree(de_ctx);
PASS;
}
static void DetectTransformStripWhitespaceRegisterTests(void)
{
UtRegisterTest("DetectTransformStripWhitespaceTest01",
DetectTransformStripWhitespaceTest01);
UtRegisterTest("DetectTransformStripWhitespaceTest02",
DetectTransformStripWhitespaceTest02);
UtRegisterTest("DetectTransformStripWhitespaceTest03",
DetectTransformStripWhitespaceTest03);
}
#endif

@ -1,30 +0,0 @@
/* Copyright (C) 2017 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*/
#ifndef SURICATA_DETECT_TRANSFORM_STRIPWHITESPACE_H
#define SURICATA_DETECT_TRANSFORM_STRIPWHITESPACE_H
/* prototypes */
void DetectTransformStripWhitespaceRegister (void);
#endif /* SURICATA_DETECT_TRANSFORM_STRIPWHITESPACE_H */
Loading…
Cancel
Save