aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Clean up new SIMD mask checking code, improve non-SIMD checks.

Browse Source
remotes/origin/master-1.1.x
Victor Julien 14 years ago
parent b421019cef
commit 2dbfdd40af
3 changed files with 218 additions and 179 deletions
Show Stats Download Patch File Download Diff File

24
src/detect-engine-siggroup.c
Unescape Escape View File

@ -175,11 +175,13 @@ void SigGroupHeadFree(SigGroupHead *sgh)
PatternMatchDestroyGroup(sgh);
#if defined(__SSE3__)
if (sgh->mask_array != NULL) {
/* mask is aligned */
SCFreeAligned(sgh->mask_array);
sgh->mask_array = NULL;
}
#endif
if (sgh->head_array != NULL) {
SCFree(sgh->head_array);
@ -1593,17 +1595,9 @@ int SigGroupHeadBuildHeadArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
return 0;
BUG_ON(sgh->head_array != NULL);
#if defined(__SSE3__)
BUG_ON(sgh->mask_array != NULL);
sgh->head_array = SCMalloc(sgh->sig_cnt * sizeof(SignatureHeader));
if (sgh->head_array == NULL)
return -1;
memset(sgh->head_array, 0, sgh->sig_cnt * sizeof(SignatureHeader));
detect_siggroup_matcharray_init_cnt++;
detect_siggroup_matcharray_memory += (sgh->sig_cnt * sizeof(SignatureHeader *));
/* mask array is 16 byte aligned for SIMD checking, also we always
* alloc a multiple of 16 bytes */
int cnt = sgh->sig_cnt;
@ -1616,6 +1610,16 @@ int SigGroupHeadBuildHeadArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
return -1;
memset(sgh->mask_array, 0, (cnt * sizeof(SignatureMask)));
#endif
sgh->head_array = SCMalloc(sgh->sig_cnt * sizeof(SignatureHeader));
if (sgh->head_array == NULL)
return -1;
memset(sgh->head_array, 0, sgh->sig_cnt * sizeof(SignatureHeader));
detect_siggroup_matcharray_init_cnt++;
detect_siggroup_matcharray_memory += (sgh->sig_cnt * sizeof(SignatureHeader *));
for (sig = 0; sig < sgh->sig_cnt; sig++) {
s = sgh->match_array[sig];
@ -1627,7 +1631,9 @@ int SigGroupHeadBuildHeadArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
sgh->head_array[idx].hdr_copy3 = s->hdr_copy3;
sgh->head_array[idx].full_sig = s;
#if defined(__SSE3__)
sgh->mask_array[idx] = s->mask;
#endif
idx++;
}

364
src/detect.c
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
/* Copyright (C) 2007-2011 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -161,6 +161,7 @@
#include "util-profiling.h"
#include "util-validate.h"
#include "util-optimize.h"
#include "util-vector.h"
extern uint8_t engine_mode;
@ -672,18 +673,8 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file)
SCReturnInt(ret);
}
#include "util-vector.h"
/**
* \brief build an array of signatures that will be inspected
*
* All signatures that can be filtered out on forehand are not added to it.
*
* \param de_ctx detection engine ctx
* \param det_ctx detection engine thread ctx -- array is stored here
* \param p packet
* \param mask Packets mask
* \param alproto application layer protocol
* \brief See if we can prefilter a signature on inexpensive checks
*
* Order of SignatureHeader access:
* 1. mask
@ -694,27 +685,169 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file)
* 5. mpm_stream_pattern_id_div8
* 5. mpm_stream_pattern_id_mod8
* 6. num
*
* \retval 0 can't match, don't inspect
* \retval 1 might match, further inspection required
*/
static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx,
DetectEngineThreadCtx *det_ctx, Packet *p, SignatureMask mask,
uint16_t alproto)
static inline int SigMatchSignaturesBuildMatchArrayAddSignature(DetectEngineThreadCtx *det_ctx,
Packet *p, SignatureHeader *s, uint16_t alproto)
{
/* if the sig has alproto and the session as well they should match */
if (s->flags & SIG_FLAG_APPLAYER && s->alproto != ALPROTO_UNKNOWN && s->alproto != alproto) {
if (s->alproto == ALPROTO_DCERPC) {
if (alproto != ALPROTO_SMB && alproto != ALPROTO_SMB2) {
SCLogDebug("DCERPC sig, alproto not SMB or SMB2");
return 0;
}
} else {
SCLogDebug("alproto mismatch");
return 0;
}
}
/* check for a pattern match of the one pattern in this sig. */
if (s->flags & SIG_FLAG_MPM_PACKET) {
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id_div_8)] & s->mpm_pattern_id_mod_8)) {
//if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id / 8)] & (1<<(s->mpm_pattern_id % 8)))) {
//SCLogDebug("mpm sig without matches (pat id %"PRIu32" check in content).", s->mpm_pattern_id);
if (!(s->flags & SIG_FLAG_MPM_PACKET_NEG)) {
/* pattern didn't match. There is one case where we will inspect
* the signature anyway: if the packet payload was added to the
* stream it is not scanned itself: the stream data is inspected.
* Inspecting both would result in duplicated alerts. There is
* one case where we are going to inspect the packet payload
* anyway: if a signature has the dsize option. */
if (!((p->flags & PKT_STREAM_ADD) && (s->flags & SIG_FLAG_DSIZE))) {
return 0;
}
} else {
SCLogDebug("but thats okay, we are looking for neg-content");
}
}
}
if (s->flags & SIG_FLAG_MPM_STREAM) {
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_stream_pattern_id_div_8)] & s->mpm_stream_pattern_id_mod_8)) {
//SCLogDebug("mpm stream sig without matches (pat id %"PRIu32" check in content).", s->mpm_stream_pattern_id);
if (!(s->flags & SIG_FLAG_MPM_STREAM_NEG)) {
/* pattern didn't match. There is one case where we will inspect
* the signature anyway: if the packet payload was added to the
* stream it is not scanned itself: the stream data is inspected.
* Inspecting both would result in duplicated alerts. There is
* one case where we are going to inspect the packet payload
* anyway: if a signature has the dsize option. */
if (!((p->flags & PKT_STREAM_ADD) && (s->flags & SIG_FLAG_DSIZE))) {
return 0;
}
} else {
SCLogDebug("but thats okay, we are looking for neg-content");
}
}
}
if (s->full_sig->flags & SIG_FLAG_MPM_URICONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->full_sig->flags & SIG_FLAG_MPM_URICONTENT_NEG)) {
return 0;
}
}
}
if (s->flags & SIG_FLAG_MPM_HCBDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HCBDCONTENT_NEG)) {
return 0;
}
}
}
if (s->flags & SIG_FLAG_MPM_HHDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HHDCONTENT_NEG)) {
return 0;
}
}
}
if (s->flags & SIG_FLAG_MPM_HRHDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HRHDCONTENT_NEG)) {
return 0;
}
}
}
if (s->flags & SIG_FLAG_MPM_HMDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HMDCONTENT_NEG)) {
return 0;
}
}
}
if (s->flags & SIG_FLAG_MPM_HCDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HCDCONTENT_NEG)) {
return 0;
}
}
}
if (s->flags & SIG_FLAG_MPM_HRUDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HRUDCONTENT_NEG)) {
return 0;
}
}
}
/* de_state check, filter out all signatures that already had a match before
* or just partially match */
if (s->flags & SIG_FLAG_STATE_MATCH) {
/* we run after DeStateDetectContinueDetection, so we might have
* state NEW here. In that case we'd want to continue detection
* for this sig. If we have NOSTATE, stateful detection didn't
* start yet for this sig, so we will inspect it.
*/
if (det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NEW &&
det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NOSTATE) {
SCLogDebug("de state not NEW or NOSTATE, ignoring");
return 0;
}
}
return 1;
}
#if defined(__SSE3__)
static inline void SigMatchSignaturesBuildMatchArraySIMD(DetectEngineThreadCtx *det_ctx,
Packet *p, SignatureMask mask, uint16_t alproto)
{
uint32_t u;
uint32_t bm; /* bit mask, 16 bits used */
SigIntId x;
int bitno = 0;
#if defined(__SSE3__)
Vector pm, sm, r1, r2;
/* load the packet mask into each byte of the vector */
pm.v = _mm_set1_epi8(mask);
#endif
/* reset previous run */
det_ctx->match_array_cnt = 0;
for (u = 0; u < det_ctx->sgh->sig_cnt; u += 16) {
SigIntId x;
int bitno = 0;
#if defined(__SSE3__)
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&det_ctx->sgh->mask_array[u]);
/* logical AND them with the packet's mask */
@ -723,14 +856,7 @@ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx,
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
/* convert into a bitarray */
bm = _mm_movemask_epi8(r2.v);
#else
bm = 0;
for (x = u; x < det_ctx->sgh->sig_cnt && bitno < 16; x++, bitno++) {
int r = ((mask & det_ctx->sgh->mask_array[x]) == det_ctx->sgh->mask_array[x]);
bm |= (r << bitno);
}
SCLogDebug("bm %04X", bm);
#endif
if (bm == 0) {
continue;
}
@ -742,149 +868,57 @@ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx,
}
SignatureHeader *s = &det_ctx->sgh->head_array[x];
/* if the sig has alproto and the session as well they should match */
if (s->flags & SIG_FLAG_APPLAYER && s->alproto != ALPROTO_UNKNOWN && s->alproto != alproto) {
if (s->alproto == ALPROTO_DCERPC) {
if (alproto != ALPROTO_SMB && alproto != ALPROTO_SMB2) {
SCLogDebug("DCERPC sig, alproto not SMB or SMB2");
continue;
}
} else {
SCLogDebug("alproto mismatch");
continue;
}
if (SigMatchSignaturesBuildMatchArrayAddSignature(det_ctx, p, s, alproto) == 1) {
/* okay, store it */
det_ctx->match_array[det_ctx->match_array_cnt] = s->full_sig;
det_ctx->match_array_cnt++;
}
}
}
}
#endif /* defined(__SSE3__) */
/* check for a pattern match of the one pattern in this sig. */
if (s->flags & SIG_FLAG_MPM_PACKET) {
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id_div_8)] & s->mpm_pattern_id_mod_8)) {
//if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id / 8)] & (1<<(s->mpm_pattern_id % 8)))) {
//SCLogDebug("mpm sig without matches (pat id %"PRIu32" check in content).", s->mpm_pattern_id);
if (!(s->flags & SIG_FLAG_MPM_PACKET_NEG)) {
/* pattern didn't match. There is one case where we will inspect
* the signature anyway: if the packet payload was added to the
* stream it is not scanned itself: the stream data is inspected.
* Inspecting both would result in duplicated alerts. There is
* one case where we are going to inspect the packet payload
* anyway: if a signature has the dsize option. */
if (!((p->flags & PKT_STREAM_ADD) && (s->flags & SIG_FLAG_DSIZE))) {
continue;
}
} else {
SCLogDebug("but thats okay, we are looking for neg-content");
}
}
}
if (s->flags & SIG_FLAG_MPM_STREAM) {
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_stream_pattern_id_div_8)] & s->mpm_stream_pattern_id_mod_8)) {
//SCLogDebug("mpm stream sig without matches (pat id %"PRIu32" check in content).", s->mpm_stream_pattern_id);
if (!(s->flags & SIG_FLAG_MPM_STREAM_NEG)) {
/* pattern didn't match. There is one case where we will inspect
* the signature anyway: if the packet payload was added to the
* stream it is not scanned itself: the stream data is inspected.
* Inspecting both would result in duplicated alerts. There is
* one case where we are going to inspect the packet payload
* anyway: if a signature has the dsize option. */
if (!((p->flags & PKT_STREAM_ADD) && (s->flags & SIG_FLAG_DSIZE))) {
continue;
}
} else {
SCLogDebug("but thats okay, we are looking for neg-content");
}
}
}
if (s->full_sig->flags & SIG_FLAG_MPM_URICONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->full_sig->flags & SIG_FLAG_MPM_URICONTENT_NEG)) {
continue;
}
}
}
if (s->flags & SIG_FLAG_MPM_HCBDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HCBDCONTENT_NEG)) {
continue;
}
}
}
if (s->flags & SIG_FLAG_MPM_HHDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HHDCONTENT_NEG)) {
continue;
}
}
}
if (s->flags & SIG_FLAG_MPM_HRHDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HRHDCONTENT_NEG)) {
continue;
}
}
}
if (s->flags & SIG_FLAG_MPM_HMDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HMDCONTENT_NEG)) {
continue;
}
}
}
if (s->flags & SIG_FLAG_MPM_HCDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HCDCONTENT_NEG)) {
continue;
}
}
}
if (s->flags & SIG_FLAG_MPM_HRUDCONTENT) {
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_http_pattern_id / 8)] &
(1 << (s->mpm_http_pattern_id % 8)))) {
if (!(s->flags & SIG_FLAG_MPM_HRUDCONTENT_NEG)) {
continue;
}
}
}
static inline void SigMatchSignaturesBuildMatchArrayNoSIMD(DetectEngineThreadCtx *det_ctx,
Packet *p, SignatureMask mask, uint16_t alproto)
{
uint32_t u;
/* de_state check, filter out all signatures that already had a match before
* or just partially match */
if (s->flags & SIG_FLAG_STATE_MATCH) {
/* we run after DeStateDetectContinueDetection, so we might have
* state NEW here. In that case we'd want to continue detection
* for this sig. If we have NOSTATE, stateful detection didn't
* start yet for this sig, so we will inspect it.
*/
if (det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NEW &&
det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NOSTATE) {
SCLogDebug("de state not NEW or NOSTATE, ignoring");
continue;
}
}
/* reset previous run */
det_ctx->match_array_cnt = 0;
for (u = 0; u < det_ctx->sgh->sig_cnt; u++) {
SignatureHeader *s = &det_ctx->sgh->head_array[u];
if ((mask & s->mask) == s->mask) {
if (SigMatchSignaturesBuildMatchArrayAddSignature(det_ctx, p, s, alproto) == 1) {
/* okay, store it */
det_ctx->match_array[det_ctx->match_array_cnt] = s->full_sig;
det_ctx->match_array_cnt++;
}
}
}
}
/**
* \brief build an array of signatures that will be inspected
*
* All signatures that can be filtered out on forehand are not added to it.
*
* \param de_ctx detection engine ctx
* \param det_ctx detection engine thread ctx -- array is stored here
* \param p packet
* \param mask Packets mask
* \param alproto application layer protocol
*/
static void SigMatchSignaturesBuildMatchArray(DetectEngineThreadCtx *det_ctx,
Packet *p, SignatureMask mask, uint16_t alproto)
{
#if defined(__SSE3__)
SigMatchSignaturesBuildMatchArraySIMD(det_ctx, p, mask, alproto);
#else
SigMatchSignaturesBuildMatchArrayNoSIMD(det_ctx, p, mask, alproto);
#endif
}
/**
* \brief Get the SigGroupHead for a packet.
*
@ -1249,10 +1283,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, p);
}
/* create our prefilter mask */
SignatureMask mask = 0;
PacketCreateMask(p, &mask, alproto, alstate, smsg);
/* if we didn't get a sig group head, we
* have nothing to do.... */
if (det_ctx->sgh == NULL) {
@ -1276,8 +1306,12 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
}
}
/* create our prefilter mask */
SignatureMask mask = 0;
PacketCreateMask(p, &mask, alproto, alstate, smsg);
/* build the match array */
SigMatchSignaturesBuildMatchArray(de_ctx, det_ctx, p, mask, alproto);
SigMatchSignaturesBuildMatchArray(det_ctx, p, mask, alproto);
/* inspect the sigs against the packet */
for (idx = 0; idx < det_ctx->match_array_cnt; idx++) {

9
src/detect.h
Unescape Escape View File

@ -871,17 +871,16 @@ typedef struct SigGroupHead_ {
uint16_t mpm_content_maxlen;
/** array of masks, used to check multiple masks against
* a packet using SIMD. */
#if defined(__SSE3__)
SignatureMask *mask_array;
#endif
/** chunk of memory containing the "header" part of each
* signature ordered as an array. Used to pre-filter the
* signatures to be inspected in a cache efficient way. */
SignatureHeader *head_array;
/** chunk of memory containing the "header" part of each
* signature ordered as an array. Used to pre-filter the
* signatures to be inspected in a cache efficient way. */
//SignatureHeader *head_array;
/* pattern matcher instances */
MpmCtx *mpm_ctx;
MpmCtx *mpm_stream_ctx;

Write Preview
Loading…
Cancel
Save