detect: Disable unused SignatureHeader code

pull/1295/head
Victor Julien 11 years ago
parent 65af1f1c5e
commit ef6875d583

@ -71,7 +71,6 @@ detect-byte-extract.c detect-byte-extract.h \
detect-bytejump.c detect-bytejump.h \
detect-bytetest.c detect-bytetest.h \
detect.c detect.h \
detect-simd.c \
detect-classtype.c detect-classtype.h \
detect-content.c detect-content.h \
detect-csum.c detect-csum.h \

@ -185,11 +185,6 @@ void SigGroupHeadFree(SigGroupHead *sgh)
PatternMatchDestroyGroup(sgh);
if (sgh->head_array != NULL) {
SCFree(sgh->head_array);
sgh->head_array = NULL;
}
if (sgh->match_array != NULL) {
detect_siggroup_matcharray_free_cnt++;
detect_siggroup_matcharray_memory -= (sgh->sig_cnt * sizeof(Signature *));
@ -1742,42 +1737,6 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
return 0;
}
int SigGroupHeadBuildHeadArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
{
Signature *s = NULL;
uint32_t idx = 0;
uint32_t sig = 0;
if (sgh == NULL)
return 0;
BUG_ON(sgh->head_array != NULL);
sgh->head_array = SCMalloc(sgh->sig_cnt * sizeof(SignatureHeader));
if (sgh->head_array == NULL)
return -1;
memset(sgh->head_array, 0, sgh->sig_cnt * sizeof(SignatureHeader));
detect_siggroup_matcharray_init_cnt++;
detect_siggroup_matcharray_memory += (sgh->sig_cnt * sizeof(SignatureHeader *));
for (sig = 0; sig < sgh->sig_cnt; sig++) {
s = sgh->match_array[sig];
if (s == NULL)
continue;
sgh->head_array[idx].hdr_copy1 = s->hdr_copy1;
sgh->head_array[idx].hdr_copy2 = s->hdr_copy2;
sgh->head_array[idx].hdr_copy3 = s->hdr_copy3;
sgh->head_array[idx].full_sig = s;
idx++;
}
return 0;
}
/**
* \brief Check if a SigGroupHead contains a Signature, whose sid is sent as an
* argument.

@ -84,8 +84,6 @@ void SigGroupHeadRegisterTests(void);
void SigGroupHeadPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
void SigGroupHeadStore(DetectEngineCtx *, SigGroupHead *);
int SigGroupHeadBuildHeadArray(DetectEngineCtx *, SigGroupHead *);
void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *, SigGroupHead *);
void SigGroupHeadSetFilestoreCount(DetectEngineCtx *, SigGroupHead *);
void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *, SigGroupHead *);

@ -1,365 +0,0 @@
/* Copyright (C) 2013 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*
* Basic detection engine
*/
#include "suricata-common.h"
#include "detect.h"
#include "util-unittest.h"
#include "util-unittest-helper.h"
#include "util-vector.h"
/* Included into detect.c */
#ifdef UNITTESTS
#include "flow-util.h"
#include "stream-tcp-reassemble.h"
#include "util-var-name.h"
/// SCLogInfo("%s %u %u %u %u", #v, (v).dw[0], (v).dw[1], (v).dw[2], (v).dw[3]);
#define VECTOR_SCLogInfo(v) { \
SCLogInfo("%s %08X %08X %08X %08X", #v, (v).dw[0], (v).dw[1], (v).dw[2], (v).dw[3]); \
}
/**
* \test Test 32 bit SIMD code.
*/
static int SigTestSIMDMask01(void)
{
#if defined (__SSE3__)
Vector pm, sm, r1, r2;
uint32_t bm = 0;
uint8_t *mask = SCMallocAligned(32, 16);
memset(mask, 0xEF, 32);
mask[31] = 0xFF;
printf("\n");
pm.v = _mm_set1_epi8(0xEF);
VECTOR_SCLogInfo(pm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[0]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm = ((uint32_t) _mm_movemask_epi8(r2.v));
SCLogInfo("bm %08x", bm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[16]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm |= ((uint32_t) _mm_movemask_epi8(r2.v)) << 16;
SCLogInfo("bm %08x", bm);
int b = 0;
for ( ; b < 32; b++){
if (bm & (1 << b)) {
SCLogInfo("b %02d, set", b);
} else {
SCLogInfo("b %02d, not set", b);
}
}
if (!(bm & (1 << 31))) {
return 1;
}
return 0;
#else
return 1;
#endif
}
/**
* \test Test 32 bit SIMD code.
*/
static int SigTestSIMDMask02(void)
{
#if defined (__SSE3__)
Vector pm, sm, r1, r2;
uint32_t bm = 0;
uint8_t *mask = SCMallocAligned(32, 16);
memset(mask, 0x01, 32);
mask[31] = 0;
pm.v = _mm_set1_epi8(0x02);
VECTOR_SCLogInfo(pm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[0]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm = ((uint32_t) _mm_movemask_epi8(r2.v));
SCLogInfo("bm %08x", bm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[16]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm |= ((uint32_t) _mm_movemask_epi8(r2.v)) << 16;
SCLogInfo("bm %08x", bm);
int b = 0;
for ( ; b < 32; b++){
if (bm & (1 << b)) {
SCLogInfo("b %02d, set", b);
} else {
SCLogInfo("b %02d, not set", b);
}
}
if (bm & (1 << 31)) {
return 1;
}
return 0;
#else
return 1;
#endif
}
/**
* \test Test 64 bit SIMD code.
*/
static int SigTestSIMDMask03(void)
{
#if defined (__SSE3__)
Vector pm, sm, r1, r2;
uint64_t bm = 0;
uint8_t *mask = SCMallocAligned(64, 16);
memset(mask, 0xEF, 64);
mask[31] = 0xFF;
mask[62] = 0xFF;
printf("\n");
pm.v = _mm_set1_epi8(0xEF);
VECTOR_SCLogInfo(pm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[0]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm = ((uint64_t) _mm_movemask_epi8(r2.v));
SCLogInfo("bm1 %"PRIxMAX, (uintmax_t)bm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[16]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 16;
SCLogInfo("bm2 %"PRIxMAX, (uintmax_t)bm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[32]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 32;
SCLogInfo("bm3 %"PRIxMAX, (uintmax_t)bm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[48]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 48;
SCLogInfo("bm4 %"PRIxMAX, (uintmax_t)bm);
int b = 0;
for ( ; b < 64; b++){
if (bm & ((uint64_t)1 << b)) {
SCLogInfo("b %02d, set", b);
} else {
SCLogInfo("b %02d, not set", b);
}
}
if (!(bm & ((uint64_t)1 << 31)) && !(bm & ((uint64_t)1 << 62))) {
return 1;
}
return 0;
#else
return 1;
#endif
}
/**
* \test Test 64 bit SIMD code.
*/
static int SigTestSIMDMask04(void)
{
#if defined (__SSE3__)
Vector pm, sm, r1, r2;
uint64_t bm = 0;
uint8_t *mask = SCMallocAligned(64, 16);
memset(mask, 0x01, 64);
mask[31] = 0;
mask[62] = 0;
pm.v = _mm_set1_epi8(0x02);
VECTOR_SCLogInfo(pm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[0]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm = ((uint64_t) _mm_movemask_epi8(r2.v));
SCLogInfo("bm1 %"PRIxMAX, (uintmax_t)bm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[16]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 16;
SCLogInfo("bm2 %"PRIxMAX, (uintmax_t)bm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[32]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 32;
SCLogInfo("bm3 %"PRIxMAX, (uintmax_t)bm);
/* load a batch of masks */
sm.v = _mm_load_si128((const __m128i *)&mask[48]);
VECTOR_SCLogInfo(sm);
/* logical AND them with the packet's mask */
r1.v = _mm_and_si128(pm.v, sm.v);
VECTOR_SCLogInfo(r1);
/* compare the result with the original mask */
r2.v = _mm_cmpeq_epi8(sm.v, r1.v);
VECTOR_SCLogInfo(r2);
/* convert into a bitarray */
bm |= (((uint64_t) _mm_movemask_epi8(r2.v)) << 48);
SCLogInfo("bm4-total %"PRIxMAX, (uintmax_t)bm);
int b = 0;
for ( ; b < 64; b++){
if (bm & ((uint64_t)1 << b)) {
SCLogInfo("b %02d, set", b);
} else {
SCLogInfo("b %02d, not set", b);
}
}
if ((bm & ((uint64_t)1 << 31)) && (bm & ((uint64_t)1 << 62))) {
return 1;
}
return 0;
#else
return 1;
#endif
}
#endif /* UNITTESTS */
void DetectSimdRegisterTests(void)
{
#ifdef UNITTESTS
UtRegisterTest("SigTestSIMDMask01", SigTestSIMDMask01, 1);
UtRegisterTest("SigTestSIMDMask02", SigTestSIMDMask02, 1);
UtRegisterTest("SigTestSIMDMask03", SigTestSIMDMask03, 1);
UtRegisterTest("SigTestSIMDMask04", SigTestSIMDMask04, 1);
#endif /* UNITTESTS */
}

@ -483,122 +483,6 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, int sig_file_excl
SCReturnInt(ret);
}
/**
* \brief See if we can prefilter a signature on inexpensive checks
*
* Order of SignatureHeader access:
* 1. flags
* 2. alproto
* 3. mpm_pattern_id_div8
* 4. mpm_pattern_id_mod8
* 5. num
*
* \retval 0 can't match, don't inspect
* \retval 1 might match, further inspection required
*/
int SigMatchSignaturesBuildMatchArrayAddSignature(DetectEngineThreadCtx *det_ctx,
Packet *p, SignatureHeader *s,
AppProto alproto)
{
/* if the sig has alproto and the session as well they should match */
if (likely(s->flags & SIG_FLAG_APPLAYER)) {
if (s->alproto != ALPROTO_UNKNOWN && s->alproto != alproto) {
if (s->alproto == ALPROTO_DCERPC) {
if (alproto != ALPROTO_SMB && alproto != ALPROTO_SMB2) {
SCLogDebug("DCERPC sig, alproto not SMB or SMB2");
return 0;
}
} else {
SCLogDebug("alproto mismatch");
return 0;
}
}
}
if (unlikely(s->flags & SIG_FLAG_DSIZE)) {
if (likely(p->payload_len < s->dsize_low || p->payload_len > s->dsize_high)) {
SCLogDebug("kicked out as p->payload_len %u, dsize low %u, hi %u",
p->payload_len, s->dsize_low, s->dsize_high);
return 0;
}
}
/* check for a pattern match of the one pattern in this sig. */
if (likely(s->flags & (SIG_FLAG_MPM_PACKET|SIG_FLAG_MPM_STREAM|SIG_FLAG_MPM_APPLAYER)))
{
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id_div_8)] & s->mpm_pattern_id_mod_8)) {
if (s->flags & SIG_FLAG_MPM_PACKET) {
if (!(s->flags & SIG_FLAG_MPM_PACKET_NEG)) {
return 0;
}
} else if (s->flags & SIG_FLAG_MPM_STREAM) {
/* filter out sigs that want pattern matches, but
* have no matches */
if (!(s->flags & SIG_FLAG_MPM_STREAM_NEG)) {
return 0;
}
} else if (s->flags & SIG_FLAG_MPM_APPLAYER) {
if (!(s->flags & SIG_FLAG_MPM_APPLAYER_NEG)) {
return 0;
}
}
}
}
/* de_state check, filter out all signatures that already had a match before
* or just partially match */
if (s->flags & SIG_FLAG_STATE_MATCH) {
/* we run after DeStateDetectContinueDetection, so we might have
* state NEW here. In that case we'd want to continue detection
* for this sig. If we have NOSTATE, stateful detection didn't
* start yet for this sig, so we will inspect it.
*/
if (det_ctx->de_state_sig_array[s->num] == DE_STATE_MATCH_NO_NEW_STATE)
return 0;
}
return 1;
}
#if defined(__SSE3__) || defined(__tile__)
/* SIMD implementations are in detect-simd.c */
#else
/* Non-SIMD implementation */
/**
* \brief build an array of signatures that will be inspected
*
* All signatures that can be filtered out on forehand are not added to it.
*
* \param de_ctx detection engine ctx
* \param det_ctx detection engine thread ctx -- array is stored here
* \param p packet
* \param mask Packets mask
* \param alproto application layer protocol
*/
void SigMatchSignaturesBuildMatchArray(DetectEngineThreadCtx *det_ctx,
Packet *p, SignatureMask mask,
AppProto alproto)
{
uint32_t u;
/* reset previous run */
det_ctx->match_array_cnt = 0;
for (u = 0; u < det_ctx->sgh->sig_cnt; u++) {
SignatureHeader *s = &det_ctx->sgh->head_array[u];
if ((mask & s->mask) == s->mask) {
if (SigMatchSignaturesBuildMatchArrayAddSignature(det_ctx, p, s, alproto) == 1) {
/* okay, store it */
det_ctx->match_array[det_ctx->match_array_cnt] = s->full_sig;
det_ctx->match_array_cnt++;
}
}
}
}
#endif /* No SIMD implementation */
int SigMatchSignaturesRunPostMatch(ThreadVars *tv,
DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p,
Signature *s)
@ -4192,8 +4076,6 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
SigGroupHead *sgh = de_ctx->sgh_array[idx];
if (sgh == NULL)
continue;
SigGroupHeadBuildHeadArray(de_ctx, sgh);
SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
SigGroupHeadSetFileMd5Flag(de_ctx, sgh);
SigGroupHeadSetFilesizeFlag(de_ctx, sgh);
@ -4204,7 +4086,6 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
}
if (de_ctx->decoder_event_sgh != NULL) {
SigGroupHeadBuildHeadArray(de_ctx, de_ctx->decoder_event_sgh);
/* no need to set filestore count here as that would make a
* signature not decode event only. */
}
@ -12274,7 +12155,9 @@ void SigRegisterTests(void)
UtRegisterTest("SigTestPorts01", SigTestPorts01, 1);
UtRegisterTest("SigTestBug01", SigTestBug01, 1);
#if 0
DetectSimdRegisterTests();
#endif
#endif /* UNITTESTS */
}

@ -333,37 +333,6 @@ typedef struct IPOnlyCIDRItem_ {
} IPOnlyCIDRItem;
/** \brief Subset of the Signature for cache efficient prefiltering
*/
typedef struct SignatureHeader_ {
union {
struct {
/* coccinelle: SignatureHeader:flags:SIG_FLAG */
uint32_t flags;
AppProto alproto;
uint16_t dsize_low;
};
uint64_t hdr_copy1;
};
union {
struct {
uint16_t dsize_high;
uint16_t mpm_pattern_id_div_8;
};
uint32_t hdr_copy2;
};
union {
struct {
uint8_t mpm_pattern_id_mod_8;
SignatureMask mask;
SigIntId num; /**< signature number, internal id */
};
uint32_t hdr_copy3;
};
/** pointer to the full signature */
struct Signature_ *full_sig;
} SignatureHeader;
/** \brief Used to start a pointer to SigMatch context
* Should never be dereferenced without casting to something else.
*/
@ -988,10 +957,6 @@ typedef struct SigGroupHead_ {
#if defined(__SSE3__) || defined(__tile__)
SignatureMask *mask_array;
#endif
/** chunk of memory containing the "header" part of each
* signature ordered as an array. Used to pre-filter the
* signatures to be inspected in a cache efficient way. */
SignatureHeader *head_array;
SigIntId *non_mpm_id_array;
uint32_t non_mpm_id_cnt; // size is cnt * sizeof(uint32_t)
@ -1184,9 +1149,6 @@ Signature *SigFindSignatureBySidGid(DetectEngineCtx *, uint32_t, uint32_t);
void SigMatchSignaturesBuildMatchArray(DetectEngineThreadCtx *,
Packet *, SignatureMask,
uint16_t);
int SigMatchSignaturesBuildMatchArrayAddSignature(DetectEngineThreadCtx *,
Packet *, SignatureHeader *,
uint16_t);
void SigMatchFree(SigMatch *sm);
void SigCleanSignatures(DetectEngineCtx *);

Loading…
Cancel
Save