diff --git a/src/Makefile.am b/src/Makefile.am index cc1357e0f8..d60f56c4b9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -71,7 +71,6 @@ detect-byte-extract.c detect-byte-extract.h \ detect-bytejump.c detect-bytejump.h \ detect-bytetest.c detect-bytetest.h \ detect.c detect.h \ -detect-simd.c \ detect-classtype.c detect-classtype.h \ detect-content.c detect-content.h \ detect-csum.c detect-csum.h \ diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index ff7fcb2d4a..3b8a9ed3bf 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -185,11 +185,6 @@ void SigGroupHeadFree(SigGroupHead *sgh) PatternMatchDestroyGroup(sgh); - if (sgh->head_array != NULL) { - SCFree(sgh->head_array); - sgh->head_array = NULL; - } - if (sgh->match_array != NULL) { detect_siggroup_matcharray_free_cnt++; detect_siggroup_matcharray_memory -= (sgh->sig_cnt * sizeof(Signature *)); @@ -1742,42 +1737,6 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh) return 0; } -int SigGroupHeadBuildHeadArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh) -{ - Signature *s = NULL; - uint32_t idx = 0; - uint32_t sig = 0; - - if (sgh == NULL) - return 0; - - BUG_ON(sgh->head_array != NULL); - - sgh->head_array = SCMalloc(sgh->sig_cnt * sizeof(SignatureHeader)); - if (sgh->head_array == NULL) - return -1; - - memset(sgh->head_array, 0, sgh->sig_cnt * sizeof(SignatureHeader)); - - detect_siggroup_matcharray_init_cnt++; - detect_siggroup_matcharray_memory += (sgh->sig_cnt * sizeof(SignatureHeader *)); - - for (sig = 0; sig < sgh->sig_cnt; sig++) { - s = sgh->match_array[sig]; - if (s == NULL) - continue; - - sgh->head_array[idx].hdr_copy1 = s->hdr_copy1; - sgh->head_array[idx].hdr_copy2 = s->hdr_copy2; - sgh->head_array[idx].hdr_copy3 = s->hdr_copy3; - sgh->head_array[idx].full_sig = s; - - idx++; - } - - return 0; -} - /** * \brief Check if a SigGroupHead contains a Signature, whose sid is sent as an * argument. diff --git a/src/detect-engine-siggroup.h b/src/detect-engine-siggroup.h index a9368ee19f..829b0cefb3 100644 --- a/src/detect-engine-siggroup.h +++ b/src/detect-engine-siggroup.h @@ -84,8 +84,6 @@ void SigGroupHeadRegisterTests(void); void SigGroupHeadPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh); void SigGroupHeadStore(DetectEngineCtx *, SigGroupHead *); - -int SigGroupHeadBuildHeadArray(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFilemagicFlag(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFilestoreCount(DetectEngineCtx *, SigGroupHead *); void SigGroupHeadSetFileMd5Flag(DetectEngineCtx *, SigGroupHead *); diff --git a/src/detect-simd.c b/src/detect-simd.c deleted file mode 100644 index 114ee589a1..0000000000 --- a/src/detect-simd.c +++ /dev/null @@ -1,365 +0,0 @@ -/* Copyright (C) 2013 Open Information Security Foundation - * - * You can copy, redistribute or modify this Program under the terms of - * the GNU General Public License version 2 as published by the Free - * Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * version 2 along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - */ - -/** - * \file - * - * \author Victor Julien - * - * Basic detection engine - */ - -#include "suricata-common.h" -#include "detect.h" - -#include "util-unittest.h" -#include "util-unittest-helper.h" -#include "util-vector.h" - -/* Included into detect.c */ - - - -#ifdef UNITTESTS -#include "flow-util.h" -#include "stream-tcp-reassemble.h" -#include "util-var-name.h" - -/// SCLogInfo("%s %u %u %u %u", #v, (v).dw[0], (v).dw[1], (v).dw[2], (v).dw[3]); -#define VECTOR_SCLogInfo(v) { \ - SCLogInfo("%s %08X %08X %08X %08X", #v, (v).dw[0], (v).dw[1], (v).dw[2], (v).dw[3]); \ -} - -/** - * \test Test 32 bit SIMD code. - */ -static int SigTestSIMDMask01(void) -{ -#if defined (__SSE3__) - Vector pm, sm, r1, r2; - uint32_t bm = 0; - - uint8_t *mask = SCMallocAligned(32, 16); - memset(mask, 0xEF, 32); - mask[31] = 0xFF; - printf("\n"); - pm.v = _mm_set1_epi8(0xEF); - VECTOR_SCLogInfo(pm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[0]); - VECTOR_SCLogInfo(sm); - - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm = ((uint32_t) _mm_movemask_epi8(r2.v)); - - SCLogInfo("bm %08x", bm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[16]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm |= ((uint32_t) _mm_movemask_epi8(r2.v)) << 16; - - SCLogInfo("bm %08x", bm); - - int b = 0; - for ( ; b < 32; b++){ - if (bm & (1 << b)) { - SCLogInfo("b %02d, set", b); - } else { - SCLogInfo("b %02d, not set", b); - - } - } - - if (!(bm & (1 << 31))) { - return 1; - } - return 0; -#else - return 1; -#endif -} - -/** - * \test Test 32 bit SIMD code. - */ -static int SigTestSIMDMask02(void) -{ -#if defined (__SSE3__) - Vector pm, sm, r1, r2; - uint32_t bm = 0; - - uint8_t *mask = SCMallocAligned(32, 16); - memset(mask, 0x01, 32); - mask[31] = 0; - pm.v = _mm_set1_epi8(0x02); - VECTOR_SCLogInfo(pm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[0]); - VECTOR_SCLogInfo(sm); - - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm = ((uint32_t) _mm_movemask_epi8(r2.v)); - - SCLogInfo("bm %08x", bm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[16]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm |= ((uint32_t) _mm_movemask_epi8(r2.v)) << 16; - - SCLogInfo("bm %08x", bm); - - int b = 0; - for ( ; b < 32; b++){ - if (bm & (1 << b)) { - SCLogInfo("b %02d, set", b); - } else { - SCLogInfo("b %02d, not set", b); - - } - } - - if (bm & (1 << 31)) { - return 1; - } - return 0; -#else - return 1; -#endif -} - -/** - * \test Test 64 bit SIMD code. - */ -static int SigTestSIMDMask03(void) -{ -#if defined (__SSE3__) - Vector pm, sm, r1, r2; - uint64_t bm = 0; - uint8_t *mask = SCMallocAligned(64, 16); - memset(mask, 0xEF, 64); - mask[31] = 0xFF; - mask[62] = 0xFF; - printf("\n"); - pm.v = _mm_set1_epi8(0xEF); - VECTOR_SCLogInfo(pm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[0]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm = ((uint64_t) _mm_movemask_epi8(r2.v)); - - SCLogInfo("bm1 %"PRIxMAX, (uintmax_t)bm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[16]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 16; - - SCLogInfo("bm2 %"PRIxMAX, (uintmax_t)bm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[32]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 32; - - SCLogInfo("bm3 %"PRIxMAX, (uintmax_t)bm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[48]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 48; - - SCLogInfo("bm4 %"PRIxMAX, (uintmax_t)bm); - - int b = 0; - for ( ; b < 64; b++){ - if (bm & ((uint64_t)1 << b)) { - SCLogInfo("b %02d, set", b); - } else { - SCLogInfo("b %02d, not set", b); - - } - } - - if (!(bm & ((uint64_t)1 << 31)) && !(bm & ((uint64_t)1 << 62))) { - return 1; - } - return 0; -#else - return 1; -#endif -} - -/** - * \test Test 64 bit SIMD code. - */ -static int SigTestSIMDMask04(void) -{ -#if defined (__SSE3__) - Vector pm, sm, r1, r2; - uint64_t bm = 0; - - uint8_t *mask = SCMallocAligned(64, 16); - memset(mask, 0x01, 64); - mask[31] = 0; - mask[62] = 0; - pm.v = _mm_set1_epi8(0x02); - VECTOR_SCLogInfo(pm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[0]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm = ((uint64_t) _mm_movemask_epi8(r2.v)); - - SCLogInfo("bm1 %"PRIxMAX, (uintmax_t)bm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[16]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 16; - - SCLogInfo("bm2 %"PRIxMAX, (uintmax_t)bm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[32]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm |= ((uint64_t) _mm_movemask_epi8(r2.v)) << 32; - - SCLogInfo("bm3 %"PRIxMAX, (uintmax_t)bm); - - /* load a batch of masks */ - sm.v = _mm_load_si128((const __m128i *)&mask[48]); - VECTOR_SCLogInfo(sm); - /* logical AND them with the packet's mask */ - r1.v = _mm_and_si128(pm.v, sm.v); - VECTOR_SCLogInfo(r1); - /* compare the result with the original mask */ - r2.v = _mm_cmpeq_epi8(sm.v, r1.v); - VECTOR_SCLogInfo(r2); - /* convert into a bitarray */ - bm |= (((uint64_t) _mm_movemask_epi8(r2.v)) << 48); - - SCLogInfo("bm4-total %"PRIxMAX, (uintmax_t)bm); - - int b = 0; - for ( ; b < 64; b++){ - if (bm & ((uint64_t)1 << b)) { - SCLogInfo("b %02d, set", b); - } else { - SCLogInfo("b %02d, not set", b); - - } - } - - if ((bm & ((uint64_t)1 << 31)) && (bm & ((uint64_t)1 << 62))) { - return 1; - } - return 0; -#else - return 1; -#endif -} -#endif /* UNITTESTS */ - -void DetectSimdRegisterTests(void) -{ -#ifdef UNITTESTS - UtRegisterTest("SigTestSIMDMask01", SigTestSIMDMask01, 1); - UtRegisterTest("SigTestSIMDMask02", SigTestSIMDMask02, 1); - UtRegisterTest("SigTestSIMDMask03", SigTestSIMDMask03, 1); - UtRegisterTest("SigTestSIMDMask04", SigTestSIMDMask04, 1); -#endif /* UNITTESTS */ -} diff --git a/src/detect.c b/src/detect.c index 28d4082501..b2ef965202 100644 --- a/src/detect.c +++ b/src/detect.c @@ -483,122 +483,6 @@ int SigLoadSignatures(DetectEngineCtx *de_ctx, char *sig_file, int sig_file_excl SCReturnInt(ret); } -/** - * \brief See if we can prefilter a signature on inexpensive checks - * - * Order of SignatureHeader access: - * 1. flags - * 2. alproto - * 3. mpm_pattern_id_div8 - * 4. mpm_pattern_id_mod8 - * 5. num - * - * \retval 0 can't match, don't inspect - * \retval 1 might match, further inspection required - */ -int SigMatchSignaturesBuildMatchArrayAddSignature(DetectEngineThreadCtx *det_ctx, - Packet *p, SignatureHeader *s, - AppProto alproto) -{ - /* if the sig has alproto and the session as well they should match */ - if (likely(s->flags & SIG_FLAG_APPLAYER)) { - if (s->alproto != ALPROTO_UNKNOWN && s->alproto != alproto) { - if (s->alproto == ALPROTO_DCERPC) { - if (alproto != ALPROTO_SMB && alproto != ALPROTO_SMB2) { - SCLogDebug("DCERPC sig, alproto not SMB or SMB2"); - return 0; - } - } else { - SCLogDebug("alproto mismatch"); - return 0; - } - } - } - - if (unlikely(s->flags & SIG_FLAG_DSIZE)) { - if (likely(p->payload_len < s->dsize_low || p->payload_len > s->dsize_high)) { - SCLogDebug("kicked out as p->payload_len %u, dsize low %u, hi %u", - p->payload_len, s->dsize_low, s->dsize_high); - return 0; - } - } - - /* check for a pattern match of the one pattern in this sig. */ - if (likely(s->flags & (SIG_FLAG_MPM_PACKET|SIG_FLAG_MPM_STREAM|SIG_FLAG_MPM_APPLAYER))) - { - /* filter out sigs that want pattern matches, but - * have no matches */ - if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id_div_8)] & s->mpm_pattern_id_mod_8)) { - if (s->flags & SIG_FLAG_MPM_PACKET) { - if (!(s->flags & SIG_FLAG_MPM_PACKET_NEG)) { - return 0; - } - } else if (s->flags & SIG_FLAG_MPM_STREAM) { - /* filter out sigs that want pattern matches, but - * have no matches */ - if (!(s->flags & SIG_FLAG_MPM_STREAM_NEG)) { - return 0; - } - } else if (s->flags & SIG_FLAG_MPM_APPLAYER) { - if (!(s->flags & SIG_FLAG_MPM_APPLAYER_NEG)) { - return 0; - } - } - } - } - - /* de_state check, filter out all signatures that already had a match before - * or just partially match */ - if (s->flags & SIG_FLAG_STATE_MATCH) { - /* we run after DeStateDetectContinueDetection, so we might have - * state NEW here. In that case we'd want to continue detection - * for this sig. If we have NOSTATE, stateful detection didn't - * start yet for this sig, so we will inspect it. - */ - if (det_ctx->de_state_sig_array[s->num] == DE_STATE_MATCH_NO_NEW_STATE) - return 0; - } - - return 1; -} - -#if defined(__SSE3__) || defined(__tile__) -/* SIMD implementations are in detect-simd.c */ -#else -/* Non-SIMD implementation */ -/** - * \brief build an array of signatures that will be inspected - * - * All signatures that can be filtered out on forehand are not added to it. - * - * \param de_ctx detection engine ctx - * \param det_ctx detection engine thread ctx -- array is stored here - * \param p packet - * \param mask Packets mask - * \param alproto application layer protocol - */ -void SigMatchSignaturesBuildMatchArray(DetectEngineThreadCtx *det_ctx, - Packet *p, SignatureMask mask, - AppProto alproto) -{ - uint32_t u; - - /* reset previous run */ - det_ctx->match_array_cnt = 0; - - for (u = 0; u < det_ctx->sgh->sig_cnt; u++) { - SignatureHeader *s = &det_ctx->sgh->head_array[u]; - if ((mask & s->mask) == s->mask) { - if (SigMatchSignaturesBuildMatchArrayAddSignature(det_ctx, p, s, alproto) == 1) { - /* okay, store it */ - det_ctx->match_array[det_ctx->match_array_cnt] = s->full_sig; - det_ctx->match_array_cnt++; - } - } - } -} -#endif /* No SIMD implementation */ - int SigMatchSignaturesRunPostMatch(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p, Signature *s) @@ -4192,8 +4076,6 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) SigGroupHead *sgh = de_ctx->sgh_array[idx]; if (sgh == NULL) continue; - - SigGroupHeadBuildHeadArray(de_ctx, sgh); SigGroupHeadSetFilemagicFlag(de_ctx, sgh); SigGroupHeadSetFileMd5Flag(de_ctx, sgh); SigGroupHeadSetFilesizeFlag(de_ctx, sgh); @@ -4204,7 +4086,6 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) } if (de_ctx->decoder_event_sgh != NULL) { - SigGroupHeadBuildHeadArray(de_ctx, de_ctx->decoder_event_sgh); /* no need to set filestore count here as that would make a * signature not decode event only. */ } @@ -12274,7 +12155,9 @@ void SigRegisterTests(void) UtRegisterTest("SigTestPorts01", SigTestPorts01, 1); UtRegisterTest("SigTestBug01", SigTestBug01, 1); +#if 0 DetectSimdRegisterTests(); +#endif #endif /* UNITTESTS */ } diff --git a/src/detect.h b/src/detect.h index 3a31af255d..8d089be924 100644 --- a/src/detect.h +++ b/src/detect.h @@ -333,37 +333,6 @@ typedef struct IPOnlyCIDRItem_ { } IPOnlyCIDRItem; -/** \brief Subset of the Signature for cache efficient prefiltering - */ -typedef struct SignatureHeader_ { - union { - struct { - /* coccinelle: SignatureHeader:flags:SIG_FLAG */ - uint32_t flags; - AppProto alproto; - uint16_t dsize_low; - }; - uint64_t hdr_copy1; - }; - union { - struct { - uint16_t dsize_high; - uint16_t mpm_pattern_id_div_8; - }; - uint32_t hdr_copy2; - }; - union { - struct { - uint8_t mpm_pattern_id_mod_8; - SignatureMask mask; - SigIntId num; /**< signature number, internal id */ - }; - uint32_t hdr_copy3; - }; - /** pointer to the full signature */ - struct Signature_ *full_sig; -} SignatureHeader; - /** \brief Used to start a pointer to SigMatch context * Should never be dereferenced without casting to something else. */ @@ -988,10 +957,6 @@ typedef struct SigGroupHead_ { #if defined(__SSE3__) || defined(__tile__) SignatureMask *mask_array; #endif - /** chunk of memory containing the "header" part of each - * signature ordered as an array. Used to pre-filter the - * signatures to be inspected in a cache efficient way. */ - SignatureHeader *head_array; SigIntId *non_mpm_id_array; uint32_t non_mpm_id_cnt; // size is cnt * sizeof(uint32_t) @@ -1184,9 +1149,6 @@ Signature *SigFindSignatureBySidGid(DetectEngineCtx *, uint32_t, uint32_t); void SigMatchSignaturesBuildMatchArray(DetectEngineThreadCtx *, Packet *, SignatureMask, uint16_t); -int SigMatchSignaturesBuildMatchArrayAddSignature(DetectEngineThreadCtx *, - Packet *, SignatureHeader *, - uint16_t); void SigMatchFree(SigMatch *sm); void SigCleanSignatures(DetectEngineCtx *);