From b5a31271512cef7c7fd2ad4b009e46b8f6490d40 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Fri, 7 Nov 2014 20:57:14 +0100 Subject: [PATCH] detect: add mask check prefilter for non mpm list Add mask array for non_mpm sigs, so that we can exclude many sigs before we merge sort. Shows 50% less non mpm sigs inspected on average. --- src/detect-engine-siggroup.c | 13 +++++++++++++ src/detect-engine.c | 9 +++++++++ src/detect.c | 30 +++++++++++++++++++++++------- src/detect.h | 11 +++++++++-- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 3b8a9ed3bf..5f3a9f14ab 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -198,6 +198,12 @@ void SigGroupHeadFree(SigGroupHead *sgh) sgh->non_mpm_id_cnt = 0; } + if (sgh->non_mpm_mask_array != NULL) { + SCFree(sgh->non_mpm_mask_array); + sgh->non_mpm_mask_array = NULL; + sgh->non_mpm_mask_cnt = 0; + } + sgh->sig_cnt = 0; if (sgh->init != NULL) { @@ -1700,6 +1706,7 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh) return 0; BUG_ON(sgh->non_mpm_id_array != NULL); + BUG_ON(sgh->non_mpm_mask_array != NULL); for (sig = 0; sig < sgh->sig_cnt; sig++) { s = sgh->match_array[sig]; @@ -1721,6 +1728,10 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh) BUG_ON(sgh->non_mpm_id_array == NULL); memset(sgh->non_mpm_id_array, 0, non_mpm * sizeof(SigIntId)); + sgh->non_mpm_mask_array = SCMalloc(non_mpm * sizeof(SignatureMask)); + BUG_ON(sgh->non_mpm_mask_array == NULL); + memset(sgh->non_mpm_mask_array, 0, non_mpm * sizeof(SignatureMask)); + for (sig = 0; sig < sgh->sig_cnt; sig++) { s = sgh->match_array[sig]; if (s == NULL) @@ -1729,9 +1740,11 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh) if (s->mpm_sm == NULL) { BUG_ON(sgh->non_mpm_id_cnt >= non_mpm); sgh->non_mpm_id_array[sgh->non_mpm_id_cnt++] = s->num; + sgh->non_mpm_mask_array[sgh->non_mpm_mask_cnt++] = s->mask; } else if (s->flags & (SIG_FLAG_MPM_PACKET_NEG|SIG_FLAG_MPM_STREAM_NEG|SIG_FLAG_MPM_APPLAYER_NEG)) { BUG_ON(sgh->non_mpm_id_cnt >= non_mpm); sgh->non_mpm_id_array[sgh->non_mpm_id_cnt++] = s->num; + sgh->non_mpm_mask_array[sgh->non_mpm_mask_cnt++] = s->mask; } } return 0; diff --git a/src/detect-engine.c b/src/detect-engine.c index f2247b5d6b..8f8cc2095d 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -1361,6 +1361,8 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data) SC_PERF_TYPE_UINT64, "NULL"); uint16_t counter_nonmpm_list = SCPerfTVRegisterAvgCounter("detect.nonmpm_list", tv, SC_PERF_TYPE_UINT64, "NULL"); + uint16_t counter_fnonmpm_list = SCPerfTVRegisterAvgCounter("detect.fnonmpm_list", tv, + SC_PERF_TYPE_UINT64, "NULL"); uint16_t counter_match_list = SCPerfTVRegisterAvgCounter("detect.match_list", tv, SC_PERF_TYPE_UINT64, "NULL"); #endif @@ -1377,6 +1379,9 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data) det_ctx->tv = tv; det_ctx->de_ctx = de_ctx; + det_ctx->non_mpm_id_array = SCCalloc(32000, sizeof(SigIntId)); // TODO proper size or dynamicly grow + BUG_ON(det_ctx->non_mpm_id_array == NULL); + if (ThreadCtxDoInit(de_ctx, det_ctx) != TM_ECODE_OK) return TM_ECODE_FAILED; @@ -1385,6 +1390,7 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data) #ifdef PROFILING det_ctx->counter_mpm_list = counter_mpm_list; det_ctx->counter_nonmpm_list = counter_nonmpm_list; + det_ctx->counter_fnonmpm_list = counter_fnonmpm_list; det_ctx->counter_match_list = counter_match_list; #endif @@ -1458,6 +1464,9 @@ TmEcode DetectEngineThreadCtxDeinit(ThreadVars *tv, void *data) PmqFree(&det_ctx->smsg_pmq[i]); } + if (det_ctx->non_mpm_id_array != NULL) + SCFree(det_ctx->non_mpm_id_array); + if (det_ctx->de_state_sig_array != NULL) SCFree(det_ctx->de_state_sig_array); if (det_ctx->match_array != NULL) diff --git a/src/detect.c b/src/detect.c index 9977908314..1b561ff912 100644 --- a/src/detect.c +++ b/src/detect.c @@ -663,17 +663,19 @@ end: } static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx, - DetectEngineThreadCtx *det_ctx, - SigGroupHead *sgh) + DetectEngineThreadCtx *det_ctx) +// SigGroupHead *sgh) { SigIntId mpm, nonmpm; det_ctx->match_array_cnt = 0; SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array; - SigIntId *nonmpm_ptr = sgh->non_mpm_id_array; + SigIntId *nonmpm_ptr = det_ctx->non_mpm_id_array; + //SigIntId *nonmpm_ptr = sgh->non_mpm_id_array; uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt; - uint32_t n_cnt = sgh->non_mpm_id_cnt; + //uint32_t n_cnt = sgh->non_mpm_id_cnt; + uint32_t n_cnt = det_ctx->non_mpm_id_cnt; SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt); - SCLogDebug("SGH non-MPM id count %d", sgh->non_mpm_id_cnt); +// SCLogDebug("SGH non-MPM id count %d", sgh->non_mpm_id_cnt); SigIntId *final_ptr; uint32_t final_cnt; SigIntId id; @@ -752,7 +754,7 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx, det_ctx->match_array_cnt = match_array - det_ctx->match_array; - BUG_ON((det_ctx->pmq.rule_id_array_cnt + sgh->non_mpm_id_cnt) < det_ctx->match_array_cnt); + BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_mpm_id_cnt) < det_ctx->match_array_cnt); } /* Return true is the list is sorted smallest to largest */ @@ -1330,6 +1332,17 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh SignatureMask mask = 0; PacketCreateMask(p, &mask, alproto, has_state, smsg, app_decoder_events); + /* prefilter non_mpm list against the mask of the packet */ + det_ctx->non_mpm_id_cnt = 0; + uint32_t x = 0; + for (x = 0; x < det_ctx->sgh->non_mpm_id_cnt; x++) { + /* only if the mask matches this rule can possibly match, + * so build the non_mpm array only for match candidates */ + if ((det_ctx->sgh->non_mpm_mask_array[x] & mask) == det_ctx->sgh->non_mpm_mask_array[x]) { + det_ctx->non_mpm_id_array[det_ctx->non_mpm_id_cnt++] = det_ctx->sgh->non_mpm_id_array[x]; + } + } + /* run the mpm for each type */ PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM); DetectMpmPrefilter(de_ctx, det_ctx, smsg, p, flags, alproto, has_state, &sms_runflags); @@ -1340,11 +1353,14 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh (uint64_t)det_ctx->pmq.rule_id_array_cnt); SCPerfCounterAddUI64(det_ctx->counter_nonmpm_list, th_v->sc_perf_pca, (uint64_t)det_ctx->sgh->non_mpm_id_cnt); + /* non mpm sigs after mask prefilter */ + SCPerfCounterAddUI64(det_ctx->counter_fnonmpm_list, + th_v->sc_perf_pca, (uint64_t)det_ctx->non_mpm_id_cnt); } #endif PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PREFILTER); - DetectPrefilterMergeSort(de_ctx, det_ctx, det_ctx->sgh); + DetectPrefilterMergeSort(de_ctx, det_ctx); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PREFILTER); PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES); diff --git a/src/detect.h b/src/detect.h index 2ed8b5a3c0..9ded0077e1 100644 --- a/src/detect.h +++ b/src/detect.h @@ -762,6 +762,9 @@ typedef struct DetectionEngineThreadCtx_ { /* the thread to which this detection engine thread belongs */ ThreadVars *tv; + SigIntId *non_mpm_id_array; + uint32_t non_mpm_id_cnt; // size is cnt * sizeof(uint32_t) + /* detection engine variables */ /** offset into the payload of the last match by: @@ -791,9 +794,10 @@ typedef struct DetectionEngineThreadCtx_ { /** id for alert counter */ uint16_t counter_alerts; -#ifdef COLLECT_SIGMATCH_LIST_STATS +#ifdef PROFILING uint16_t counter_mpm_list; uint16_t counter_nonmpm_list; + uint16_t counter_fnonmpm_list; uint16_t counter_match_list; #endif @@ -964,7 +968,10 @@ typedef struct SigGroupHead_ { #endif SigIntId *non_mpm_id_array; - uint32_t non_mpm_id_cnt; // size is cnt * sizeof(uint32_t) + uint32_t non_mpm_id_cnt; // size is cnt * sizeof(SigIntId) + + SignatureMask *non_mpm_mask_array; + uint32_t non_mpm_mask_cnt; // size is cnt * sizeof(SignatureMask) /* pattern matcher instances */ MpmCtx *mpm_proto_other_ctx;