aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

detect: add mask check prefilter for non mpm list

Browse Source 
Add mask array for non_mpm sigs, so that we can exclude many sigs before
we merge sort.

Shows 50% less non mpm sigs inspected on average.
pull/1295/head
Victor Julien 11 years ago
parent 904441327c
commit b5a3127151
4 changed files with 54 additions and 9 deletions
Show Stats Download Patch File Download Diff File

13
src/detect-engine-siggroup.c
Unescape Escape View File

@ -198,6 +198,12 @@ void SigGroupHeadFree(SigGroupHead *sgh)
sgh->non_mpm_id_cnt = 0; sgh->non_mpm_id_cnt = 0;
} }
if (sgh->non_mpm_mask_array != NULL) {
SCFree(sgh->non_mpm_mask_array);
sgh->non_mpm_mask_array = NULL;
sgh->non_mpm_mask_cnt = 0;
}
sgh->sig_cnt = 0; sgh->sig_cnt = 0;
if (sgh->init != NULL) { if (sgh->init != NULL) {
@ -1700,6 +1706,7 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
return 0; return 0;
BUG_ON(sgh->non_mpm_id_array != NULL); BUG_ON(sgh->non_mpm_id_array != NULL);
BUG_ON(sgh->non_mpm_mask_array != NULL);
for (sig = 0; sig < sgh->sig_cnt; sig++) { for (sig = 0; sig < sgh->sig_cnt; sig++) {
s = sgh->match_array[sig]; s = sgh->match_array[sig];
@ -1721,6 +1728,10 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
BUG_ON(sgh->non_mpm_id_array == NULL); BUG_ON(sgh->non_mpm_id_array == NULL);
memset(sgh->non_mpm_id_array, 0, non_mpm * sizeof(SigIntId)); memset(sgh->non_mpm_id_array, 0, non_mpm * sizeof(SigIntId));
sgh->non_mpm_mask_array = SCMalloc(non_mpm * sizeof(SignatureMask));
BUG_ON(sgh->non_mpm_mask_array == NULL);
memset(sgh->non_mpm_mask_array, 0, non_mpm * sizeof(SignatureMask));
for (sig = 0; sig < sgh->sig_cnt; sig++) { for (sig = 0; sig < sgh->sig_cnt; sig++) {
s = sgh->match_array[sig]; s = sgh->match_array[sig];
if (s == NULL) if (s == NULL)
@ -1729,9 +1740,11 @@ int SigGroupHeadBuildNonMpmArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
if (s->mpm_sm == NULL) { if (s->mpm_sm == NULL) {
BUG_ON(sgh->non_mpm_id_cnt >= non_mpm); BUG_ON(sgh->non_mpm_id_cnt >= non_mpm);
sgh->non_mpm_id_array[sgh->non_mpm_id_cnt++] = s->num; sgh->non_mpm_id_array[sgh->non_mpm_id_cnt++] = s->num;
sgh->non_mpm_mask_array[sgh->non_mpm_mask_cnt++] = s->mask;
} else if (s->flags & (SIG_FLAG_MPM_PACKET_NEG|SIG_FLAG_MPM_STREAM_NEG|SIG_FLAG_MPM_APPLAYER_NEG)) { } else if (s->flags & (SIG_FLAG_MPM_PACKET_NEG|SIG_FLAG_MPM_STREAM_NEG|SIG_FLAG_MPM_APPLAYER_NEG)) {
BUG_ON(sgh->non_mpm_id_cnt >= non_mpm); BUG_ON(sgh->non_mpm_id_cnt >= non_mpm);
sgh->non_mpm_id_array[sgh->non_mpm_id_cnt++] = s->num; sgh->non_mpm_id_array[sgh->non_mpm_id_cnt++] = s->num;
sgh->non_mpm_mask_array[sgh->non_mpm_mask_cnt++] = s->mask;
} }
} }
return 0; return 0;

9
src/detect-engine.c
Unescape Escape View File

@ -1361,6 +1361,8 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
SC_PERF_TYPE_UINT64, "NULL"); SC_PERF_TYPE_UINT64, "NULL");
uint16_t counter_nonmpm_list = SCPerfTVRegisterAvgCounter("detect.nonmpm_list", tv, uint16_t counter_nonmpm_list = SCPerfTVRegisterAvgCounter("detect.nonmpm_list", tv,
SC_PERF_TYPE_UINT64, "NULL"); SC_PERF_TYPE_UINT64, "NULL");
uint16_t counter_fnonmpm_list = SCPerfTVRegisterAvgCounter("detect.fnonmpm_list", tv,
SC_PERF_TYPE_UINT64, "NULL");
uint16_t counter_match_list = SCPerfTVRegisterAvgCounter("detect.match_list", tv, uint16_t counter_match_list = SCPerfTVRegisterAvgCounter("detect.match_list", tv,
SC_PERF_TYPE_UINT64, "NULL"); SC_PERF_TYPE_UINT64, "NULL");
#endif #endif
@ -1377,6 +1379,9 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
det_ctx->tv = tv; det_ctx->tv = tv;
det_ctx->de_ctx = de_ctx; det_ctx->de_ctx = de_ctx;
det_ctx->non_mpm_id_array = SCCalloc(32000, sizeof(SigIntId)); // TODO proper size or dynamicly grow
BUG_ON(det_ctx->non_mpm_id_array == NULL);
if (ThreadCtxDoInit(de_ctx, det_ctx) != TM_ECODE_OK) if (ThreadCtxDoInit(de_ctx, det_ctx) != TM_ECODE_OK)
return TM_ECODE_FAILED; return TM_ECODE_FAILED;
@ -1385,6 +1390,7 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
#ifdef PROFILING #ifdef PROFILING
det_ctx->counter_mpm_list = counter_mpm_list; det_ctx->counter_mpm_list = counter_mpm_list;
det_ctx->counter_nonmpm_list = counter_nonmpm_list; det_ctx->counter_nonmpm_list = counter_nonmpm_list;
det_ctx->counter_fnonmpm_list = counter_fnonmpm_list;
det_ctx->counter_match_list = counter_match_list; det_ctx->counter_match_list = counter_match_list;
#endif #endif
@ -1458,6 +1464,9 @@ TmEcode DetectEngineThreadCtxDeinit(ThreadVars *tv, void *data)
PmqFree(&det_ctx->smsg_pmq[i]); PmqFree(&det_ctx->smsg_pmq[i]);
} }
if (det_ctx->non_mpm_id_array != NULL)
SCFree(det_ctx->non_mpm_id_array);
if (det_ctx->de_state_sig_array != NULL) if (det_ctx->de_state_sig_array != NULL)
SCFree(det_ctx->de_state_sig_array); SCFree(det_ctx->de_state_sig_array);
if (det_ctx->match_array != NULL) if (det_ctx->match_array != NULL)

30
src/detect.c
Unescape Escape View File

@ -663,17 +663,19 @@ end:
} }
static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx, static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
DetectEngineThreadCtx *det_ctx, DetectEngineThreadCtx *det_ctx)
SigGroupHead *sgh) // SigGroupHead *sgh)
{ {
SigIntId mpm, nonmpm; SigIntId mpm, nonmpm;
det_ctx->match_array_cnt = 0; det_ctx->match_array_cnt = 0;
SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array; SigIntId *mpm_ptr = det_ctx->pmq.rule_id_array;
SigIntId *nonmpm_ptr = sgh->non_mpm_id_array; SigIntId *nonmpm_ptr = det_ctx->non_mpm_id_array;
//SigIntId *nonmpm_ptr = sgh->non_mpm_id_array;
uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt; uint32_t m_cnt = det_ctx->pmq.rule_id_array_cnt;
uint32_t n_cnt = sgh->non_mpm_id_cnt; //uint32_t n_cnt = sgh->non_mpm_id_cnt;
uint32_t n_cnt = det_ctx->non_mpm_id_cnt;
SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt); SCLogDebug("PMQ rule id array count %d", det_ctx->pmq.rule_id_array_cnt);
SCLogDebug("SGH non-MPM id count %d", sgh->non_mpm_id_cnt); // SCLogDebug("SGH non-MPM id count %d", sgh->non_mpm_id_cnt);
SigIntId *final_ptr; SigIntId *final_ptr;
uint32_t final_cnt; uint32_t final_cnt;
SigIntId id; SigIntId id;
@ -752,7 +754,7 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
det_ctx->match_array_cnt = match_array - det_ctx->match_array; det_ctx->match_array_cnt = match_array - det_ctx->match_array;
BUG_ON((det_ctx->pmq.rule_id_array_cnt + sgh->non_mpm_id_cnt) < det_ctx->match_array_cnt); BUG_ON((det_ctx->pmq.rule_id_array_cnt + det_ctx->non_mpm_id_cnt) < det_ctx->match_array_cnt);
} }
/* Return true is the list is sorted smallest to largest */ /* Return true is the list is sorted smallest to largest */
@ -1330,6 +1332,17 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
SignatureMask mask = 0; SignatureMask mask = 0;
PacketCreateMask(p, &mask, alproto, has_state, smsg, app_decoder_events); PacketCreateMask(p, &mask, alproto, has_state, smsg, app_decoder_events);
/* prefilter non_mpm list against the mask of the packet */
det_ctx->non_mpm_id_cnt = 0;
uint32_t x = 0;
for (x = 0; x < det_ctx->sgh->non_mpm_id_cnt; x++) {
/* only if the mask matches this rule can possibly match,
* so build the non_mpm array only for match candidates */
if ((det_ctx->sgh->non_mpm_mask_array[x] & mask) == det_ctx->sgh->non_mpm_mask_array[x]) {
det_ctx->non_mpm_id_array[det_ctx->non_mpm_id_cnt++] = det_ctx->sgh->non_mpm_id_array[x];
}
}
/* run the mpm for each type */ /* run the mpm for each type */
PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM); PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM);
DetectMpmPrefilter(de_ctx, det_ctx, smsg, p, flags, alproto, has_state, &sms_runflags); DetectMpmPrefilter(de_ctx, det_ctx, smsg, p, flags, alproto, has_state, &sms_runflags);
@ -1340,11 +1353,14 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
(uint64_t)det_ctx->pmq.rule_id_array_cnt); (uint64_t)det_ctx->pmq.rule_id_array_cnt);
SCPerfCounterAddUI64(det_ctx->counter_nonmpm_list, th_v->sc_perf_pca, SCPerfCounterAddUI64(det_ctx->counter_nonmpm_list, th_v->sc_perf_pca,
(uint64_t)det_ctx->sgh->non_mpm_id_cnt); (uint64_t)det_ctx->sgh->non_mpm_id_cnt);
/* non mpm sigs after mask prefilter */
SCPerfCounterAddUI64(det_ctx->counter_fnonmpm_list,
th_v->sc_perf_pca, (uint64_t)det_ctx->non_mpm_id_cnt);
} }
#endif #endif
PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PREFILTER); PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PREFILTER);
DetectPrefilterMergeSort(de_ctx, det_ctx, det_ctx->sgh); DetectPrefilterMergeSort(de_ctx, det_ctx);
PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PREFILTER); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PREFILTER);
PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES); PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES);

11
src/detect.h
Unescape Escape View File

@ -762,6 +762,9 @@ typedef struct DetectionEngineThreadCtx_ {
/* the thread to which this detection engine thread belongs */ /* the thread to which this detection engine thread belongs */
ThreadVars *tv; ThreadVars *tv;
SigIntId *non_mpm_id_array;
uint32_t non_mpm_id_cnt; // size is cnt * sizeof(uint32_t)
/* detection engine variables */ /* detection engine variables */
/** offset into the payload of the last match by: /** offset into the payload of the last match by:
@ -791,9 +794,10 @@ typedef struct DetectionEngineThreadCtx_ {
/** id for alert counter */ /** id for alert counter */
uint16_t counter_alerts; uint16_t counter_alerts;
#ifdef COLLECT_SIGMATCH_LIST_STATS #ifdef PROFILING
uint16_t counter_mpm_list; uint16_t counter_mpm_list;
uint16_t counter_nonmpm_list; uint16_t counter_nonmpm_list;
uint16_t counter_fnonmpm_list;
uint16_t counter_match_list; uint16_t counter_match_list;
#endif #endif
@ -964,7 +968,10 @@ typedef struct SigGroupHead_ {
#endif #endif
SigIntId *non_mpm_id_array; SigIntId *non_mpm_id_array;
uint32_t non_mpm_id_cnt; // size is cnt * sizeof(uint32_t) uint32_t non_mpm_id_cnt; // size is cnt * sizeof(SigIntId)
SignatureMask *non_mpm_mask_array;
uint32_t non_mpm_mask_cnt; // size is cnt * sizeof(SignatureMask)
/* pattern matcher instances */ /* pattern matcher instances */
MpmCtx *mpm_proto_other_ctx; MpmCtx *mpm_proto_other_ctx;

Write Preview
Loading…
Cancel
Save