From b84d6d402f7d73a892a819ba038b67ee2e5a946c Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 1 Oct 2015 15:33:42 +0200 Subject: [PATCH] detect grouping: multiple whitelist conditions Instead of the binary yes/no whitelisting used so far, use different values for different sorts of whitelist reasons. The port list will be sorted by whitelist value first, then by rule count. The goal is to whitelist groups that have weak sigs: - 1 byte pattern groups - SYN sigs Rules that check for SYN packets are mostly scan detection rules. They will be checked often as SYN packets are very common. e.g. alert tcp any any -> any 22 (flags:S,12; sid:123;) This patch adds whitelisting for SYN-sigs, so that the sigs end up in as unique groups as possible. - negated mpm sigs Currently negated mpm sigs are inspected often, so they are quite expensive. For this reason, try to whitelist them. These values are set during 'stage 1', rule preprocessing. --- src/detect-engine-siggroup.c | 2 +- src/detect.c | 120 ++++++++++++++++++++++++++++++++--- src/detect.h | 5 ++ 3 files changed, 116 insertions(+), 11 deletions(-) diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 86afa4e796..96a6775ad8 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -630,7 +630,7 @@ int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHea (*dst)->init->sig_array[idx] = (*dst)->init->sig_array[idx] | src->init->sig_array[idx]; if (src->init->whitelist) - (*dst)->init->whitelist = 1; + (*dst)->init->whitelist = MAX((*dst)->init->whitelist, src->init->whitelist); if (src->mpm_content_minlen != 0) { if ((*dst)->mpm_content_minlen == 0) diff --git a/src/detect.c b/src/detect.c index 681fc08216..2353969244 100644 --- a/src/detect.c +++ b/src/detect.c @@ -2859,6 +2859,40 @@ static void SigParseApplyDsizeToContent(Signature *s) } } +/** \brief Pure-PCRE or bytetest rule */ +int RuleInspectsPayloadHasNoMpm(const Signature *s) +{ + if (s->mpm_sm == NULL && s->sm_lists[DETECT_SM_LIST_PMATCH] != NULL) + return 1; + return 0; +} + +int RuleGetMpmPatternSize(const Signature *s) +{ + if (s->mpm_sm == NULL) + return -1; + int mpm_list = SigMatchListSMBelongsTo(s, s->mpm_sm); + if (mpm_list < 0) + return -1; + const DetectContentData *cd = (const DetectContentData *)s->mpm_sm->ctx; + if (cd == NULL) + return -1; + return (int)cd->content_len; +} + +int RuleMpmIsNegated(const Signature *s) +{ + if (s->mpm_sm == NULL) + return 0; + int mpm_list = SigMatchListSMBelongsTo(s, s->mpm_sm); + if (mpm_list < 0) + return 0; + const DetectContentData *cd = (const DetectContentData *)s->mpm_sm->ctx; + if (cd == NULL) + return 0; + return (cd->flags & DETECT_CONTENT_NEGATED); +} + int RulesGroupByProto(DetectEngineCtx *de_ctx) { Signature *s = de_ctx->sig_list; @@ -2998,6 +3032,49 @@ static int PortIsWhitelisted(const DetectPort *a, int ipproto) return 0; } +static int RuleSetWhitelist(Signature *s) +{ + DetectPort *p = NULL; + if (s->flags & SIG_FLAG_TOSERVER) + p = s->dp; + else if (s->flags & SIG_FLAG_TOCLIENT) + p = s->sp; + else + return 0; + + /* for sigs that don't use 'any' as port, see if we want to + * whitelist poor sigs */ + int wl = 0; + if (!(p->port == 0 && p->port2 == 65535)) { + /* pure pcre, bytetest, etc rules */ + if (RuleInspectsPayloadHasNoMpm(s)) { + SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id); + wl = 99; + + } else if (RuleMpmIsNegated(s)) { + SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id); + wl = 77; + + /* one byte pattern in packet/stream payloads */ + } else if (s->mpm_sm != NULL && + SigMatchListSMBelongsTo(s, s->mpm_sm) == DETECT_SM_LIST_PMATCH && + RuleGetMpmPatternSize(s) == 1) + { + SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id); + wl = 55; + + } else if (DetectFlagsSignatureNeedsSynPackets(s) && + DetectFlagsSignatureNeedsSynOnlyPackets(s)) + { + SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id); + wl = 33; + } + } + + s->whitelist = wl; + return wl; +} + int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx); int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b); @@ -3040,11 +3117,14 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint3 goto next; } + int wl = s->whitelist; while (p) { DetectPort *tmp = DetectPortCopySingle(de_ctx, p); BUG_ON(tmp == NULL); SigGroupHeadAppendSig(de_ctx, &tmp->sh, s); - tmp->sh->init->whitelist = PortIsWhitelisted(tmp, ipproto); + + int pwl = PortIsWhitelisted(tmp, ipproto) ? 111 : 0; + tmp->sh->init->whitelist = MAX(wl, pwl); if (tmp->sh->init->whitelist) { SCLogDebug("%s/%s Rule %u whitelisted port group %u:%u", direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient", @@ -3107,10 +3187,11 @@ static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint3 } #if 0 for (iter = list ; iter != NULL; iter = iter->next) { - SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s)", + SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)", iter->port, iter->port2, iter->sh, iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own", - iter->sh->init->whitelist ? "true" : "false"); + iter->sh->init->whitelist ? "true" : "false", + iter->sh->init->whitelist); } #endif SCLogInfo("%s %s: %u port groups, %u unique SGH's, %u copies", @@ -3223,6 +3304,8 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) SignatureCreateMask(tmp_s); SigParseApplyDsizeToContent(tmp_s); + RuleSetWhitelist(tmp_s); + de_ctx->sig_cnt++; } @@ -3246,23 +3329,40 @@ error: return -1; } -static int PortGroupIsWhitelisted(const DetectPort *a) +static int PortGroupWhitelist(const DetectPort *a) { return a->sh->init->whitelist; } int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b) { - if (PortGroupIsWhitelisted(a) && !PortGroupIsWhitelisted(b)) + if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) { + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); return 1; - if (!PortGroupIsWhitelisted(a) && PortGroupIsWhitelisted(b)) + } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) { + SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); return 0; - if (a->sh->sig_cnt > b->sh->sig_cnt) { - SCLogDebug("pg %u:%u %u > %u:%u %u", - a->port, a->port2, a->sh->sig_cnt, - b->port, b->port2, b->sh->sig_cnt); + } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) { + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); return 1; + } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) { + if (a->sh->sig_cnt > b->sh->sig_cnt) { + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + return 1; + } } + + SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); return 0; } diff --git a/src/detect.h b/src/detect.h index 11c2c19339..03c354da8d 100644 --- a/src/detect.h +++ b/src/detect.h @@ -460,6 +460,11 @@ typedef struct Signature_ { /* SigMatch list used for adding content and friends. E.g. file_data; */ int list; + /** score to influence rule grouping. A higher value leads to a higher + * likelyhood of a rulegroup with this sig ending up as a contained + * group. */ + int whitelist; + /* Be careful, this pointer is only valid while parsing the sig, * to warn the user about any possible problem */ char *sig_str;