diff --git a/src/detect-content.c b/src/detect-content.c index fd58bb731e..fd79135119 100644 --- a/src/detect-content.c +++ b/src/detect-content.c @@ -102,7 +102,7 @@ TestOffsetDepth(MpmMatch *m, DetectContentData *co, u_int16_t pktoff) { } /* This function is called recursively (if nescessary) to be able - * to determite whether or not a chain of content matches connected + * to determine whether or not a chain of content matches connected * with 'within' and 'distance' options fully matches. The reason it * was done like this is to make sure we can handle partial matches * that turn out to fail being followed by full matches later in the @@ -118,9 +118,9 @@ TestWithinDistanceOffsetDepth(ThreadVars *t, PatternMatcherThread *pmt, MpmMatch MpmMatch *nm = pmt->mtc.match[co->id].top; for (; nm; nm = nm->next) { - //printf("TestWithinDistanceOffsetDepth: nm->offset %u, m->offset %u\n", nm->offset, m->offset); + //printf("TestWithinDistanceOffsetDepth: nm->offset %u, m->offset %u, pktoff %u\n", nm->offset, m->offset, pktoff); if (nm->offset >= pktoff) { - if ((co->within == 0 || (co->within && + if ((!(co->flags & DETECT_CONTENT_WITHIN) || (co->within > 0 && (nm->offset > m->offset) && ((nm->offset - m->offset + co->content_len) <= co->within)))) { @@ -128,13 +128,13 @@ TestWithinDistanceOffsetDepth(ThreadVars *t, PatternMatcherThread *pmt, MpmMatch // "nm->offset %u, m->offset %u\n", nm->offset - m->offset + co->content_len, // co->within, nm->offset, m->offset); - if (co->distance == 0 || (co->distance && - (nm->offset > m->offset) && + if (!(co->flags & DETECT_CONTENT_DISTANCE) || + ((nm->offset > m->offset) && ((nm->offset - m->offset) >= co->distance))) { //printf("TestWithinDistanceOffsetDepth: MATCH: %u >= DISTANCE(%u), " - // "nm->offset %u, m->offset %u\n", nm->offset - m->offset, - // co->distance, nm->offset, m->offset); + // "nm->offset %u, m->offset %u\n", nm->offset - m->offset, + // co->distance, nm->offset, m->offset); if (TestOffsetDepth(nm, co, pktoff) == 1) { return TestWithinDistanceOffsetDepth(t, pmt, nm, nsm->next, pktoff); } @@ -169,6 +169,9 @@ DoDetectContent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signature * co->flags & DETECT_CONTENT_DISTANCE_NEXT) && pmt->de_checking_distancewithin == 0) { + //printf("DoDetectContent: Content \""); PrintRawUriFp(stdout, co->content, co->content_len); + //printf("\" DETECT_CONTENT_WITHIN_NEXT or DETECT_CONTENT_DISTANCE_NEXT is true\n"); + /* indicate to the detection engine the next sigmatch(es) * are part of this match chain */ pmt->de_checking_distancewithin = 1; @@ -176,10 +179,10 @@ DoDetectContent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signature * for (; m != NULL; m = m->next) { /* first check our match for offset and depth */ if (TestOffsetDepth(m, co, pmt->pkt_off) == 1) { + //printf("DoDetectContent: TestOffsetDepth returned 1\n"); ret = TestWithinDistanceOffsetDepth(t, pmt, m, sm->next, pmt->pkt_off); if (ret == 1) { - /* update pkt ptrs, content doesn't use this, - * but pcre does */ + //printf("DoDetectContent: TestWithinDistanceOffsetDepth returned 1\n"); pmt->pkt_ptr = p->tcp_payload + m->offset; pmt->pkt_off = m->offset; match = 1; @@ -187,6 +190,7 @@ DoDetectContent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signature * } } } + /* Okay, this is complicated... on the first match of a match chain, * we do the whole match of that chain (a chain here means a number * of consecutive content matches that relate to each other with @@ -200,6 +204,7 @@ DoDetectContent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signature * { pmt->de_checking_distancewithin = 0; match = 1; + /* Getting here means we are not in checking an within/distance chain. * This means we can just inspect this content match on it's own. So * Let's see if at least one of the matches within the offset and depth @@ -221,10 +226,11 @@ DoDetectContent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signature * } } else { for (; m != NULL; m = m->next) { - ret = TestOffsetDepth(m,co, pmt->pkt_off); + ret = TestOffsetDepth(m,co, 0); /* no offset as we inspect each + * match on it's own */ if (ret == 1) { - /* update pkt ptrs, content doesn't use this, - * but pcre does */ + /* update pkt ptrs, this content run doesn't + * use this, but pcre does */ pmt->pkt_ptr = p->tcp_payload + m->offset; pmt->pkt_off = m->offset; match = 1; @@ -258,7 +264,8 @@ int DetectContentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Sig return 0; #ifdef DEBUG - printf("content \'%s\' matched %u time(s) at offsets: ", co->content, len); + printf("content \""); PrintRawUriFp(stdout, co->content, co->content_len); + printf("\" matched %u time(s) at offsets: ", len); MpmMatch *tmpm = NULL; for (tmpm = pmt->mtc.match[co->id].top; tmpm != NULL; tmpm = tmpm->next) { diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index bf21cfd62c..bc08d0da51 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -18,6 +18,15 @@ #include "detect-content.h" #include "detect-uricontent.h" +u_int32_t PacketPatternScan(ThreadVars *t, PatternMatcherThread *pmt, Packet *p) { + u_int32_t ret; + + ret = pmt->mc->Search(pmt->mc_scan, &pmt->mtc, p->tcp_payload, p->tcp_payload_len); + + //printf("PacketPatternMatch: ret %u\n", ret); + return ret; +} + u_int32_t PacketPatternMatch(ThreadVars *t, PatternMatcherThread *pmt, Packet *p) { u_int32_t ret; @@ -76,6 +85,27 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) { } } +static int g_content_scan = 0; +static int g_uricontent_scan = 0; +static int g_content_search = 0; +static int g_uricontent_search = 0; +static int g_content_maxdepth = 0; +static int g_content_minoffset = 0; +static int g_content_total = 0; + +void DbgPrintScanSearchStats() { +#if 0 + printf(": content scan %d, search %d (%02.1f%%) :\n", g_content_scan, g_content_search, + (float)(g_content_scan/(float)(g_content_scan+g_content_search))*100); +// printf(": uricontent scan %d, urisearch %d (%02f%%) :\n", g_uricontent_scan, g_uricontent_search, +// (float)(g_uricontent_scan/(float)(g_uricontent_scan+g_uricontent_search))*100); + printf(": content maxdepth %d, total %d (%02.1f%%) :\n", g_content_maxdepth, g_content_total, + (float)(g_content_maxdepth/(float)(g_content_total))*100); + printf(": content minoffset %d, total %d (%02.1f%%) :\n", g_content_minoffset, g_content_total, + (float)(g_content_minoffset/(float)(g_content_total))*100); +#endif +} + /* * * TODO @@ -122,11 +152,19 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) /* intialize contexes */ if (sh->flags & SIG_GROUP_HAVECONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { + /* search */ sh->mpm_ctx = malloc(sizeof(MpmCtx)); if (sh->mpm_ctx == NULL) goto error; MpmInitCtx(sh->mpm_ctx, MPM_WUMANBER); + + /* scan */ + sh->mpm_scan_ctx = malloc(sizeof(MpmCtx)); + if (sh->mpm_scan_ctx == NULL) + goto error; + + MpmInitCtx(sh->mpm_scan_ctx, MPM_WUMANBER); } if (sh->flags & SIG_GROUP_HAVEURICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) { sh->mpm_uri_ctx = malloc(sizeof(MpmCtx)); @@ -138,6 +176,10 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) u_int16_t mpm_content_maxlen = 0, mpm_uricontent_maxlen = 0; u_int32_t mpm_content_cnt = 0, mpm_uricontent_cnt = 0; + u_int16_t mpm_content_maxdepth = 65535, mpm_content_minoffset = 65535; + u_int16_t mpm_content_maxdepth_one = 65535, mpm_content_minoffset_one = 65535; + int mpm_content_depth_present = -1; + int mpm_content_offset_present = -1; /* for each signature in this group do */ for (sig = 0; sig < sh->sig_cnt; sig++) { @@ -150,15 +192,25 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) cnt++; u_int16_t content_maxlen = 0, uricontent_maxlen = 0; + u_int16_t content_minlen = 0, uricontent_minlen = 0; u_int16_t content_cnt = 0, uricontent_cnt = 0; + u_int16_t content_maxdepth = 65535; + u_int16_t content_maxdepth_one = 65535; + u_int16_t content_minoffset = 65535; + u_int16_t content_minoffset_one = 65535; SigMatch *sm; + /* determine the length of the longest pattern */ for (sm = s->match; sm != NULL; sm = sm->next) { if (sm->type == DETECT_CONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { DetectContentData *cd = (DetectContentData *)sm->ctx; if (cd->content_len > content_maxlen) content_maxlen = cd->content_len; + if (content_minlen == 0) content_minlen = cd->content_len; + else if (cd->content_len < content_minlen) + content_minlen = cd->content_len; + mpm_content_cnt++; content_cnt++; } else if (sm->type == DETECT_URICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) { @@ -166,22 +218,131 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) if (ud->uricontent_len > uricontent_maxlen) uricontent_maxlen = ud->uricontent_len; + if (uricontent_minlen == 0) uricontent_minlen = ud->uricontent_len; + else if (ud->uricontent_len < uricontent_minlen) + uricontent_minlen = ud->uricontent_len; + mpm_uricontent_cnt++; uricontent_cnt++; } } + /* determine the min offset and max depth of the longest pattern(s) */ + for (sm = s->match; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_CONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { + DetectContentData *cd = (DetectContentData *)sm->ctx; + if (cd->content_len == content_maxlen) { + if (content_maxdepth > cd->depth) + content_maxdepth = cd->depth; + + if (content_minoffset > cd->offset) + content_minoffset = cd->offset; + } + } else if (sm->type == DETECT_URICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) { + DetectUricontentData *ud = (DetectUricontentData *)sm->ctx; + if (ud->uricontent_len == uricontent_maxlen) { + } + } + } + + int content_depth_atleastone = 0; + int content_offset_atleastone = 0; + /* determine if we have at least one pattern with a depth */ + for (sm = s->match; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_CONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { + DetectContentData *cd = (DetectContentData *)sm->ctx; + if (cd->depth) { + content_depth_atleastone = 1; + + if (content_maxdepth_one > cd->depth) + content_maxdepth_one = cd->depth; + } + if (cd->offset) { + content_offset_atleastone = 1; + + if (content_minoffset_one > cd->offset) + content_minoffset_one = cd->offset; + } + } + } + + if (mpm_content_depth_present == -1) mpm_content_depth_present = content_depth_atleastone; + else if (content_depth_atleastone == 0) { + mpm_content_depth_present = 0; + } + + if (mpm_content_offset_present == -1) mpm_content_offset_present = content_offset_atleastone; + else if (content_offset_atleastone == 0) { + mpm_content_offset_present = 0; + } + + if (content_maxdepth == 65535) + content_maxdepth = 0; + if (content_maxdepth_one == 65535) + content_maxdepth_one = 0; + if (content_minoffset == 65535) + content_minoffset = 0; + if (content_minoffset_one == 65535) + content_minoffset_one = 0; + + if (content_maxdepth != 0) { + //printf("content_maxdepth %u (sid %u)\n", content_maxdepth, s->id); + } + if (content_minoffset != 0) { + //printf("content_minoffset %u (sid %u)\n", content_minoffset, s->id); + } + + if (mpm_content_maxdepth > content_maxdepth) + mpm_content_maxdepth = content_maxdepth; + if (mpm_content_maxdepth_one > content_maxdepth_one) + mpm_content_maxdepth_one = content_maxdepth_one; + if (mpm_content_minoffset > content_minoffset) + mpm_content_minoffset = content_minoffset; + if (mpm_content_minoffset_one > content_minoffset_one) + mpm_content_minoffset_one = content_minoffset_one; + if (content_cnt) { if (mpm_content_maxlen == 0) mpm_content_maxlen = content_maxlen; if (mpm_content_maxlen > content_maxlen) mpm_content_maxlen = content_maxlen; + + if (sh->mpm_content_minlen == 0) sh->mpm_content_minlen = content_minlen; + if (sh->mpm_content_minlen > content_minlen) + sh->mpm_content_minlen = content_minlen; } if (uricontent_maxlen) { if (mpm_uricontent_maxlen == 0) mpm_uricontent_maxlen = uricontent_maxlen; if (mpm_uricontent_maxlen > uricontent_maxlen) mpm_uricontent_maxlen = uricontent_maxlen; } +//#if 0 + /* scan ctx */ + for (sm = s->match; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_CONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { + DetectContentData *cd = (DetectContentData *)sm->ctx; + if (mpm_content_maxlen == cd->content_len) { + if (cd->flags & DETECT_CONTENT_NOCASE) { + sh->mpm_scan_ctx->AddPatternNocase(sh->mpm_scan_ctx, cd->content, cd->content_len, cd->id); + } else { + sh->mpm_scan_ctx->AddPattern(sh->mpm_scan_ctx, cd->content, cd->content_len, cd->id); + } + break; /* just add one per sig, + * TODO see if we can select the best one */ + } +/* + } else if (sm->type == DETECT_URICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) { + DetectUricontentData *ud = (DetectUricontentData *)sm->ctx; + if (ud->flags & DETECT_URICONTENT_NOCASE) { + sh->mpm_uri_ctx->AddPatternNocase(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, ud->id); + } else { + sh->mpm_uri_ctx->AddPattern(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, ud->id); + } +*/ + } + } +//#endif + /* search ctx */ for (sm = s->match; sm != NULL; sm = sm->next) { if (sm->type == DETECT_CONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { DetectContentData *cd = (DetectContentData *)sm->ctx; @@ -205,11 +366,35 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) /* content */ if (sh->flags & SIG_GROUP_HAVECONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { + /* search ctx */ if (sh->mpm_ctx->Prepare != NULL) { sh->mpm_ctx->Prepare(sh->mpm_ctx); } - if (mpm_content_cnt && mpm_content_maxlen > 1) - printf("mpm_content_cnt %u, mpm_content_maxlen %d\n", mpm_content_cnt, mpm_content_maxlen); + /* scan ctx */ + if (sh->mpm_scan_ctx->Prepare != NULL) { + sh->mpm_scan_ctx->Prepare(sh->mpm_scan_ctx); + } + + if (mpm_content_cnt && mpm_content_maxlen > 1) { + //printf("mpm_content_cnt %u, mpm_content_maxlen %d\n", mpm_content_cnt, mpm_content_maxlen); + g_content_scan++; + } else { + g_content_search++; + } +// printf("(sh %p) mpm_content_cnt %u, mpm_content_maxlen %u, mpm_content_minlen %u\n", sh, mpm_content_cnt, mpm_content_maxlen, sh->mpm_content_minlen); + + if (mpm_content_maxdepth) { +// printf("mpm_content_maxdepth %u\n", mpm_content_maxdepth); + g_content_maxdepth++; + } + if (mpm_content_minoffset) { +// printf("mpm_content_minoffset %u\n", mpm_content_minoffset); + g_content_minoffset++; + } + g_content_total++; + +// if (mpm_content_depth_present) printf("(sh %p) at least one depth: %d, depth %u\n", sh, mpm_content_depth_present, mpm_content_maxdepth_one); +// if (mpm_content_offset_present) printf("(sh %p) at least one offset: %d, offset %u\n", sh, mpm_content_offset_present, mpm_content_minoffset_one); //sh->mpm_ctx->PrintCtx(sh->mpm_ctx); } @@ -218,8 +403,12 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) if (sh->mpm_uri_ctx->Prepare != NULL) { sh->mpm_uri_ctx->Prepare(sh->mpm_uri_ctx); } - if (mpm_uricontent_cnt && mpm_uricontent_maxlen > 1) - printf("mpm_uricontent_cnt %u, mpm_uricontent_maxlen %d\n", mpm_uricontent_cnt, mpm_uricontent_maxlen); + if (mpm_uricontent_cnt && mpm_uricontent_maxlen > 1) { +// printf("mpm_uricontent_cnt %u, mpm_uricontent_maxlen %d\n", mpm_uricontent_cnt, mpm_uricontent_maxlen); + g_uricontent_scan++; + } else { + g_uricontent_search++; + } //sh->mpm_uri_ctx->PrintCtx(sh->mpm_uri_ctx); } diff --git a/src/detect-engine-mpm.h b/src/detect-engine-mpm.h index 0ef791de45..ddcb58f73d 100644 --- a/src/detect-engine-mpm.h +++ b/src/detect-engine-mpm.h @@ -5,7 +5,6 @@ MpmCtx mpm_ctx[1]; u_int32_t PacketPatternMatch(ThreadVars *, PatternMatcherThread *, Packet *); -int PacketPatternScan(ThreadVars *t, Packet *p, u_int8_t mpm_instance); void PacketPatternCleanup(ThreadVars *, PatternMatcherThread *); void PatternMatchPrepare(MpmCtx *); int PatternMatchPrepareGroup(DetectEngineCtx *, SigGroupHead *); diff --git a/src/detect.c b/src/detect.c index ab4d618ca7..881094cd37 100644 --- a/src/detect.c +++ b/src/detect.c @@ -52,6 +52,7 @@ SigMatch *SigMatchAlloc(void); void SigMatchFree(SigMatch *sm); int SignatureTupleCmp(SignatureTuple *a, SignatureTuple *b); int SignatureTupleCmpRaw(DetectAddressGroup *src, DetectAddressGroup *dst, DetectPort *sp, DetectPort *dp, u_int8_t proto, SignatureTuple *b); +void DetectExitPrintStats(ThreadVars *tv, void *data); /* tm module api functions */ int Detect(ThreadVars *, Packet *, void *, PacketQueue *); @@ -62,10 +63,23 @@ void TmModuleDetectRegister (void) { tmm_modules[TMM_DETECT].name = "Detect"; tmm_modules[TMM_DETECT].Init = DetectThreadInit; tmm_modules[TMM_DETECT].Func = Detect; + tmm_modules[TMM_DETECT].ExitPrintStats = DetectExitPrintStats; tmm_modules[TMM_DETECT].Deinit = DetectThreadDeinit; tmm_modules[TMM_DETECT].RegisterTests = NULL; } +void DetectExitPrintStats(ThreadVars *tv, void *data) { + PatternMatcherThread *pmt = (PatternMatcherThread *)data; + if (pmt == NULL) + return; + + printf(" - (%s) Pkts %u, Scanned %u (%02.1f), Searched %u (%02.1f).\n", tv->name, + pmt->pkts, pmt->pkts_scanned, + (float)(pmt->pkts_scanned/(float)(pmt->pkts)*100), + pmt->pkts_searched, + (float)(pmt->pkts_searched/(float)(pmt->pkts)*100)); +} + void SigLoadSignatures (void) { Signature *prevsig = NULL, *sig; @@ -101,14 +115,14 @@ void SigLoadSignatures (void) return; prevsig->next = sig; prevsig = sig; - /* - sig = SigInit("alert udp any any -> any any (msg:\"ViCtOr nocase test\"; sid:4; rev:13; content:\"ViCtOr\"; nocase; content:\"ViCtOr\"; nocase; depth:150;)"); + sig = SigInit("alert udp any any -> any any (msg:\"ViCtOr nocase test\"; sid:4; rev:13; content:\"ViCtOr!!\"; offset:100; depth:150; nocase; content:\"ViCtOr!!\"; nocase; offset:99; depth:150;)"); if (sig == NULL) return; prevsig->next = sig; prevsig = sig; + sig = SigInit("alert ip any any -> 1.2.3.4 any (msg:\"ViCtOr case test\"; sid:2001; content:\"ViCtOr\"; depth:150;)"); if (sig == NULL) return; @@ -209,6 +223,7 @@ void SigLoadSignatures (void) //FILE *fp = fopen("/home/victor/rules/emerging-p2p.rules", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-web-small.rules", "r"); //FILE *fp = fopen("/home/victor/rules/web-misc.rules", "r"); + //FILE *fp = fopen("/home/victor/rules/imap.rules", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-malware.rules", "r"); //FILE *fp = fopen("/home/victor/rules/vips-all.sigs", "r"); //FILE *fp = fopen("/home/victor/rules/all_noip.rules", "r"); @@ -329,12 +344,15 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) u_int32_t idx,sig; SigGroupHead *sgh = NULL; + pmt->pkts++; + SigMatchIPOnlySignatures(th_v,pmt,p); /* we assume we don't have an uri when we start inspection */ pmt->de_have_httpuri = 0; pmt->de_scanned_httpuri = 0; pmt->mc = NULL; + pmt->mc_scan = NULL; pmt->mcu = NULL; /* find the right mpm instance */ @@ -345,6 +363,7 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) if (ag != NULL) { if (ag->port == NULL) { pmt->mc = ag->sh->mpm_ctx; + pmt->mc_scan = ag->sh->mpm_scan_ctx; pmt->mcu = ag->sh->mpm_uri_ctx; sgh = ag->sh; @@ -358,6 +377,7 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) DetectPort *dport = DetectPortLookupGroup(sport->dst_ph,p->dp); if (dport != NULL) { pmt->mc = dport->sh->mpm_ctx; + pmt->mc_scan = dport->sh->mpm_scan_ctx; pmt->mcu = dport->sh->mpm_uri_ctx; sgh = dport->sh; } @@ -373,42 +393,53 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) return 0; } - if (pmt->mc != NULL) { + if (p->tcp_payload_len > 0 && pmt->mc != NULL) { /* run the pattern matcher against the packet */ - //u_int32_t cnt = - PacketPatternMatch(th_v, pmt, p); - //printf("cnt %u\n", cnt); + if (sgh->mpm_content_minlen > p->tcp_payload_len) { + //printf("Not scanning as pkt payload is smaller than the min content length.\n"); + } else { + pmt->pkts_scanned++; + u_int32_t cnt = PacketPatternScan(th_v, pmt, p); + //printf("scan: cnt %u\n", cnt); + if (cnt > 0) { + pmt->pkts_searched++; + cnt += PacketPatternMatch(th_v, pmt, p); + //printf("search: cnt %u\n", cnt); + } + } } /* inspect the sigs against the packet */ for (idx = 0; idx < sgh->sig_cnt; idx++) { sig = sgh->match_array[idx]; s = g_de_ctx->sig_array[sig]; + //printf("Sig %u\n", s->id); + /* check the source & dst port in the sig */ + if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) { + if (!(s->flags & SIG_FLAG_DP_ANY)) { + DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp); + if (dport == NULL) + continue; + } + if (!(s->flags & SIG_FLAG_SP_ANY)) { + DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp); + if (sport == NULL) + continue; + } + } + /* check the source address */ if (!(s->flags & SIG_FLAG_SRC_ANY)) { DetectAddressGroup *saddr = DetectAddressLookupGroup(&s->src,&p->src); if (saddr == NULL) continue; } - + /* check the destination address */ if (!(s->flags & SIG_FLAG_DST_ANY)) { DetectAddressGroup *daddr = DetectAddressLookupGroup(&s->dst,&p->dst); if (daddr == NULL) continue; } - /* check the source port in the sig */ - if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) { - if (!(s->flags & SIG_FLAG_SP_ANY)) { - DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp); - if (sport == NULL) - continue; - } - if (!(s->flags & SIG_FLAG_DP_ANY)) { - DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp); - if (dport == NULL) - continue; - } - } /* reset pkt ptr and offset */ pmt->pkt_ptr = NULL; pmt->pkt_off = 0; @@ -460,16 +491,6 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) /* only if the last matched as well, we have a hit */ if (sm == NULL) { - if (s->id > 100) { - printf("Signature %u matched: %s, flow: toserver %s toclient %s proto %u, SP %s (%u) DP %s (%u) sig sp: ", - s->id, s->msg ? s->msg : "", - p->flowflags & FLOW_PKT_TOSERVER ? "TRUE":"FALSE", - p->flowflags & FLOW_PKT_TOCLIENT ? "TRUE":"FALSE", - p->proto, s->flags & SIG_FLAG_SP_ANY ? "ANY":"NOTANY", p->sp, - s->flags & SIG_FLAG_DP_ANY ? "ANY":"NOTANY", p->dp); - DetectPortPrint(s->sp); printf(" dp: "); - DetectPortPrint(s->dp); printf("\n"); - } fmatch = 1; if (!(s->flags & SIG_FLAG_NOALERT)) { @@ -1297,6 +1318,7 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr de_ctx->mpm_unique++; } else { sgr->sh->mpm_ctx = mpmsh->mpm_ctx; + sgr->sh->mpm_scan_ctx = mpmsh->mpm_scan_ctx; sgr->sh->flags |= SIG_GROUP_HEAD_MPM_COPY; SigGroupHeadClearContent(sgr->sh); @@ -1695,6 +1717,7 @@ static int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, De de_ctx->mpm_unique++; } else { dp->sh->mpm_ctx = mpmsh->mpm_ctx; + dp->sh->mpm_scan_ctx = mpmsh->mpm_scan_ctx; dp->sh->flags |= SIG_GROUP_HEAD_MPM_COPY; SigGroupHeadClearContent(dp->sh); @@ -1903,9 +1926,9 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) { DetectPortHashFree(); DetectPortSpHashFree(); - //DetectAddressGroupPrintMemory(); - //DetectSigGroupPrintMemory(); - //DetectPortPrintMemory(); +// DetectAddressGroupPrintMemory(); +// DetectSigGroupPrintMemory(); +// DetectPortPrintMemory(); //#endif if (!(de_ctx->flags & DE_QUIET)) { printf("* MPM memory %u (dynamic %u, ctxs %u)\n", @@ -1979,7 +2002,7 @@ void DbgPrintSigs2(SigGroupHead *sgh) { /* shortcut for debugging. If enabled Stage5 will * print sigid's for all groups */ -#define PRINTSIGS +//#define PRINTSIGS /* just printing */ int SigAddressPrepareStage5(void) { @@ -2026,7 +2049,7 @@ int SigAddressPrepareStage5(void) { DetectPort *dp = sp->dst_ph; for ( ; dp != NULL; dp = dp->next) { printf(" * Dst port(range): "); DetectPortPrint(dp); - //printf(" (sh %p)", dp->sh); + printf(" (sigs %u)", dp->sh->sig_cnt); #ifdef PRINTSIGS printf(" - "); for (i = 0; i < dp->sh->sig_cnt; i++) { @@ -2057,6 +2080,7 @@ int SigAddressPrepareStage5(void) { DetectPort *dp = sp->dst_ph; for ( ; dp != NULL; dp = dp->next) { printf(" * Dst port(range): "); DetectPortPrint(dp); + printf(" (sigs %u)", dp->sh->sig_cnt); #ifdef PRINTSIGS printf(" - "); for (i = 0; i < dp->sh->sig_cnt; i++) { @@ -2099,6 +2123,7 @@ int SigAddressPrepareStage5(void) { DetectPort *dp = sp->dst_ph; for ( ; dp != NULL; dp = dp->next) { printf(" * Dst port(range): "); DetectPortPrint(dp); + printf(" (sigs %u)", dp->sh->sig_cnt); #ifdef PRINTSIGS printf(" - "); for (i = 0; i < dp->sh->sig_cnt; i++) { @@ -2129,6 +2154,7 @@ int SigAddressPrepareStage5(void) { DetectPort *dp = sp->dst_ph; for ( ; dp != NULL; dp = dp->next) { printf(" * Dst port(range): "); DetectPortPrint(dp); + printf(" (sigs %u)", dp->sh->sig_cnt); #ifdef PRINTSIGS printf(" - "); for (i = 0; i < dp->sh->sig_cnt; i++) { @@ -2171,6 +2197,7 @@ int SigAddressPrepareStage5(void) { DetectPort *dp = sp->dst_ph; for ( ; dp != NULL; dp = dp->next) { printf(" * Dst port(range): "); DetectPortPrint(dp); + printf(" (sigs %u)", dp->sh->sig_cnt); #ifdef PRINTSIGS printf(" - "); for (i = 0; i < dp->sh->sig_cnt; i++) { @@ -2201,6 +2228,7 @@ int SigAddressPrepareStage5(void) { DetectPort *dp = sp->dst_ph; for ( ; dp != NULL; dp = dp->next) { printf(" * Dst port(range): "); DetectPortPrint(dp); + printf(" (sigs %u)", dp->sh->sig_cnt); #ifdef PRINTSIGS printf(" - "); for (i = 0; i < dp->sh->sig_cnt; i++) { @@ -2231,6 +2259,7 @@ int SigAddressPrepareStage5(void) { DetectPort *dp = sp->dst_ph; for ( ; dp != NULL; dp = dp->next) { printf(" * Dst port(range): "); DetectPortPrint(dp); + printf(" (sigs %u)", dp->sh->sig_cnt); #ifdef PRINTSIGS printf(" - "); for (i = 0; i < dp->sh->sig_cnt; i++) { @@ -2253,7 +2282,7 @@ int SigGroupBuild (DetectEngineCtx *de_ctx) { SigAddressPrepareStage2(de_ctx); SigAddressPrepareStage3(de_ctx); // SigAddressPrepareStage5(); - + DbgPrintScanSearchStats(); // DetectAddressGroupPrintMemory(); // DetectSigGroupPrintMemory(); // DetectPortPrintMemory(); @@ -2846,6 +2875,261 @@ end: return result; } +int SigTest10 (void) { + u_int8_t *buf = (u_int8_t *) + "ABC"; + u_int16_t buflen = strlen((char *)buf); + Packet p; + ThreadVars th_v; + PatternMatcherThread *pmt; + int result = 0; + + memset(&th_v, 0, sizeof(th_v)); + memset(&p, 0, sizeof(p)); + p.src.family = AF_INET; + p.dst.family = AF_INET; + p.tcp_payload = buf; + p.tcp_payload_len = buflen; + p.proto = IPPROTO_TCP; + + g_de_ctx = DetectEngineCtxInit(); + if (g_de_ctx == NULL) { + goto end; + } + + g_de_ctx->flags |= DE_QUIET; + + g_de_ctx->sig_list = SigInit("alert tcp any any -> any any (msg:\"Long content test (1)\"; content:\"ABCD\"; depth:4; sid:1;)"); + if (g_de_ctx->sig_list == NULL) { + result = 0; + goto end; + } + g_de_ctx->sig_list->next = SigInit("alert tcp any any -> any any (msg:\"Long content test (2)\"; content:\"VWXYZ\"; sid:2;)"); + if (g_de_ctx->sig_list->next == NULL) { + result = 0; + goto end; + } + + SigGroupBuild(g_de_ctx); + PatternMatchPrepare(mpm_ctx); + PatternMatcherThreadInit(&th_v, (void *)&pmt); + + SigMatchSignatures(&th_v, pmt, &p); + if (PacketAlertCheck(&p, 1) && PacketAlertCheck(&p, 2)) + result = 0; + else + result = 1; + + SigGroupCleanup(); + SigCleanSignatures(); + PatternMatcherThreadDeinit(&th_v, (void *)pmt); + PatternMatchDestroy(mpm_ctx); + DetectEngineCtxFree(g_de_ctx); +end: + return result; +} + +int SigTest11 (void) { + u_int8_t *buf = (u_int8_t *) + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+"; + u_int16_t buflen = strlen((char *)buf); + Packet p; + ThreadVars th_v; + PatternMatcherThread *pmt; + int result = 0; + + memset(&th_v, 0, sizeof(th_v)); + memset(&p, 0, sizeof(p)); + p.src.family = AF_INET; + p.dst.family = AF_INET; + p.tcp_payload = buf; + p.tcp_payload_len = buflen; + p.proto = IPPROTO_TCP; + + g_de_ctx = DetectEngineCtxInit(); + if (g_de_ctx == NULL) { + goto end; + } + + g_de_ctx->flags |= DE_QUIET; + + g_de_ctx->sig_list = SigInit("alert tcp any any -> any any (msg:\"Scan vs Search (1)\"; content:\"ABCDEFGHIJ\"; content:\"klmnop\"; content:\"1234\"; sid:1;)"); + if (g_de_ctx->sig_list == NULL) { + result = 0; + goto end; + } + g_de_ctx->sig_list->next = SigInit("alert tcp any any -> any any (msg:\"Scan vs Search (2)\"; content:\"VWXYZabcde\"; content:\"5678\"; content:\"89\"; sid:2;)"); + if (g_de_ctx->sig_list->next == NULL) { + result = 0; + goto end; + } + + SigGroupBuild(g_de_ctx); + PatternMatchPrepare(mpm_ctx); + PatternMatcherThreadInit(&th_v, (void *)&pmt); + + SigMatchSignatures(&th_v, pmt, &p); + if (PacketAlertCheck(&p, 1) && PacketAlertCheck(&p, 2)) + result = 1; + else + result = 0; + + SigGroupCleanup(); + SigCleanSignatures(); + PatternMatcherThreadDeinit(&th_v, (void *)pmt); + PatternMatchDestroy(mpm_ctx); + DetectEngineCtxFree(g_de_ctx); +end: + return result; +} + +int SigTest12 (void) { + u_int8_t *buf = (u_int8_t *) + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+"; + u_int16_t buflen = strlen((char *)buf); + Packet p; + ThreadVars th_v; + PatternMatcherThread *pmt; + int result = 0; + + memset(&th_v, 0, sizeof(th_v)); + memset(&p, 0, sizeof(p)); + p.src.family = AF_INET; + p.dst.family = AF_INET; + p.tcp_payload = buf; + p.tcp_payload_len = buflen; + p.proto = IPPROTO_TCP; + + g_de_ctx = DetectEngineCtxInit(); + if (g_de_ctx == NULL) { + goto end; + } + + g_de_ctx->flags |= DE_QUIET; + + g_de_ctx->sig_list = SigInit("alert tcp any any -> any any (msg:\"Content order test\"; content:\"ABCDEFGHIJ\"; content:\"klmnop\"; content:\"1234\"; sid:1;)"); + if (g_de_ctx->sig_list == NULL) { + result = 0; + goto end; + } + + SigGroupBuild(g_de_ctx); + PatternMatchPrepare(mpm_ctx); + PatternMatcherThreadInit(&th_v, (void *)&pmt); + + SigMatchSignatures(&th_v, pmt, &p); + if (PacketAlertCheck(&p, 1)) + result = 1; + else + result = 0; + + SigGroupCleanup(); + SigCleanSignatures(); + PatternMatcherThreadDeinit(&th_v, (void *)pmt); + PatternMatchDestroy(mpm_ctx); + DetectEngineCtxFree(g_de_ctx); +end: + return result; +} + +int SigTest13 (void) { + u_int8_t *buf = (u_int8_t *) + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+"; + u_int16_t buflen = strlen((char *)buf); + Packet p; + ThreadVars th_v; + PatternMatcherThread *pmt; + int result = 0; + + memset(&th_v, 0, sizeof(th_v)); + memset(&p, 0, sizeof(p)); + p.src.family = AF_INET; + p.dst.family = AF_INET; + p.tcp_payload = buf; + p.tcp_payload_len = buflen; + p.proto = IPPROTO_TCP; + + g_de_ctx = DetectEngineCtxInit(); + if (g_de_ctx == NULL) { + goto end; + } + + g_de_ctx->flags |= DE_QUIET; + + g_de_ctx->sig_list = SigInit("alert tcp any any -> any any (msg:\"Content order test\"; content:\"ABCDEFGHIJ\"; content:\"1234\"; content:\"klmnop\"; sid:1;)"); + if (g_de_ctx->sig_list == NULL) { + result = 0; + goto end; + } + + SigGroupBuild(g_de_ctx); + PatternMatchPrepare(mpm_ctx); + PatternMatcherThreadInit(&th_v, (void *)&pmt); + + SigMatchSignatures(&th_v, pmt, &p); + if (PacketAlertCheck(&p, 1)) + result = 1; + else + result = 0; + + SigGroupCleanup(); + SigCleanSignatures(); + PatternMatcherThreadDeinit(&th_v, (void *)pmt); + PatternMatchDestroy(mpm_ctx); + DetectEngineCtxFree(g_de_ctx); +end: + return result; +} + +int SigTest14 (void) { + u_int8_t *buf = (u_int8_t *) + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()_+"; + u_int16_t buflen = strlen((char *)buf); + Packet p; + ThreadVars th_v; + PatternMatcherThread *pmt; + int result = 0; + + memset(&th_v, 0, sizeof(th_v)); + memset(&p, 0, sizeof(p)); + p.src.family = AF_INET; + p.dst.family = AF_INET; + p.tcp_payload = buf; + p.tcp_payload_len = buflen; + p.proto = IPPROTO_TCP; + + g_de_ctx = DetectEngineCtxInit(); + if (g_de_ctx == NULL) { + goto end; + } + + g_de_ctx->flags |= DE_QUIET; + + g_de_ctx->sig_list = SigInit("alert tcp any any -> any any (msg:\"Content order test\"; content:\"ABCDEFGHIJ\"; content:\"1234\"; content:\"klmnop\"; distance:0; sid:1;)"); + if (g_de_ctx->sig_list == NULL) { + result = 0; + goto end; + } + + SigGroupBuild(g_de_ctx); + PatternMatchPrepare(mpm_ctx); + PatternMatcherThreadInit(&th_v, (void *)&pmt); + + SigMatchSignatures(&th_v, pmt, &p); + if (PacketAlertCheck(&p, 1)) + result = 0; + else + result = 1; + + SigGroupCleanup(); + SigCleanSignatures(); + PatternMatcherThreadDeinit(&th_v, (void *)pmt); + PatternMatchDestroy(mpm_ctx); + DetectEngineCtxFree(g_de_ctx); +end: + return result; +} + void SigRegisterTests(void) { SigParseRegisterTests(); UtRegisterTest("SigTest01 -- HTTP URI cap", SigTest01, 1); @@ -2857,5 +3141,10 @@ void SigRegisterTests(void) { UtRegisterTest("SigTest07 -- uricontent HTTP/1.1 mismatch test", SigTest07, 1); UtRegisterTest("SigTest08 -- uricontent HTTP/1.0 match test", SigTest08, 1); UtRegisterTest("SigTest09 -- uricontent HTTP/1.0 mismatch test", SigTest09, 1); + UtRegisterTest("SigTest10 -- long content match, longer than pkt", SigTest10, 1); + UtRegisterTest("SigTest11 -- scan vs search", SigTest11, 1); + UtRegisterTest("SigTest12 -- content order matching, normal", SigTest12, 1); + UtRegisterTest("SigTest13 -- content order matching, diff order", SigTest13, 1); + UtRegisterTest("SigTest14 -- content order matching, distance 0", SigTest14, 1); } diff --git a/src/detect.h b/src/detect.h index 5bb121e954..0620c1c2e1 100644 --- a/src/detect.h +++ b/src/detect.h @@ -36,10 +36,16 @@ typedef struct _PatternMatcherThread { * * XXX rename to mpm_ctx as soon as the threading * thing above is renamed as well */ - MpmCtx *mc; + MpmCtx *mc; /* search ctx */ + MpmCtx *mc_scan; /* scan ctx */ MpmCtx *mcu; + //MpmCtx *mcu_scan; MpmThreadCtx mtc; MpmThreadCtx mtcu; + + u_int32_t pkts; + u_int32_t pkts_scanned; + u_int32_t pkts_searched; } PatternMatcherThread; typedef struct _Signature { @@ -158,8 +164,11 @@ typedef struct _SigGroupHead { u_int8_t flags; /* pattern matcher instance */ - MpmCtx *mpm_ctx; + MpmCtx *mpm_ctx; /* search */ + MpmCtx *mpm_scan_ctx; /* scan */ + u_int16_t mpm_content_minlen; MpmCtx *mpm_uri_ctx; + u_int16_t mpm_uricontent_minlen; /* number of sigs in this head */ u_int32_t sig_cnt;