From 0219b767b81dbf73d658c38deebc9d752b6714dc Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sat, 17 Jul 2010 18:38:58 -0700 Subject: [PATCH] Fix a content pattern matching bug related to signature grouping and mpm_ctx sharing. In certain conditions (signature combinations) the mpm_stream_ctx (the ctx that handles stream pattern scanning) wasn't properly setup. --- src/detect-engine-mpm.c | 211 ++++++++++++++++++++++++++--------- src/detect-engine-siggroup.c | 30 +++-- src/detect.c | 31 ++++- 3 files changed, 209 insertions(+), 63 deletions(-) diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index fb123037ae..cbef584e0a 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -75,6 +75,87 @@ SCEnumCharMap sc_mpm_algo_map[] = { #endif }; +/** + * \brief check if a signature has patterns that are to be inspected + * against a packets payload (as opposed to the stream payload) + * + * \param s signature + * + * \retval 1 true + * \retval 0 false + */ +int SignatureHasPacketContent(Signature *s) { + SCEnter(); + + if (s == NULL) { + SCReturnInt(0); + } + + if (!(s->flags & SIG_FLAG_MPM)) { + SCLogDebug("no mpm"); + SCReturnInt(0); + } + + if (s->alproto != ALPROTO_UNKNOWN) { + SCLogDebug("inspecting app layer"); + SCReturnInt(0); + } + + SigMatch *sm = s->pmatch; + if (sm == NULL) { + SCReturnInt(0); + } + + for ( ;sm != NULL; sm = sm->next) { + if (sm->type == DETECT_CONTENT) { + SCReturnInt(1); + } + } + + SCReturnInt(0); +} + +/** + * \brief check if a signature has patterns that are to be inspected + * against the stream payload (as opposed to the individual packets + * payload(s)) + * + * \param s signature + * + * \retval 1 true + * \retval 0 false + */ +int SignatureHasStreamContent(Signature *s) { + SCEnter(); + + if (s == NULL) { + SCReturnInt(0); + } + + if (!(s->flags & SIG_FLAG_MPM)) { + SCLogDebug("no mpm"); + SCReturnInt(0); + } + + if (s->flags & SIG_FLAG_DSIZE) { + SCLogDebug("dsize"); + SCReturnInt(0); + } + + SigMatch *sm = s->pmatch; + if (sm == NULL) { + SCReturnInt(0); + } + + for ( ;sm != NULL; sm = sm->next) { + if (sm->type == DETECT_CONTENT) { + SCReturnInt(1); + } + } + + SCReturnInt(0); +} + /** \brief Function to return the default multi pattern matcher algorithm to be * used by the engine @@ -292,15 +373,18 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) { } /* stream content */ - if (sh->flags & SIG_GROUP_HAVESTREAMCONTENT && sh->mpm_stream_ctx != NULL && - !(sh->flags & SIG_GROUP_HEAD_MPM_STREAM_COPY)) { - SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx, sh); - mpm_table[sh->mpm_stream_ctx->mpm_type].DestroyCtx(sh->mpm_stream_ctx); - SCFree(sh->mpm_stream_ctx); - - /* ready for reuse */ - sh->mpm_stream_ctx = NULL; - sh->flags &= ~SIG_GROUP_HAVESTREAMCONTENT; + if (sh->flags & SIG_GROUP_HAVESTREAMCONTENT) { + if (sh->mpm_stream_ctx != NULL) { + if (!(sh->flags & SIG_GROUP_HEAD_MPM_STREAM_COPY)) { + SCLogDebug("destroying mpm_stream_ctx %p (sh %p)", sh->mpm_stream_ctx, sh); + mpm_table[sh->mpm_stream_ctx->mpm_type].DestroyCtx(sh->mpm_stream_ctx); + SCFree(sh->mpm_stream_ctx); + + /* ready for reuse */ + sh->mpm_stream_ctx = NULL; + sh->flags &= ~SIG_GROUP_HAVESTREAMCONTENT; + } + } } } @@ -628,35 +712,34 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead } } - if (s->flags & SIG_FLAG_DSIZE) { - scan_packet = 1; - } else if (s->alproto == ALPROTO_UNKNOWN) { - scan_packet = 1; - scan_stream = 1; - } else { - scan_stream = 1; - } + scan_packet = SignatureHasPacketContent(s); + scan_stream = SignatureHasStreamContent(s); if (scan_packet) { - /* add the content to the "packet" mpm */ - if (co->flags & DETECT_CONTENT_NOCASE) { - mpm_table[sgh->mpm_ctx->mpm_type].AddPatternNocase(sgh->mpm_ctx, - co->content, co->content_len, offset, depth, co->id, - s->num, flags); - } else { - mpm_table[sgh->mpm_ctx->mpm_type].AddPattern(sgh->mpm_ctx, - co->content, co->content_len, offset, depth, co->id, - s->num, flags); + if (sgh->flags & SIG_GROUP_HAVECONTENT && !(sgh->flags & SIG_GROUP_HEAD_MPM_COPY)) { + /* add the content to the "packet" mpm */ + if (co->flags & DETECT_CONTENT_NOCASE) { + mpm_table[sgh->mpm_ctx->mpm_type].AddPatternNocase(sgh->mpm_ctx, + co->content, co->content_len, offset, depth, co->id, + s->num, flags); + } else { + mpm_table[sgh->mpm_ctx->mpm_type].AddPattern(sgh->mpm_ctx, + co->content, co->content_len, offset, depth, co->id, + s->num, flags); + } } } if (scan_stream) { - /* add the content to the "stream" mpm */ - if (co->flags & DETECT_CONTENT_NOCASE) { - mpm_table[sgh->mpm_stream_ctx->mpm_type].AddPatternNocase(sgh->mpm_stream_ctx, - co->content, co->content_len, offset, depth, co->id, s->num, flags); - } else { - mpm_table[sgh->mpm_stream_ctx->mpm_type].AddPattern(sgh->mpm_stream_ctx, - co->content, co->content_len, offset, depth, co->id, s->num, flags); + if (sgh->flags & SIG_GROUP_HAVESTREAMCONTENT && !(sgh->flags & SIG_GROUP_HEAD_MPM_STREAM_COPY)) { + SCLogDebug("mpm_stream_ctx %p", sgh->mpm_stream_ctx); + /* add the content to the "stream" mpm */ + if (co->flags & DETECT_CONTENT_NOCASE) { + mpm_table[sgh->mpm_stream_ctx->mpm_type].AddPatternNocase(sgh->mpm_stream_ctx, + co->content, co->content_len, offset, depth, co->id, s->num, flags); + } else { + mpm_table[sgh->mpm_stream_ctx->mpm_type].AddPattern(sgh->mpm_stream_ctx, + co->content, co->content_len, offset, depth, co->id, s->num, flags); + } } } @@ -694,7 +777,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) { Signature *s = NULL; SigMatch *sm = NULL; - uint32_t co_cnt = 0; + uint32_t has_co_packet = 0; /**< our sgh has packet payload inspecting content */ + uint32_t has_co_stream = 0; /**< our shg has stream inspecting content */ uint32_t ur_cnt = 0; uint32_t cnt = 0; uint32_t sig = 0; @@ -717,12 +801,11 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) if (s == NULL) continue; - /* find flow setting of this rule */ - for (sm = s->pmatch; sm != NULL; sm = sm->next) { - if (sm->type == DETECT_CONTENT) { - co_cnt++; - s->flags |= SIG_FLAG_MPM; - } + if (SignatureHasPacketContent(s) == 1) { + has_co_packet = 1; + } + if (SignatureHasStreamContent(s) == 1) { + has_co_stream = 1; } for (sm = s->umatch; sm != NULL; sm = sm->next) { @@ -733,9 +816,12 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) } } - if (co_cnt > 0) { + if (has_co_packet > 0) { sh->flags |= SIG_GROUP_HAVECONTENT; } + if (has_co_stream > 0) { + sh->flags |= SIG_GROUP_HAVESTREAMCONTENT; + } if (ur_cnt > 0) { sh->flags |= SIG_GROUP_HAVEURICONTENT; } @@ -753,20 +839,21 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) #else MpmInitCtx(sh->mpm_ctx, de_ctx->mpm_matcher, de_ctx->cuda_rc_mod_handle); #endif - //if (sh->flags & SIG_GROUP_HAVESTREAMCONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_STREAM_COPY)) { - sh->mpm_stream_ctx = SCMalloc(sizeof(MpmCtx)); - if (sh->mpm_stream_ctx == NULL) - goto error; + } + + if (sh->flags & SIG_GROUP_HAVESTREAMCONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_STREAM_COPY)) { + sh->mpm_stream_ctx = SCMalloc(sizeof(MpmCtx)); + if (sh->mpm_stream_ctx == NULL) + goto error; - memset(sh->mpm_stream_ctx, 0x00, sizeof(MpmCtx)); + memset(sh->mpm_stream_ctx, 0x00, sizeof(MpmCtx)); #ifndef __SC_CUDA_SUPPORT__ - MpmInitCtx(sh->mpm_stream_ctx, de_ctx->mpm_matcher, -1); + MpmInitCtx(sh->mpm_stream_ctx, de_ctx->mpm_matcher, -1); #else - MpmInitCtx(sh->mpm_stream_ctx, de_ctx->mpm_matcher, de_ctx->cuda_rc_mod_handle); + MpmInitCtx(sh->mpm_stream_ctx, de_ctx->mpm_matcher, de_ctx->cuda_rc_mod_handle); #endif - //} - } + if (sh->flags & SIG_GROUP_HAVEURICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) { sh->mpm_uri_ctx = SCMalloc(sizeof(MpmCtx)); if (sh->mpm_uri_ctx == NULL) @@ -991,14 +1078,34 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) } } + /* load the patterns */ + PatternMatchPreprarePopulateMpm(de_ctx, sh); + /* content */ if (sh->flags & SIG_GROUP_HAVECONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) { - /* load the patterns */ - PatternMatchPreprarePopulateMpm(de_ctx, sh); if (mpm_table[sh->mpm_ctx->mpm_type].Prepare != NULL) { mpm_table[sh->mpm_ctx->mpm_type].Prepare(sh->mpm_ctx); } + + if (mpm_content_maxdepth) { + // printf("mpm_content_maxdepth %" PRIu32 "\n", mpm_content_maxdepth); + g_content_maxdepth++; + } + if (mpm_content_minoffset) { + // printf("mpm_content_minoffset %" PRIu32 "\n", mpm_content_minoffset); + g_content_minoffset++; + } + g_content_total++; + + //if (mpm_content_depth_present) printf("(sh %p) at least one depth: %" PRId32 ", depth %" PRIu32 "\n", sh, mpm_content_depth_present, mpm_content_maxdepth_one); + //if (mpm_content_offset_present) printf("(sh %p) at least one offset: %" PRId32 ", offset %" PRIu32 "\n", sh, mpm_content_offset_present, mpm_content_minoffset_one); + //sh->mpm_ctx->PrintCtx(sh->mpm_ctx); + } + + /* content */ + if (sh->flags & SIG_GROUP_HAVESTREAMCONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_STREAM_COPY)) { + SCLogDebug("preparing mpm_stream_ctx %p", sh->mpm_stream_ctx); if (mpm_table[sh->mpm_stream_ctx->mpm_type].Prepare != NULL) { mpm_table[sh->mpm_stream_ctx->mpm_type].Prepare(sh->mpm_stream_ctx); } diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 0a0cc790a7..7a88503b75 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -1441,36 +1441,48 @@ int SigGroupHeadClearUricontent(SigGroupHead *sh) */ int SigGroupHeadLoadStreamContent(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { + SCEnter(); + Signature *s = NULL; SigMatch *sm = NULL; uint32_t sig = 0; DetectContentData *co = NULL; - if (sgh == NULL) - return 0; + if (sgh == NULL) { + SCReturnInt(0); + } - if (DetectContentMaxId(de_ctx) == 0) - return 0; + if (DetectContentMaxId(de_ctx) == 0) { + SCReturnInt(0); + } BUG_ON(sgh->init == NULL); sgh->init->stream_content_size = (DetectContentMaxId(de_ctx) / 8) + 1; sgh->init->stream_content_array = SCMalloc(sgh->init->stream_content_size); - if (sgh->init->stream_content_array == NULL) - return -1; + if (sgh->init->stream_content_array == NULL) { + SCReturnInt(-1); + } memset(sgh->init->stream_content_array,0, sgh->init->stream_content_size); for (sig = 0; sig < sgh->sig_cnt; sig++) { s = sgh->match_array[sig]; + + SCLogDebug("s %"PRIu32, s->id); + if (s == NULL) continue; - if (!(s->flags & SIG_FLAG_MPM)) + if (!(s->flags & SIG_FLAG_MPM)) { + SCLogDebug("no mpm"); continue; + } - if (s->flags & SIG_FLAG_DSIZE) + if (s->flags & SIG_FLAG_DSIZE) { + SCLogDebug("dsize"); continue; + } sm = s->pmatch; if (sm == NULL) @@ -1485,7 +1497,7 @@ int SigGroupHeadLoadStreamContent(DetectEngineCtx *de_ctx, SigGroupHead *sgh) } } - return 0; + SCReturnInt(0); } /** diff --git a/src/detect.c b/src/detect.c index 5923331c09..e1cd46ea36 100644 --- a/src/detect.c +++ b/src/detect.c @@ -777,10 +777,15 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh /* have a look at the reassembled stream (if any) */ if (p->flowflags & FLOW_PKT_ESTABLISHED) { + SCLogDebug("p->flowflags & FLOW_PKT_ESTABLISHED"); if (smsg != NULL && det_ctx->sgh->mpm_stream_ctx != NULL) { cnt = StreamPatternSearch(th_v, det_ctx, p, smsg, flags); SCLogDebug("cnt %u", cnt); + } else { + SCLogDebug("smsg NULL (%p) or det_ctx->sgh->mpm_stream_ctx NULL (%p)", smsg, det_ctx->sgh->mpm_stream_ctx); } + } else { + SCLogDebug("NOT p->flowflags & FLOW_PKT_ESTABLISHED"); } if (p->payload_len > 0 && det_ctx->sgh->mpm_ctx != NULL && @@ -2473,7 +2478,7 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd } else { /* XXX write dedicated function for this */ dp->sh->mpm_ctx = mpmsh->mpm_ctx; - //SCLogDebug("replacing dp->sh, so setting mpm_content_maxlen to %u", mpmsh->mpm_content_maxlen); + //SCLogDebug("replacing dp->sh, so setting mpm_content_maxlen to %u (was %u)", mpmsh->mpm_content_maxlen, dp->sh->mpm_content_maxlen); //dp->sh->mpm_content_maxlen = mpmsh->mpm_content_maxlen; dp->sh->flags |= SIG_GROUP_HEAD_MPM_COPY; SigGroupHeadClearContent(dp->sh); @@ -2482,6 +2487,27 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd } } + /* content */ + SigGroupHeadLoadStreamContent(de_ctx, dp->sh); + if (dp->sh->init->stream_content_size == 0) { + de_ctx->mpm_none++; + } else { + /* now have a look if we can reuse a mpm ctx */ + SigGroupHead *mpmsh = SigGroupHeadMpmStreamHashLookup(de_ctx, dp->sh); + if (mpmsh == NULL) { + SigGroupHeadMpmStreamHashAdd(de_ctx, dp->sh); + + de_ctx->mpm_unique++; + } else { + SCLogDebug("replacing mpm_stream_ctx %p by %p", dp->sh->mpm_stream_ctx, mpmsh->mpm_stream_ctx); + dp->sh->mpm_stream_ctx = mpmsh->mpm_stream_ctx; + dp->sh->flags |= SIG_GROUP_HEAD_MPM_STREAM_COPY; + SigGroupHeadClearStreamContent(dp->sh); + + de_ctx->mpm_reuse++; + } + } + SigGroupHeadLoadUricontent(de_ctx, dp->sh); if (dp->sh->init->uri_content_size == 0) { de_ctx->mpm_uri_none++; @@ -2834,7 +2860,7 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx) { for (f = 0; f < FLOW_STATES; f++) { printf("\n"); for (proto = 0; proto < 256; proto++) { - if (proto != 0) + if (proto != 6) continue; for (global_src_gr = de_ctx->flow_gh[f].src_gh[proto]->ipv4_head; global_src_gr != NULL; @@ -2898,6 +2924,7 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx) { for ( ; dp != NULL; dp = dp->next) { printf(" 4 Dst port(range): "); DetectPortPrint(dp); printf(" (sigs %" PRIu32 ", sgh %p, maxlen %" PRIu32 ")", dp->sh->sig_cnt, dp->sh, dp->sh->mpm_content_maxlen); + printf(" mpm_ctx %p, mpm_stream_ctx %p", dp->sh->mpm_ctx, dp->sh->mpm_stream_ctx); #ifdef PRINTSIGS printf(" - "); for (u = 0; u < dp->sh->sig_cnt; u++) {