Remove more scan references.

remotes/origin/master-1.0.x
Victor Julien 16 years ago
parent bee4e04664
commit ec47f840f3

@ -293,10 +293,10 @@ uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx
SCReturnUInt(ALPROTO_UNKNOWN);
}
/* see if we can limit the data we scan */
uint16_t scanlen = buflen;
if (scanlen > dir->max_depth)
scanlen = dir->max_depth;
/* see if we can limit the data we inspect */
uint16_t searchlen = buflen;
if (searchlen > dir->max_depth)
searchlen = dir->max_depth;
uint16_t proto = ALPROTO_UNKNOWN;
uint32_t cnt = 0;
@ -304,11 +304,13 @@ uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx
cnt = mpm_table[dir->mpm_ctx.mpm_type].Search(&dir->mpm_ctx,
&tdir->mpm_ctx,
&tdir->pmq, buf,
scanlen);
searchlen);
#else
Packet *p = SCMalloc(sizeof(Packet));
if (p == NULL) goto end;
if (p == NULL)
goto end;
memset(p, 0, sizeof(Packet));
p->cuda_done = 0;
p->cuda_free_packet = 1;
p->cuda_search = 0;
@ -316,7 +318,7 @@ uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx
p->cuda_mtc = &tdir->mpm_ctx;
p->cuda_pmq = &tdir->pmq;
p->payload = buf;
p->payload_len = scanlen;
p->payload_len = searchlen;
B2gCudaPushPacketTo_tv_CMB2_APC(p);
SCMutexLock(&p->cuda_mutex_q);
SCondWait(&p->cuda_cond_q, &p->cuda_mutex_q);
@ -324,7 +326,7 @@ uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx
SCMutexUnlock(&p->cuda_mutex_q);
cnt = p->cuda_matches;
#endif
SCLogDebug("scan cnt %" PRIu32 "", cnt);
SCLogDebug("search cnt %" PRIu32 "", cnt);
if (cnt == 0) {
proto = ALPROTO_UNKNOWN;
goto end;

@ -68,13 +68,15 @@ uint16_t PatternMatchDefaultMatcher(void) {
return mpm_algo_val;
}
/** \brief Pattern match, scan part -- searches for only 'scan' patterns,
* normally one per signature.
/** \brief Pattern match -- searches for only one pattern per signature.
*
* \param tv threadvars
* \param det_ctx detection engine thread ctx
* \param p packet to scan
* \param p packet to inspect
*
* \retval ret number of matches
*/
uint32_t PacketPatternScan(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
uint32_t PacketPatternSearch(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
Packet *p)
{
SCEnter();
@ -105,13 +107,15 @@ uint32_t PacketPatternScan(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
SCReturnInt(ret);
}
/** \brief Uri Pattern match, scan part -- searches for only 'scan' patterns,
* normally one per signature.
/** \brief Uri Pattern match -- searches for one pattern per signature.
*
* \param tv threadvars
* \param det_ctx detection engine thread ctx
* \param p packet to scan
* \param p packet to inspect
*
* \retval ret number of matches
*/
uint32_t UriPatternScan(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
uint32_t UriPatternSearch(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
uint8_t *uri, uint16_t uri_len)
{
SCEnter();
@ -209,7 +213,6 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) {
}
}
static int g_uricontent_scan = 0;
static int g_uricontent_search = 0;
static int g_content_maxdepth = 0;
static int g_content_minoffset = 0;
@ -224,10 +227,9 @@ static int g_content_sigcnt4 = 0;
static int g_content_sigcnt5 = 0;
static int g_content_sigcnt10= 0;
void DbgPrintScanSearchStats() {
void DbgPrintSearchStats() {
#if 0
printf(" - MPM: scan %" PRId32 ", search %" PRId32 " (%02.1f%%) :\n", g_content_scan, g_content_search,
(float)(g_content_scan/(float)(g_content_scan+g_content_search))*100);
printf(" - MPM: search %" PRId32 "\n", g_content_search);
printf(" - MPM: maxdepth %" PRId32 ", total %" PRId32 " (%02.1f%%) :\n", g_content_maxdepth, g_content_total,
(float)(g_content_maxdepth/(float)(g_content_total))*100);
printf(" - MPM: minoffset %" PRId32 ", total %" PRId32 " (%02.1f%%) :\n", g_content_minoffset, g_content_total,
@ -249,7 +251,7 @@ typedef struct ContentHash_ {
uint16_t cnt;
uint8_t use; /* use no matter what */
uint8_t nosearch; /* single match, no search after
* scan match (for this pattern) */
* mpm match (for this pattern) */
} ContentHash;
uint32_t ContentHashFunc(HashTable *ht, void *data, uint16_t datalen) {
@ -297,26 +299,31 @@ void ContentHashFree(void *ch) {
/** \brief Predict a strength value for patterns
*
* Patterns with high character diversity score higher.
* Alpha chars score not so high
* Other printable + a few common codes a little higher
* Everything else highest.
* Longer patterns score better than short patters.
* Patterns with high character diversity score higher.
* Alpha chars score not so high
* Other printable + a few common codes a little higher
* Everything else highest.
* Longer patterns score better than short patters.
*
* \param pat pattern
* \param patlen length of the patternn
*
* \retval s pattern score
*/
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen, uint16_t len) {
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen) {
uint8_t a[256];
memset(&a,0,sizeof(a));
memset(&a, 0 ,sizeof(a));
uint32_t s = 0;
uint16_t u = 0;
for (u = 0; u < patlen; u++) {
if (a[pat[u]] == 0) {
if (isalpha(pat[u]))
s+=3;
s += 3;
else if (isprint(pat[u]) || pat[u] == 0x00 || pat[u] == 0x01 || pat[u] == 0xFF)
s+=4;
s += 4;
else
s+=6;//5
s += 6;
a[pat[u]] = 1;
} else {
@ -414,7 +421,9 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
goto error;
if (cnt == 1) {
SCLogDebug("sig has just one pattern, so we know we will use it in the scan phase and no searching will be necessary.");
SCLogDebug("sig has just one pattern, so we know we will "
"use it in the mpm phase and no searching will "
"be necessary.");
ch->nosearch = 1;
ch->use = 1;
}
@ -443,7 +452,7 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
}
}
/* now determine which one to add to the scan phase */
/* now determine which one to add to the mpm phase */
for (sig = 0; sig < sgh->sig_cnt; sig++) {
uint32_t num = sgh->match_array[sig];
@ -451,7 +460,7 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
if (s == NULL)
continue;
ContentHash *scan_ch = NULL;
ContentHash *mpm_ch = NULL;
SigMatch *sm = s->pmatch;
for ( ; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_CONTENT) {
@ -480,59 +489,37 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
SCLogDebug("lookup_ch->use %u, cnt %u", lookup_ch->use, lookup_ch->cnt);
if (scan_ch == NULL) {
SCLogDebug("scan_ch == NULL, so selecting lookup_ch->ptr->id %"PRIu32"", lookup_ch->ptr->id);
scan_ch = lookup_ch;
if (mpm_ch == NULL) {
SCLogDebug("mpm_ch == NULL, so selecting lookup_ch->ptr->id %"PRIu32"", lookup_ch->ptr->id);
mpm_ch = lookup_ch;
} else {
//if (lookup_ch->use == 0) {
uint32_t ls = PatternStrength(lookup_ch->ptr->content,lookup_ch->ptr->content_len,sgh->mpm_content_maxlen);
uint32_t ss = PatternStrength(scan_ch->ptr->content,scan_ch->ptr->content_len,sgh->mpm_content_maxlen);
if (ls > ss) {
SCLogDebug("lookup_ch->ptr->id %"PRIu32" selected over %"PRIu32"", lookup_ch->ptr->id, scan_ch->ptr->id);
scan_ch = lookup_ch;
}
else if (ls == ss) {
/* if 2 patterns are of equal strength, we pick the longest */
if (lookup_ch->ptr->content_len > scan_ch->ptr->content_len) {
SCLogDebug("lookup_ch->ptr->id %"PRIu32" selected over %"PRIu32" as the first is longer", lookup_ch->ptr->id, scan_ch->ptr->id);
scan_ch = lookup_ch;
}
} else {
SCLogDebug("sticking with scan_ch");
uint32_t ls = PatternStrength(lookup_ch->ptr->content,lookup_ch->ptr->content_len);
uint32_t ss = PatternStrength(mpm_ch->ptr->content,mpm_ch->ptr->content_len);
if (ls > ss) {
SCLogDebug("lookup_ch->ptr->id %"PRIu32" selected over %"PRIu32"", lookup_ch->ptr->id, mpm_ch->ptr->id);
mpm_ch = lookup_ch;
}
else if (ls == ss) {
/* if 2 patterns are of equal strength, we pick the longest */
if (lookup_ch->ptr->content_len > mpm_ch->ptr->content_len) {
SCLogDebug("lookup_ch->ptr->id %"PRIu32" selected over %"PRIu32" as the first is longer", lookup_ch->ptr->id, mpm_ch->ptr->id);
mpm_ch = lookup_ch;
}
#if 0
} else {
if (scan_ch->use == 0)
scan_ch = lookup_ch;
else {
uint32_t ls = PatternStrength(lookup_ch->ptr->content,lookup_ch->ptr->content_len,sgh->mpm_content_maxlen);
uint32_t ss = PatternStrength(scan_ch->ptr->content,scan_ch->ptr->content_len,sgh->mpm_content_maxlen);
if (ls > ss) {
SCLogDebug("lookup_ch->ptr->id %"PRIu32" selected over %"PRIu32"", lookup_ch->ptr->id, scan_ch->ptr->id);
scan_ch = lookup_ch;
}
/* if 2 patterns are of equal strength, we pick the longest */
else if (ls == ss) {
if (lookup_ch->ptr->content_len > scan_ch->ptr->content_len) {
SCLogDebug("lookup_ch->ptr->id %"PRIu32" selected over %"PRIu32" as the first is longer", lookup_ch->ptr->id, scan_ch->ptr->id);
scan_ch = lookup_ch;
}
}
}
SCLogDebug("sticking with mpm_ch");
}
#endif
}
ContentHashFree(ch);
}
}
/* now add the scan_ch to the mpm ctx */
if (scan_ch != NULL) {
DetectContentData *co = scan_ch->ptr;
/* now add the mpm_ch to the mpm ctx */
if (mpm_ch != NULL) {
DetectContentData *co = mpm_ch->ptr;
uint16_t offset = s->flags & SIG_FLAG_RECURSIVE ? 0 : co->offset;
uint16_t depth = s->flags & SIG_FLAG_RECURSIVE ? 0 : co->depth;
offset = scan_ch->cnt ? 0 : offset;
depth = scan_ch->cnt ? 0 : depth;
offset = mpm_ch->cnt ? 0 : offset;
depth = mpm_ch->cnt ? 0 : depth;
uint8_t flags = 0;
if (co->flags & DETECT_CONTENT_NOCASE) {
@ -541,37 +528,10 @@ static int PatternMatchPreprarePopulateMpm(DetectEngineCtx *de_ctx, SigGroupHead
mpm_table[sgh->mpm_ctx->mpm_type].AddPattern(sgh->mpm_ctx, co->content, co->content_len, offset, depth, co->id, s->num, flags);
}
SCLogDebug("%"PRIu32" adding co->id %"PRIu32" to the scan phase (s->num %"PRIu32")", s->id, co->id, s->num);
SCLogDebug("%"PRIu32" adding co->id %"PRIu32" to the mpm phase (s->num %"PRIu32")", s->id, co->id, s->num);
} else {
SCLogDebug("%"PRIu32" no scan pattern selected", s->id);
SCLogDebug("%"PRIu32" no mpm pattern selected", s->id);
}
#if 0
/* add the rest of the patterns to the search ctx */
for (sm = s->pmatch ; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_CONTENT) {
DetectContentData *co = (DetectContentData *)sm->ctx;
if (co == NULL)
continue;
/* skip the one we already added */
if (scan_ch != NULL && co == scan_ch->ptr) {
SCLogDebug("%"PRIu32" co->id %"PRIu32" not added to search, already in scan", s->id, co->id);
continue;
}
uint16_t offset = s->flags & SIG_FLAG_RECURSIVE ? 0 : co->offset;
uint16_t depth = s->flags & SIG_FLAG_RECURSIVE ? 0 : co->depth;
if (co->flags & DETECT_CONTENT_NOCASE) {
mpm_table[sgh->mpm_ctx->mpm_type].AddPatternNocase(sgh->mpm_ctx, co->content, co->content_len, offset, depth, co->id, s->num);
} else {
mpm_table[sgh->mpm_ctx->mpm_type].AddPattern(sgh->mpm_ctx, co->content, co->content_len, offset, depth, co->id, s->num);
}
SCLogDebug("%"PRIu32" adding co->id %"PRIu32" to the search phase", s->id, co->id);
}
}
#endif
}
if (fast_pattern != NULL)
@ -860,14 +820,14 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
uricontent_minlen = ud->uricontent_len;
}
}
char uricontent_scanadded = 0;
char uricontent_mpmadded = 0;
for (sm = s->match; sm != NULL; sm = sm->next) {
if (sm->type == DETECT_URICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) {
DetectUricontentData *ud = (DetectUricontentData *)sm->ctx;
/* only add the pattern if: we didn't add a pattern already,
* length is the same as maxlen (ie we only add the longest pattern) */
if (!uricontent_scanadded && uricontent_maxlen == ud->uricontent_len) {
if (!uricontent_mpmadded && uricontent_maxlen == ud->uricontent_len) {
uint8_t flags = 0;
if (ud->flags & DETECT_URICONTENT_NOCASE) {
@ -875,17 +835,7 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
} else {
mpm_table[sh->mpm_uri_ctx->mpm_type].AddPattern(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, 0, 0, ud->id, s->num, flags);
}
uricontent_scanadded = 1;
/* otherwise it's a 'search' pattern */
#if 0
} else {
if (ud->flags & DETECT_URICONTENT_NOCASE) {
mpm_table[sh->mpm_uri_ctx->mpm_type].AddPatternNocase(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, 0, 0, ud->id, s->num);
} else {
mpm_table[sh->mpm_uri_ctx->mpm_type].AddPattern(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, 0, 0, ud->id, s->num);
}
#endif
uricontent_mpmadded = 1;
}
}
}
@ -921,9 +871,6 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
mpm_table[sh->mpm_uri_ctx->mpm_type].Prepare(sh->mpm_uri_ctx);
}
if (mpm_uricontent_cnt && sh->mpm_uricontent_maxlen > 1) {
// printf("mpm_uricontent_cnt %" PRIu32 ", mpm_uricontent_maxlen %" PRId32 "\n", mpm_uricontent_cnt, mpm_uricontent_maxlen);
g_uricontent_scan++;
} else {
g_uricontent_search++;
}

@ -5,8 +5,8 @@
uint16_t PatternMatchDefaultMatcher(void);
uint32_t PacketPatternScan(ThreadVars *, DetectEngineThreadCtx *, Packet *);
uint32_t UriPatternScan(ThreadVars *, DetectEngineThreadCtx *, uint8_t *, uint16_t);
uint32_t PacketPatternSearch(ThreadVars *, DetectEngineThreadCtx *, Packet *);
uint32_t UriPatternSearch(ThreadVars *, DetectEngineThreadCtx *, uint8_t *, uint16_t);
void PacketPatternCleanup(ThreadVars *, DetectEngineThreadCtx *);
@ -24,7 +24,7 @@ void PatternMatchDestroyGroup(SigGroupHead *);
TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **);
TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *);
void DbgPrintScanSearchStats();
void DbgPrintSearchStats();
#endif /* __DETECT_ENGINE_MPM_H__ */

@ -229,7 +229,7 @@ int DetectFastPatternTest04(void)
}
/**
* \test Checks that a fast_pattern is used in the Scan phase.
* \test Checks that a fast_pattern is used in the mpm phase.
*/
int DetectFastPatternTest05(void)
{
@ -269,9 +269,9 @@ int DetectFastPatternTest05(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) != 0)
if (PacketPatternSearch(&th_v, det_ctx, &p) != 0)
result = 1;
SigGroupCleanup(de_ctx);
@ -285,7 +285,7 @@ end:
}
/**
* \test Checks that a fast_pattern is used in the Scan phase.
* \test Checks that a fast_pattern is used in the mpm phase.
*/
int DetectFastPatternTest06(void)
{
@ -323,9 +323,9 @@ int DetectFastPatternTest06(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) != 0)
if (PacketPatternSearch(&th_v, det_ctx, &p) != 0)
result = 1;
SigGroupCleanup(de_ctx);
@ -339,7 +339,7 @@ end:
}
/**
* \test Checks that a fast_pattern is used in the Scan phase, when the payload
* \test Checks that a fast_pattern is used in the mpm phase, when the payload
* doesn't contain the fast_pattern string within it.
*/
int DetectFastPatternTest07(void)
@ -378,9 +378,9 @@ int DetectFastPatternTest07(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) == 0)
if (PacketPatternSearch(&th_v, det_ctx, &p) == 0)
result = 1;
SigGroupCleanup(de_ctx);
@ -394,8 +394,8 @@ end:
}
/**
* \test Checks that a fast_pattern is used in the Scan phase and that we get
* exactly 1 match for the scan phase.
* \test Checks that a fast_pattern is used in the mpm phase and that we get
* exactly 1 match for the mpm phase.
*/
int DetectFastPatternTest08(void)
{
@ -433,9 +433,9 @@ int DetectFastPatternTest08(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) == 1)
if (PacketPatternSearch(&th_v, det_ctx, &p) == 1)
result = 1;
SigGroupCleanup(de_ctx);
@ -449,7 +449,7 @@ end:
}
/**
* \test Checks that a fast_pattern is used in the Scan phase, when the payload
* \test Checks that a fast_pattern is used in the mpm phase, when the payload
* doesn't contain the fast_pattern string within it.
*/
int DetectFastPatternTest09(void)
@ -488,9 +488,9 @@ int DetectFastPatternTest09(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) == 0)
if (PacketPatternSearch(&th_v, det_ctx, &p) == 0)
result = 1;
SigGroupCleanup(de_ctx);
@ -506,7 +506,7 @@ end:
/**
* \test Checks that a the SigInit chooses the fast_pattern with better pattern
* strength, when we have multiple fast_patterns in the Signature. Also
* checks that we get a match for the fast_pattern from the Scan phase.
* checks that we get a match for the fast_pattern from the mpm phase.
*/
int DetectFastPatternTest10(void)
{
@ -544,9 +544,9 @@ int DetectFastPatternTest10(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) == 1)
if (PacketPatternSearch(&th_v, det_ctx, &p) == 1)
result = 1;
SigGroupCleanup(de_ctx);
@ -562,7 +562,7 @@ end:
/**
* \test Checks that a the SigInit chooses the fast_pattern with better pattern
* strength, when we have multiple fast_patterns in the Signature. Also
* checks that we get no matches for the fast_pattern from the Scan phase.
* checks that we get no matches for the fast_pattern from the mpm phase.
*/
int DetectFastPatternTest11(void)
{
@ -600,9 +600,9 @@ int DetectFastPatternTest11(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) == 0)
if (PacketPatternSearch(&th_v, det_ctx, &p) == 0)
result = 1;
@ -618,7 +618,7 @@ end:
}
/**
* \test Checks that we don't get a match for the scan phase.
* \test Checks that we don't get a match for the mpm phase.
*/
int DetectFastPatternTest12(void)
{
@ -656,9 +656,9 @@ int DetectFastPatternTest12(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) == 0)
if (PacketPatternSearch(&th_v, det_ctx, &p) == 0)
result = 1;
SigGroupCleanup(de_ctx);
@ -674,7 +674,7 @@ end:
/**
* \test Checks that a the SigInit chooses the fast_pattern with a better
* strength from the available patterns, when we don't specify a
* fast_pattern. We also check that we get a match from the Scan
* fast_pattern. We also check that we get a match from the mpm
* phase.
*/
int DetectFastPatternTest13(void)
@ -713,9 +713,9 @@ int DetectFastPatternTest13(void)
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the scan phase */
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
if (PacketPatternScan(&th_v, det_ctx, &p) == 1)
if (PacketPatternSearch(&th_v, det_ctx, &p) == 1)
result = 1;
SigGroupCleanup(de_ctx);

@ -215,7 +215,7 @@ static int DetectHttpCookieSetup (DetectEngineCtx *de_ctx, Signature *s, char *s
SCFree(pm->ctx);
SCFree(pm);
/* Flagged the signature as to scan the app layer data */
/* Flagged the signature as to inspect the app layer data */
s->flags |= SIG_FLAG_APPLAYER;
return 0;

@ -202,7 +202,7 @@ static int DetectHttpMethodSetup(DetectEngineCtx *de_ctx, Signature *s, char *st
SCFree(pm->ctx);
SCFree(pm);
/* Flagged the signature as to scan the app layer data */
/* Flagged the signature as to inspect the app layer data */
s->flags |= SIG_FLAG_APPLAYER;
SCReturnInt(0);

@ -244,7 +244,7 @@ int DetectUricontentSetup (DetectEngineCtx *de_ctx, Signature *s, char *contents
cd->id = de_ctx->uricontent_max_id;
de_ctx->uricontent_max_id++;
/* Flagged the signature as to scan the app layer data */
/* Flagged the signature as to inspect the app layer data */
s->flags |= SIG_FLAG_APPLAYER;
SCReturnInt(0);
@ -258,7 +258,7 @@ error:
* \brief Checks if the content sent as the argument, has a uricontent which
* has been provided in the rule. This match function matches the
* normalized http uri against the given rule using multi pattern
* scan/search algorithms.
* search algorithms.
*
* \param t Pointer to the tv for this detection module instance
* \param det_ctx Pointer to the detection engine thread context
@ -273,10 +273,10 @@ int DoDetectAppLayerUricontentMatch (ThreadVars *tv, DetectEngineThreadCtx *det_
int ret = 0;
/* run the pattern matcher against the uri */
if (det_ctx->sgh->mpm_uricontent_maxlen > uri_len) {
SCLogDebug("not scanning as pkt payload is smaller than the "
SCLogDebug("not searching as pkt payload is smaller than the "
"largest uricontent length we need to match");
} else {
SCLogDebug("scan: (%p, maxlen %" PRIu32 ", sgh->sig_cnt "
SCLogDebug("search: (%p, maxlen %" PRIu32 ", sgh->sig_cnt "
"%" PRIu32 ")", det_ctx->sgh, det_ctx->sgh->
mpm_uricontent_maxlen, det_ctx->sgh->sig_cnt);
@ -288,9 +288,9 @@ int DoDetectAppLayerUricontentMatch (ThreadVars *tv, DetectEngineThreadCtx *det_
else if (det_ctx->sgh->mpm_uricontent_maxlen == 4) det_ctx->pkts_uri_searched4++;
else det_ctx->pkts_uri_searched++;
ret += UriPatternScan(tv, det_ctx, uri, uri_len);
ret += UriPatternSearch(tv, det_ctx, uri, uri_len);
SCLogDebug("post scan: cnt %" PRIu32 ", searchable %" PRIu32 "",
SCLogDebug("post search: cnt %" PRIu32 ", searchable %" PRIu32 "",
ret, det_ctx->pmq.searchable);
det_ctx->pmq.searchable = 0;
}
@ -322,7 +322,7 @@ int DetectAppLayerUricontentMatch (ThreadVars *tv, DetectEngineThreadCtx *det_ct
size_t idx = 0;
htp_tx_t *tx = NULL;
/* if we don't have a uri, don't bother scanning */
/* if we don't have a uri, don't bother inspecting */
if (det_ctx->de_have_httpuri == FALSE) {
SCLogDebug("We don't have uri");
SCReturnInt(0);
@ -826,7 +826,7 @@ end:
return result;
}
/** \test Check the working of scan/search once per packet only in applayer
/** \test Check the working of search once per packet only in applayer
* match */
static int DetectUriSigTest03(void) {
int result = 0;

@ -285,7 +285,7 @@ static int DetectUrilenSetup (DetectEngineCtx *de_ctx, Signature *s, char *urile
SigMatchAppendAppLayer(s, sm);
/* Flagged the signature as to scan the app layer data */
/* Flagged the signature as to inspect the app layer data */
s->flags |= SIG_FLAG_APPLAYER;
SCReturnInt(0);

@ -167,7 +167,7 @@ void DetectExitPrintStats(ThreadVars *tv, void *data) {
tv->name, det_ctx->uris, det_ctx->pkts_uri_searched,
(float)(det_ctx->pkts_uri_searched/(float)(det_ctx->uris)*100));
SCLogInfo("%"PRIu64" sigs per scan match on avg needed inspection, total scans %"PRIu64", less than 25 sigs need inspect %"PRIu64", more than 100 sigs need inspect %"PRIu64", more than 1000 %"PRIu64" max %"PRIu64"", det_ctx->scans_match ? det_ctx->scans_sigs / det_ctx->scans_match : 0, det_ctx->scans_match, det_ctx->scans_sigsmin25, det_ctx->scans_sigsplus100, det_ctx->scans_sigsplus1000, det_ctx->scans_sigsmax);
SCLogInfo("%"PRIu64" sigs per mpm match on avg needed inspection, total mpm searches %"PRIu64", less than 25 sigs need inspect %"PRIu64", more than 100 sigs need inspect %"PRIu64", more than 1000 %"PRIu64" max %"PRIu64"", det_ctx->mpm_match ? det_ctx->mpm_sigs / det_ctx->mpm_match : 0, det_ctx->mpm_match, det_ctx->mpm_sigsmin25, det_ctx->mpm_sigsplus100, det_ctx->mpm_sigsplus1000, det_ctx->mpm_sigsmax);
}
/** \brief Create the path if default-rule-path was specified
@ -525,10 +525,11 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
if (p->payload_len > 0 && det_ctx->sgh->mpm_ctx != NULL && !(p->flags & PKT_NOPAYLOAD_INSPECTION)) {
/* run the pattern matcher against the packet */
if (det_ctx->sgh->mpm_content_maxlen > p->payload_len) {
SCLogDebug("not scanning as pkt payload is smaller than the largest content length we need to match");
SCLogDebug("not mpm-inspecting as pkt payload is smaller than "
"the largest content length we need to match");
} else {
SCLogDebug("scan: (%p, maxlen %" PRIu32 ", sgh->sig_cnt %" PRIu32 ")", det_ctx->sgh, det_ctx->sgh->mpm_content_maxlen, det_ctx->sgh->sig_cnt);
/* scan, but only if the noscan flag isn't set */
SCLogDebug("search: (%p, maxlen %" PRIu32 ", sgh->sig_cnt %" PRIu32 ")",
det_ctx->sgh, det_ctx->sgh->mpm_content_maxlen, det_ctx->sgh->sig_cnt);
if (det_ctx->sgh->mpm_content_maxlen == 1) det_ctx->pkts_searched1++;
else if (det_ctx->sgh->mpm_content_maxlen == 2) det_ctx->pkts_searched2++;
@ -536,31 +537,32 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
else if (det_ctx->sgh->mpm_content_maxlen == 4) det_ctx->pkts_searched4++;
else det_ctx->pkts_searched++;
cnt = PacketPatternScan(th_v, det_ctx, p);
cnt = PacketPatternSearch(th_v, det_ctx, p);
if (cnt > 0) {
det_ctx->scans_match++;
det_ctx->scans_sigs += det_ctx->pmq.sig_id_array_cnt;
det_ctx->mpm_match++;
det_ctx->mpm_sigs += det_ctx->pmq.sig_id_array_cnt;
if (det_ctx->pmq.sig_id_array_cnt < 25) {
det_ctx->scans_sigsmin25++;
det_ctx->mpm_sigsmin25++;
} else if (det_ctx->pmq.sig_id_array_cnt > 1000) {
det_ctx->scans_sigsplus1000++;
det_ctx->mpm_sigsplus1000++;
} else if (det_ctx->pmq.sig_id_array_cnt > 100) {
det_ctx->scans_sigsplus100++;
det_ctx->mpm_sigsplus100++;
}
if (det_ctx->pmq.sig_id_array_cnt > det_ctx->scans_sigsmax)
det_ctx->scans_sigsmax = det_ctx->pmq.sig_id_array_cnt;
if (det_ctx->pmq.sig_id_array_cnt > det_ctx->mpm_sigsmax)
det_ctx->mpm_sigsmax = det_ctx->pmq.sig_id_array_cnt;
}
SCLogDebug("post scan: cnt %" PRIu32 ", searchable %" PRIu32 ", sigs %"PRIu32" (out of %"PRIu32")", cnt, det_ctx->pmq.searchable, det_ctx->pmq.sig_id_array_cnt, det_ctx->sgh->sig_cnt);
SCLogDebug("post search: cnt %" PRIu32 ", searchable %" PRIu32 ", sigs %"PRIu32" (out of %"PRIu32")", cnt, det_ctx->pmq.searchable, det_ctx->pmq.sig_id_array_cnt, det_ctx->sgh->sig_cnt);
det_ctx->pmq.searchable = 0;
}
}
/* If we have the uricontent multi pattern matcher signatures in
signature list, then scan the received HTTP uri(s) in the htp state
against those patterns */
if (det_ctx->sgh->flags & SIG_GROUP_HAVEURICONTENT && p->flow != NULL && alproto == ALPROTO_HTTP)
signature list, then search the received HTTP uri(s) in the htp
state against those patterns */
if (det_ctx->sgh->flags & SIG_GROUP_HAVEURICONTENT && p->flow != NULL &&
alproto == ALPROTO_HTTP)
{
SCMutexLock(&p->flow->m);
cnt = DetectUricontentInspectMpm(th_v, det_ctx, alstate);
@ -2823,7 +2825,7 @@ int SigGroupBuild (DetectEngineCtx *de_ctx) {
#endif
// SigAddressPrepareStage5(de_ctx);
DbgPrintScanSearchStats();
DbgPrintSearchStats();
// DetectAddressPrintMemory();
// DetectSigGroupPrintMemory();
// DetectPortPrintMemory();
@ -3759,14 +3761,12 @@ static int SigTest11Real (int mpm_type) {
de_ctx->mpm_matcher = mpm_type;
de_ctx->flags |= DE_QUIET;
de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any any (msg:\"Scan vs Search (1)\"; content:\"ABCDEFGHIJ\"; content:\"klmnop\"; content:\"1234\"; sid:1;)");
de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any any (content:\"ABCDEFGHIJ\"; content:\"klmnop\"; content:\"1234\"; sid:1;)");
if (de_ctx->sig_list == NULL) {
result = 0;
goto end;
}
de_ctx->sig_list->next = SigInit(de_ctx,"alert tcp any any -> any any (msg:\"Scan vs Search (2)\"; content:\"VWXYZabcde\"; content:\"5678\"; content:\"89\"; sid:2;)");
de_ctx->sig_list->next = SigInit(de_ctx,"alert tcp any any -> any any (content:\"VWXYZabcde\"; content:\"5678\"; content:\"89\"; sid:2;)");
if (de_ctx->sig_list->next == NULL) {
result = 0;
goto end;
}
@ -3776,8 +3776,6 @@ static int SigTest11Real (int mpm_type) {
SigMatchSignatures(&th_v, de_ctx, det_ctx, &p);
if (PacketAlertCheck(&p, 1) && PacketAlertCheck(&p, 2))
result = 1;
else
result = 0;
AppLayerParserCleanupState(&ssn);
SigGroupCleanup(de_ctx);
@ -8455,9 +8453,9 @@ void SigRegisterTests(void) {
UtRegisterTest("SigTest10B3g -- long content match, longer than pkt", SigTest10B3g, 1);
UtRegisterTest("SigTest10Wm -- long content match, longer than pkt", SigTest10Wm, 1);
UtRegisterTest("SigTest11B2g -- scan vs search", SigTest11B2g, 1);
UtRegisterTest("SigTest11B3g -- scan vs search", SigTest11B3g, 1);
UtRegisterTest("SigTest11Wm -- scan vs search", SigTest11Wm, 1);
UtRegisterTest("SigTest11B2g -- mpm searching", SigTest11B2g, 1);
UtRegisterTest("SigTest11B3g -- mpm searching", SigTest11B3g, 1);
UtRegisterTest("SigTest11Wm -- mpm searching", SigTest11Wm, 1);
UtRegisterTest("SigTest12B2g -- content order matching, normal", SigTest12B2g, 1);
UtRegisterTest("SigTest12B3g -- content order matching, normal", SigTest12B3g, 1);

@ -400,12 +400,12 @@ typedef struct DetectionEngineThreadCtx_ {
Tmq *cuda_mpm_rc_disp_outq;
#endif
uint64_t scans_match;
uint64_t scans_sigs;
uint64_t scans_sigsmin25;
uint64_t scans_sigsplus100;
uint64_t scans_sigsplus1000;
uint64_t scans_sigsmax;
uint64_t mpm_match;
uint64_t mpm_sigs;
uint64_t mpm_sigsmin25;
uint64_t mpm_sigsplus100;
uint64_t mpm_sigsplus1000;
uint64_t mpm_sigsmax;
} DetectEngineThreadCtx;
/** \brief a single match condition for a signature */

@ -3,7 +3,7 @@
* Copyright (c) 2009 Victor Julien <victor@inliniac.net>
*
* Ideas:
* - B2g does a full match in the scan phase of up to 'm' characters,
* - B2g does a full match in the search of up to 'm' characters,
* in case of a case insensitive search we could say it's match if
* the pattern is of len 'm' or just compare the rest of the chars.
*
@ -133,7 +133,6 @@ void B2gPrintInfo(MpmCtx *mpm_ctx) {
printf(" B2gPattern %" PRIuMAX "\n", (uintmax_t)sizeof(B2gPattern));
printf(" B2gHashItem %" PRIuMAX "\n", (uintmax_t)sizeof(B2gHashItem));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Scan Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Total Patterns: %" PRIu32 "\n", mpm_ctx->total_pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);
@ -644,8 +643,8 @@ void B2gPrintSearchStats(MpmThreadCtx *mpm_thread_ctx) {
B2gThreadCtx *tctx = (B2gThreadCtx *)mpm_thread_ctx->ctx;
printf("B2g Thread Search stats (tctx %p)\n", tctx);
printf("Total calls/scans: %" PRIu32 "\n", tctx->stat_calls);
printf("Avg m/scan: %0.2f\n", tctx->stat_calls ? (float)((float)tctx->stat_m_total / (float)tctx->stat_calls) : 0);
printf("Total calls: %" PRIu32 "\n", tctx->stat_calls);
printf("Avg m/search: %0.2f\n", tctx->stat_calls ? (float)((float)tctx->stat_m_total / (float)tctx->stat_calls) : 0);
printf("D != 0 (possible match): %" PRIu32 "\n", tctx->stat_d0);
printf("Avg hash items per bucket %0.2f (%" PRIu32 ")\n", tctx->stat_d0 ? (float)((float)tctx->stat_d0_hashloop / (float)tctx->stat_d0) : 0, tctx->stat_d0_hashloop);
printf("Loop match: %" PRIu32 "\n", tctx->stat_loop_match);
@ -752,7 +751,7 @@ void B2gInitCtx (MpmCtx *mpm_ctx, int module_handle) {
if (b2g_hash_size == 0)
B2gGetConfig();
/* init defaults scan/search functions */
/* init defaults search functions */
ctx->Search = b2g_func;
SCReturn;
@ -1007,7 +1006,6 @@ uint32_t B2gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
d = ((d << 1) & ctx->B2G[h]);
j = j - 1;
} while (d != 0 && j != 0);
//printf("scan: d %" PRIu32 ", j %" PRIu32 "\n", d, j);
/* (partial) match, move on to verification */
if (d != 0) {

@ -72,10 +72,10 @@ typedef struct B2gCtx_ {
uint8_t s0;
/* we store our own multi byte scan ptr here for B2gSearch1 */
/* we store our own multi byte search func ptr here for B2gSearch1 */
uint32_t (*Search)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
/* we store our own multi byte scan ptr here for B2gSearch1 */
/* we store our own multi byte search func ptr here for B2gSearch1 */
uint32_t (*MBSearch2)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
uint32_t (*MBSearch)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
} B2gCtx;

@ -3,7 +3,7 @@
* Copyright (c) 2009 Victor Julien <victor@inliniac.net>
*
* Ideas:
* - B3g does a full match in the scan phase of up to 'm' characters,
* - B3g does a full match in the search of up to 'm' characters,
* in case of a case insensitive search we could say it's match if
* the pattern is of len 'm' or just compare the rest of the chars.
*
@ -121,7 +121,6 @@ void B3gPrintInfo(MpmCtx *mpm_ctx) {
printf(" B3gPattern %" PRIuMAX "\n", (uintmax_t)sizeof(B3gPattern));
printf(" B3gHashItem %" PRIuMAX "\n", (uintmax_t)sizeof(B3gHashItem));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Scan Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Total Patterns: %" PRIu32 "\n", mpm_ctx->total_pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);
@ -619,8 +618,8 @@ void B3gPrintSearchStats(MpmThreadCtx *mpm_thread_ctx) {
B3gThreadCtx *tctx = (B3gThreadCtx *)mpm_thread_ctx->ctx;
printf("B3g Thread Search stats (tctx %p)\n", tctx);
printf("Total calls/scans: %" PRIu32 "\n", tctx->stat_calls);
printf("Avg m/scan: %0.2f\n", tctx->stat_calls ? (float)((float)tctx->stat_m_total / (float)tctx->stat_calls) : 0);
printf("Total calls: %" PRIu32 "\n", tctx->stat_calls);
printf("Avg m/search: %0.2f\n", tctx->stat_calls ? (float)((float)tctx->stat_m_total / (float)tctx->stat_calls) : 0);
printf("D != 0 (possible match): %" PRIu32 "\n", tctx->stat_d0);
printf("Avg hash items per bucket %0.2f (%" PRIu32 ")\n", tctx->stat_d0 ? (float)((float)tctx->stat_d0_hashloop / (float)tctx->stat_d0) : 0, tctx->stat_d0_hashloop);
printf("Loop match: %" PRIu32 "\n", tctx->stat_loop_match);
@ -965,11 +964,9 @@ uint32_t B3gSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcher
do {
uint16_t h = B3G_HASH(u8_tolower(buf[pos + j - 1]), u8_tolower(buf[pos + j - 0]),u8_tolower(buf[pos + j + 1]));
// printf("scan: h %" PRIu32 ", %c.%c.%c\n", h, u8_tolower(buf[pos + j - 1]), u8_tolower(buf[pos + j - 0]),u8_tolower(buf[pos + j + 1]));
d = ((d << 1) & ctx->B3G[h]);
j = j - 1;
} while (d != 0 && j != 0);
// printf("scan: d %" PRIu32 ", j %" PRIu32 "\n", d, j);
/* (partial) match, move on to verification */
if (d != 0) {

@ -64,7 +64,7 @@ typedef struct B3gCtx_ {
uint32_t (*Search)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
/* we store our own multi byte scan ptr here for B3gSearch1 */
/* we store our own multi byte search func ptr here for B3gSearch1 */
uint32_t (*MBSearch2)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
uint32_t (*MBSearch)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);

@ -148,7 +148,6 @@ void WmPrintInfo(MpmCtx *mpm_ctx) {
printf(" WmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(WmPattern));
printf(" WmHashItem %" PRIuMAX "\n", (uintmax_t)sizeof(WmHashItem));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Scan Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Total Patterns: %" PRIu32 "\n", mpm_ctx->total_pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);
@ -684,7 +683,6 @@ int WmPreparePatterns(MpmCtx *mpm_ctx) {
* we should do some performance testing
* */
/* scan */
if (ctx->hash_size == 0) {
if (mpm_ctx->pattern_cnt < 50) {
ctx->hash_size = HASH9_SIZE;

@ -39,9 +39,9 @@ typedef struct WmCtx_ {
for the BloomFilter. */
WmHashItem hash1[256];
/* we store our own scan ptr here for WmSearch1 */
/* we store our own search func ptr here for WmSearch1 */
uint32_t (*Search)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
/* we store our own multi byte scan ptr here for WmSearch1 */
/* we store our own multi byte search func ptr here for WmSearch1 */
uint32_t (*MBSearch)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
/* pattern arrays */

@ -85,7 +85,6 @@ typedef struct MpmCtx_ {
uint32_t endmatches;
uint32_t scan_pattern_cnt; /* scan patterns */
uint32_t pattern_cnt; /* unique patterns */
uint32_t total_pattern_cnt; /* total patterns added */

Loading…
Cancel
Save