From cce2d114e8a8175240c65aba572666b5faa05709 Mon Sep 17 00:00:00 2001 From: Justin Viiret Date: Wed, 11 May 2016 14:10:27 +1000 Subject: [PATCH] spm: add and use new SPM API This new API allows for different SPM implementations, using a function pointer table like that used for MPM. This change also switches over the paths that make use of DetectContentData (which previously used BoyerMoore directly) to the new API. --- src/app-layer-detect-proto.c | 40 +++- src/detect-content.c | 166 ++++++++++++--- src/detect-content.h | 12 +- src/detect-engine-content-inspection.c | 7 +- src/detect-engine.c | 18 ++ src/detect-http-client-body.c | 2 +- src/detect-http-hh.c | 2 +- src/detect-http-hrh.c | 2 +- src/detect-http-server-body.c | 2 +- src/detect-http-ua.c | 2 +- src/detect-nocase.c | 14 +- src/detect-uricontent.c | 3 +- src/detect-uricontent.h | 1 - src/detect.h | 9 + src/runmode-unittests.c | 1 + src/suricata.c | 1 + src/util-spm-bm.c | 131 ++++++++++++ src/util-spm-bm.h | 2 + src/util-spm.c | 275 ++++++++++++++++++++++++- src/util-spm.h | 55 +++++ 20 files changed, 687 insertions(+), 58 deletions(-) diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c index 323e0857ab..aeed07259a 100644 --- a/src/app-layer-detect-proto.c +++ b/src/app-layer-detect-proto.c @@ -142,6 +142,9 @@ typedef struct AppLayerProtoDetectCtx_ { * implemented if needed. Waste of space otherwise. */ AppLayerProtoDetectCtxIpproto ctx_ipp[FLOW_PROTO_DEFAULT]; + /* Global SPM thread context prototype. */ + SpmGlobalThreadCtx *spm_global_thread_ctx; + AppLayerProtoDetectProbingParser *ctx_pp; /* Indicates the protocols that have registered themselves @@ -157,6 +160,7 @@ struct AppLayerProtoDetectThreadCtx_ { PatternMatcherQueue pmq; /* The value 2 is for direction(0 - toserver, 1 - toclient). */ MpmThreadCtx mpm_tctx[FLOW_PROTO_DEFAULT][2]; + SpmThreadCtx *spm_thread_ctx; }; /* The global app layer proto detection context. */ @@ -167,6 +171,7 @@ static AppLayerProtoDetectCtx alpd_ctx; /** \internal * \brief Handle SPM search for Signature */ static AppProto AppLayerProtoDetectPMMatchSignature(const AppLayerProtoDetectPMSignature *s, + AppLayerProtoDetectThreadCtx *tctx, uint8_t *buf, uint16_t buflen, uint8_t ipproto) { @@ -191,10 +196,7 @@ static AppProto AppLayerProtoDetectPMMatchSignature(const AppLayerProtoDetectPMS SCLogDebug("s->co->offset (%"PRIu16") s->cd->depth (%"PRIu16")", s->cd->offset, s->cd->depth); - if (s->cd->flags & DETECT_CONTENT_NOCASE) - found = BoyerMooreNocase(s->cd->content, s->cd->content_len, sbuf, sbuflen, s->cd->bm_ctx); - else - found = BoyerMoore(s->cd->content, s->cd->content_len, sbuf, sbuflen, s->cd->bm_ctx); + found = SpmScan(s->cd->spm_ctx, tctx->spm_thread_ctx, sbuf, sbuflen); if (found != NULL) proto = s->alproto; @@ -260,7 +262,7 @@ static AppProto AppLayerProtoDetectPMGetProto(AppLayerProtoDetectThreadCtx *tctx const AppLayerProtoDetectPMSignature *s = pm_ctx->map[tctx->pmq.rule_id_array[cnt]]; while (s != NULL) { AppProto proto = AppLayerProtoDetectPMMatchSignature(s, - buf, searchlen, ipproto); + tctx, buf, searchlen, ipproto); /* store each unique proto once */ if (proto != ALPROTO_UNKNOWN && @@ -1240,13 +1242,20 @@ static int AppLayerProtoDetectPMRegisterPattern(uint8_t ipproto, AppProto alprot DetectContentData *cd; int ret = 0; - cd = DetectContentParseEncloseQuotes(pattern); + cd = DetectContentParseEncloseQuotes(alpd_ctx.spm_global_thread_ctx, + pattern); if (cd == NULL) goto error; cd->depth = depth; cd->offset = offset; if (!is_cs) { - BoyerMooreCtxToNocase(cd->bm_ctx, cd->content, cd->content_len); + /* Rebuild as nocase */ + SpmDestroyCtx(cd->spm_ctx); + cd->spm_ctx = SpmInitCtx(cd->content, cd->content_len, 1, + alpd_ctx.spm_global_thread_ctx); + if (cd->spm_ctx == NULL) { + goto error; + } cd->flags |= DETECT_CONTENT_NOCASE; } if (depth < cd->content_len) @@ -1518,6 +1527,13 @@ int AppLayerProtoDetectSetup(void) memset(&alpd_ctx, 0, sizeof(alpd_ctx)); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + alpd_ctx.spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + if (alpd_ctx.spm_global_thread_ctx == NULL) { + SCLogError(SC_ERR_FATAL, "Unable to alloc SpmGlobalThreadCtx."); + exit(EXIT_FAILURE); + } + for (i = 0; i < FLOW_PROTO_DEFAULT; i++) { for (j = 0; j < 2; j++) { MpmInitCtx(&alpd_ctx.ctx_ipp[i].ctx_pm[j].mpm_ctx, MPM_AC); @@ -1550,6 +1566,8 @@ int AppLayerProtoDetectDeSetup(void) } } + SpmDestroyGlobalThreadCtx(alpd_ctx.spm_global_thread_ctx); + AppLayerProtoDetectFreeProbingParsers(alpd_ctx.ctx_pp); SCReturnInt(0); @@ -1674,6 +1692,11 @@ AppLayerProtoDetectThreadCtx *AppLayerProtoDetectGetCtxThread(void) } } + alpd_tctx->spm_thread_ctx = SpmMakeThreadCtx(alpd_ctx.spm_global_thread_ctx); + if (alpd_tctx->spm_thread_ctx == NULL) { + goto error; + } + goto end; error: if (alpd_tctx != NULL) @@ -1699,6 +1722,9 @@ void AppLayerProtoDetectDestroyCtxThread(AppLayerProtoDetectThreadCtx *alpd_tctx } } PmqFree(&alpd_tctx->pmq); + if (alpd_tctx->spm_thread_ctx != NULL) { + SpmDestroyThreadCtx(alpd_tctx->spm_thread_ctx); + } SCFree(alpd_tctx); SCReturn; diff --git a/src/detect-content.c b/src/detect-content.c index 372a5da1ea..1bbb6f7c18 100644 --- a/src/detect-content.c +++ b/src/detect-content.c @@ -41,7 +41,7 @@ #include "util-unittest.h" #include "util-print.h" #include "util-debug.h" -#include "util-spm-bm.h" +#include "util-spm.h" #include "threads.h" #include "util-unittest-helper.h" #include "pkt-var.h" @@ -217,7 +217,8 @@ error: * \brief DetectContentParse * \initonly */ -DetectContentData *DetectContentParse (char *contentstr) +DetectContentData *DetectContentParse(SpmGlobalThreadCtx *spm_global_thread_ctx, + char *contentstr) { DetectContentData *cd = NULL; uint8_t *content = NULL; @@ -245,8 +246,15 @@ DetectContentData *DetectContentParse (char *contentstr) memcpy(cd->content, content, len); cd->content_len = len; - /* Prepare Boyer Moore context for searching faster */ - cd->bm_ctx = BoyerMooreCtxInit(cd->content, cd->content_len); + /* Prepare SPM search context. */ + cd->spm_ctx = SpmInitCtx(cd->content, cd->content_len, 0, + spm_global_thread_ctx); + if (cd->spm_ctx == NULL) { + SCFree(content); + SCFree(cd); + return NULL; + } + cd->depth = 0; cd->offset = 0; cd->within = 0; @@ -257,7 +265,8 @@ DetectContentData *DetectContentParse (char *contentstr) } -DetectContentData *DetectContentParseEncloseQuotes(char *contentstr) +DetectContentData *DetectContentParseEncloseQuotes(SpmGlobalThreadCtx *spm_global_thread_ctx, + char *contentstr) { char str[strlen(contentstr) + 3]; // 2 for quotes, 1 for \0 @@ -266,7 +275,7 @@ DetectContentData *DetectContentParseEncloseQuotes(char *contentstr) str[strlen(contentstr) + 1] = '\"'; str[strlen(contentstr) + 2] = '\0'; - return DetectContentParse(str); + return DetectContentParse(spm_global_thread_ctx, str); } /** @@ -370,7 +379,7 @@ int DetectContentSetup(DetectEngineCtx *de_ctx, Signature *s, char *contentstr) DetectContentData *cd = NULL; SigMatch *sm = NULL; - cd = DetectContentParse(contentstr); + cd = DetectContentParse(de_ctx->spm_global_thread_ctx, contentstr); if (cd == NULL) goto error; DetectContentPrint(cd); @@ -415,7 +424,7 @@ void DetectContentFree(void *ptr) if (cd == NULL) SCReturn; - BoyerMooreCtxDeInit(cd->bm_ctx); + SpmDestroyCtx(cd->spm_ctx); SCFree(cd); SCReturn; @@ -433,7 +442,11 @@ int DetectContentParseTest01 (void) char *teststring = "\"abc\\:def\""; char *teststringparsed = "abc:def"; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (memcmp(cd->content, teststringparsed, strlen(teststringparsed)) != 0) { SCLogDebug("expected %s got ", teststringparsed); @@ -446,6 +459,7 @@ int DetectContentParseTest01 (void) SCLogDebug("expected %s got NULL: ", teststringparsed); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -459,7 +473,11 @@ int DetectContentParseTest02 (void) char *teststring = "\"abc\\;def\""; char *teststringparsed = "abc;def"; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (memcmp(cd->content, teststringparsed, strlen(teststringparsed)) != 0) { SCLogDebug("expected %s got ", teststringparsed); @@ -472,6 +490,7 @@ int DetectContentParseTest02 (void) SCLogDebug("expected %s got NULL: ", teststringparsed); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -485,7 +504,11 @@ int DetectContentParseTest03 (void) char *teststring = "\"abc\\\"def\""; char *teststringparsed = "abc\"def"; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (memcmp(cd->content, teststringparsed, strlen(teststringparsed)) != 0) { SCLogDebug("expected %s got ", teststringparsed); @@ -498,6 +521,7 @@ int DetectContentParseTest03 (void) SCLogDebug("expected %s got NULL: ", teststringparsed); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -511,7 +535,11 @@ int DetectContentParseTest04 (void) char *teststring = "\"abc\\\\def\""; char *teststringparsed = "abc\\def"; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { uint16_t len = (cd->content_len > strlen(teststringparsed)); if (memcmp(cd->content, teststringparsed, len) != 0) { @@ -525,6 +553,7 @@ int DetectContentParseTest04 (void) SCLogDebug("expected %s got NULL: ", teststringparsed); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -537,7 +566,11 @@ int DetectContentParseTest05 (void) DetectContentData *cd = NULL; char *teststring = "\"abc\\def\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { SCLogDebug("expected NULL got "); PrintRawUriFp(stdout,cd->content,cd->content_len); @@ -545,6 +578,7 @@ int DetectContentParseTest05 (void) result = 0; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -558,7 +592,11 @@ int DetectContentParseTest06 (void) char *teststring = "\"a|42|c|44|e|46|\""; char *teststringparsed = "abcdef"; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { uint16_t len = (cd->content_len > strlen(teststringparsed)); if (memcmp(cd->content, teststringparsed, len) != 0) { @@ -572,6 +610,7 @@ int DetectContentParseTest06 (void) SCLogDebug("expected %s got NULL: ", teststringparsed); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -584,12 +623,17 @@ int DetectContentParseTest07 (void) DetectContentData *cd = NULL; char *teststring = "\"\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { SCLogDebug("expected NULL got %p: ", cd); result = 0; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -602,12 +646,17 @@ int DetectContentParseTest08 (void) DetectContentData *cd = NULL; char *teststring = "\"\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { SCLogDebug("expected NULL got %p: ", cd); result = 0; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -887,14 +936,18 @@ int DetectContentParseTest09(void) DetectContentData *cd = NULL; char *teststring = "!\"boo\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (cd->flags & DETECT_CONTENT_NEGATED) result = 1; DetectContentFree(cd); } - + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -904,13 +957,18 @@ int DetectContentParseTest10(void) DetectContentData *cd = NULL; char *teststring = "!\"boo\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (cd->flags & DETECT_CONTENT_NEGATED) result = 1; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -920,13 +978,18 @@ int DetectContentParseNegTest11(void) DetectContentData *cd = NULL; char *teststring = "\"boo\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (!(cd->flags & DETECT_CONTENT_NEGATED)) result = 1; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -936,13 +999,18 @@ int DetectContentParseNegTest12(void) DetectContentData *cd = NULL; char *teststring = "\"boo\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (!(cd->flags & DETECT_CONTENT_NEGATED)) result = 1; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -952,13 +1020,18 @@ int DetectContentParseNegTest13(void) DetectContentData *cd = NULL; char *teststring = "!\"boo\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (cd->flags & DETECT_CONTENT_NEGATED) result = 1; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -968,13 +1041,18 @@ int DetectContentParseNegTest14(void) DetectContentData *cd = NULL; char *teststring = " \"!boo\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (!(cd->flags & DETECT_CONTENT_NEGATED)) result = 1; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -984,13 +1062,18 @@ int DetectContentParseNegTest15(void) DetectContentData *cd = NULL; char *teststring = " !\"boo\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { if (cd->flags & DETECT_CONTENT_NEGATED) result = 1; DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -1000,11 +1083,16 @@ int DetectContentParseNegTest16(void) DetectContentData *cd = NULL; char *teststring = " \"boo\""; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd != NULL) { result = (cd->content_len == 3 && memcmp(cd->content,"boo",3) == 0); DetectContentFree(cd); } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); return result; } @@ -2095,12 +2183,17 @@ int DetectContentParseTest41(void) teststring[idx++] = '\"'; teststring[idx++] = '\0'; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd == NULL) { SCLogDebug("expected not NULL"); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); SCFree(teststring); DetectContentFree(cd); return result; @@ -2125,12 +2218,17 @@ int DetectContentParseTest42(void) teststring[idx++] = '\"'; teststring[idx++] = '\0'; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd == NULL) { SCLogDebug("expected not NULL"); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); SCFree(teststring); DetectContentFree(cd); return result; @@ -2156,12 +2254,17 @@ int DetectContentParseTest43(void) teststring[idx++] = '\"'; teststring[idx++] = '\0'; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd == NULL) { SCLogDebug("expected not NULL"); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); SCFree(teststring); DetectContentFree(cd); return result; @@ -2190,12 +2293,17 @@ int DetectContentParseTest44(void) teststring[idx++] = '\"'; teststring[idx++] = '\0'; - cd = DetectContentParse(teststring); + uint16_t spm_matcher = SinglePatternMatchDefaultMatcher(); + SpmGlobalThreadCtx *spm_global_thread_ctx = SpmInitGlobalThreadCtx(spm_matcher); + FAIL_IF(spm_global_thread_ctx == NULL); + + cd = DetectContentParse(spm_global_thread_ctx, teststring); if (cd == NULL) { SCLogDebug("expected not NULL"); result = 0; } + SpmDestroyGlobalThreadCtx(spm_global_thread_ctx); SCFree(teststring); DetectContentFree(cd); return result; diff --git a/src/detect-content.h b/src/detect-content.h index 51300e7ea9..5976adb868 100644 --- a/src/detect-content.h +++ b/src/detect-content.h @@ -73,7 +73,7 @@ ((c)->flags & DETECT_CONTENT_FAST_PATTERN_CHOP)) -#include "util-spm-bm.h" +#include "util-spm.h" typedef struct DetectContentData_ { uint8_t *content; @@ -92,8 +92,8 @@ typedef struct DetectContentData_ { uint16_t offset; int32_t distance; int32_t within; - /* Boyer Moore context (for spm search) */ - BmCtx *bm_ctx; + /* SPM search context. */ + SpmCtx *spm_ctx; /* pointer to replacement data */ uint8_t *replace; } DetectContentData; @@ -101,10 +101,12 @@ typedef struct DetectContentData_ { /* prototypes */ void DetectContentRegister (void); uint32_t DetectContentMaxId(DetectEngineCtx *); -DetectContentData *DetectContentParse (char *contentstr); +DetectContentData *DetectContentParse(SpmGlobalThreadCtx *spm_global_thread_ctx, + char *contentstr); int DetectContentDataParse(const char *keyword, const char *contentstr, uint8_t **pstr, uint16_t *plen, uint32_t *flags); -DetectContentData *DetectContentParseEncloseQuotes(char *); +DetectContentData *DetectContentParseEncloseQuotes(SpmGlobalThreadCtx *spm_global_thread_ctx, + char *contentstr); int DetectContentSetup(DetectEngineCtx *de_ctx, Signature *s, char *contentstr); void DetectContentPrint(DetectContentData *); diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index be6d09f6cd..9a13d28faf 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -48,7 +48,6 @@ #include "app-layer-dcerpc.h" #include "util-spm.h" -#include "util-spm-bm.h" #include "util-debug.h" #include "util-print.h" @@ -279,10 +278,8 @@ int DetectEngineContentInspection(DetectEngineCtx *de_ctx, DetectEngineThreadCtx * greater than sbuffer_len found is anyways NULL */ /* do the actual search */ - if (cd->flags & DETECT_CONTENT_NOCASE) - found = BoyerMooreNocase(cd->content, cd->content_len, sbuffer, sbuffer_len, cd->bm_ctx); - else - found = BoyerMoore(cd->content, cd->content_len, sbuffer, sbuffer_len, cd->bm_ctx); + found = SpmScan(cd->spm_ctx, det_ctx->spm_thread_ctx, sbuffer, + sbuffer_len); /* next we evaluate the result in combination with the * negation flag. */ diff --git a/src/detect-engine.c b/src/detect-engine.c index 0912167293..1546aef3ee 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -831,6 +831,13 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix) de_ctx->mpm_matcher = PatternMatchDefaultMatcher(); de_ctx->spm_matcher = SinglePatternMatchDefaultMatcher(); + + de_ctx->spm_global_thread_ctx = SpmInitGlobalThreadCtx(de_ctx->spm_matcher); + if (de_ctx->spm_global_thread_ctx == NULL) { + SCLogDebug("Unable to alloc SpmGlobalThreadCtx."); + goto error; + } + DetectEngineCtxLoadConf(de_ctx); SigGroupHeadHashInit(de_ctx); @@ -941,6 +948,8 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) SigGroupCleanup(de_ctx); + SpmDestroyGlobalThreadCtx(de_ctx->spm_global_thread_ctx); + MpmFactoryDeRegisterAllMpmCtxProfiles(de_ctx); DetectEngineCtxFreeThreadKeywordData(de_ctx); @@ -1419,6 +1428,11 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx * PmqSetup(&det_ctx->pmq); + det_ctx->spm_thread_ctx = SpmMakeThreadCtx(de_ctx->spm_global_thread_ctx); + if (det_ctx->spm_thread_ctx == NULL) { + return TM_ECODE_FAILED; + } + /* sized to the max of our sgh settings. A max setting of 0 implies that all * sgh's have: sgh->non_mpm_store_cnt == 0 */ if (de_ctx->non_mpm_store_cnt_max > 0) { @@ -1634,6 +1648,10 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) PmqFree(&det_ctx->pmq); + if (det_ctx->spm_thread_ctx != NULL) { + SpmDestroyThreadCtx(det_ctx->spm_thread_ctx); + } + if (det_ctx->non_mpm_id_array != NULL) SCFree(det_ctx->non_mpm_id_array); diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c index e5c9a5af94..729629a098 100644 --- a/src/detect-http-client-body.c +++ b/src/detect-http-client-body.c @@ -123,7 +123,7 @@ void DetectHttpClientBodyFree(void *ptr) if (hcbd->content != NULL) SCFree(hcbd->content); - BoyerMooreCtxDeInit(hcbd->bm_ctx); + SpmDestroyCtx(hcbd->spm_ctx); SCFree(hcbd); SCReturn; diff --git a/src/detect-http-hh.c b/src/detect-http-hh.c index 883cdbeffa..859782f9f5 100644 --- a/src/detect-http-hh.c +++ b/src/detect-http-hh.c @@ -118,7 +118,7 @@ void DetectHttpHHFree(void *ptr) if (hhhd->content != NULL) SCFree(hhhd->content); - BoyerMooreCtxDeInit(hhhd->bm_ctx); + SpmDestroyCtx(hhhd->spm_ctx); SCFree(hhhd); return; diff --git a/src/detect-http-hrh.c b/src/detect-http-hrh.c index 4c3cdb24e6..2663b306b2 100644 --- a/src/detect-http-hrh.c +++ b/src/detect-http-hrh.c @@ -118,7 +118,7 @@ void DetectHttpHRHFree(void *ptr) if (hrhhd->content != NULL) SCFree(hrhhd->content); - BoyerMooreCtxDeInit(hrhhd->bm_ctx); + SpmDestroyCtx(hrhhd->spm_ctx); SCFree(hrhhd); return; diff --git a/src/detect-http-server-body.c b/src/detect-http-server-body.c index 828e84e500..c2b7a078df 100644 --- a/src/detect-http-server-body.c +++ b/src/detect-http-server-body.c @@ -127,7 +127,7 @@ void DetectHttpServerBodyFree(void *ptr) if (hsbd->content != NULL) SCFree(hsbd->content); - BoyerMooreCtxDeInit(hsbd->bm_ctx); + SpmDestroyCtx(hsbd->spm_ctx); SCFree(hsbd); SCReturn; diff --git a/src/detect-http-ua.c b/src/detect-http-ua.c index c3ee465d76..237b8f26e6 100644 --- a/src/detect-http-ua.c +++ b/src/detect-http-ua.c @@ -119,7 +119,7 @@ void DetectHttpUAFree(void *ptr) if (huad->content != NULL) SCFree(huad->content); - BoyerMooreCtxDeInit(huad->bm_ctx); + SpmDestroyCtx(huad->spm_ctx); SCFree(huad); return; diff --git a/src/detect-nocase.c b/src/detect-nocase.c index 9968e3c457..b9dbda34cf 100644 --- a/src/detect-nocase.c +++ b/src/detect-nocase.c @@ -117,9 +117,21 @@ static int DetectNocaseSetup (DetectEngineCtx *de_ctx, Signature *s, char *nulls SCLogError(SC_ERR_INVALID_SIGNATURE, "can't use multiple nocase modifiers with the same content"); goto end; } + + /* for consistency in later use (e.g. by MPM construction and hashing), + * coerce the content string to lower-case. */ + for (uint8_t *c = cd->content; c < cd->content + cd->content_len; c++) { + *c = u8_tolower(*c); + } + cd->flags |= DETECT_CONTENT_NOCASE; /* Recreate the context with nocase chars */ - BoyerMooreCtxToNocase(cd->bm_ctx, cd->content, cd->content_len); + SpmDestroyCtx(cd->spm_ctx); + cd->spm_ctx = SpmInitCtx(cd->content, cd->content_len, 1, + de_ctx->spm_global_thread_ctx); + if (cd->spm_ctx == NULL) { + goto end; + } ret = 0; end: diff --git a/src/detect-uricontent.c b/src/detect-uricontent.c index 6b79636b3c..2955dab0ff 100644 --- a/src/detect-uricontent.c +++ b/src/detect-uricontent.c @@ -54,7 +54,6 @@ #include "util-unittest-helper.h" #include "util-binsearch.h" #include "util-spm.h" -#include "util-spm-bm.h" #include "conf.h" /* prototypes */ @@ -95,7 +94,7 @@ void DetectUricontentFree(void *ptr) if (cd == NULL) SCReturn; - BoyerMooreCtxDeInit(cd->bm_ctx); + SpmDestroyCtx(cd->spm_ctx); SCFree(cd); SCReturn; diff --git a/src/detect-uricontent.h b/src/detect-uricontent.h index c32a923e35..812e69db9c 100644 --- a/src/detect-uricontent.h +++ b/src/detect-uricontent.h @@ -27,7 +27,6 @@ #include "detect-content.h" -#include "util-spm-bm.h" #include "app-layer-htp.h" /* prototypes */ diff --git a/src/detect.h b/src/detect.h index 67dee32db2..333381457b 100644 --- a/src/detect.h +++ b/src/detect.h @@ -33,6 +33,7 @@ #include "packet-queue.h" #include "util-mpm.h" +#include "util-spm.h" #include "util-hash.h" #include "util-hashlist.h" #include "util-debug.h" @@ -589,6 +590,10 @@ typedef struct DetectEngineCtx_ { uint16_t mpm_matcher; /**< mpm matcher this ctx uses */ uint16_t spm_matcher; /**< spm matcher this ctx uses */ + /* spm thread context prototype, built as spm matchers are constructed and + * later used to construct thread context for each thread. */ + SpmGlobalThreadCtx *spm_global_thread_ctx; + /* Config options */ uint16_t max_uniq_toclient_groups; @@ -818,6 +823,10 @@ typedef struct DetectEngineThreadCtx_ { MpmThreadCtx mtcs; /**< thread ctx for stream mpm */ PatternMatcherQueue pmq; + /** SPM thread context used for scanning. This has been cloned from the + * prototype held by DetectEngineCtx. */ + SpmThreadCtx *spm_thread_ctx; + /** ip only rules ctx */ DetectEngineIPOnlyThreadCtx io_ctx; diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index 14df0c49e6..523d81afdd 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -151,6 +151,7 @@ void RunUnittests(int list_unittests, char *regex_arg) #ifdef __SC_CUDA_SUPPORT__ MpmCudaEnvironmentSetup(); #endif + SpmTableSetup(); AppLayerSetup(); diff --git a/src/suricata.c b/src/suricata.c index d6285382ac..37ba4aa90f 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2225,6 +2225,7 @@ static int PostConfLoadedSetup(SCInstance *suri) #ifdef __SC_CUDA_SUPPORT__ MpmCudaEnvironmentSetup(); #endif + SpmTableSetup(); switch (suri->checksum_validation) { case 0: diff --git a/src/util-spm-bm.c b/src/util-spm-bm.c index dd16bb99dd..906927697e 100644 --- a/src/util-spm-bm.c +++ b/src/util-spm-bm.c @@ -34,6 +34,7 @@ #include "suricata.h" #include "util-spm-bm.h" +#include "util-spm.h" #include "util-debug.h" #include "util-error.h" #include "util-memcpy.h" @@ -380,3 +381,133 @@ uint8_t *BoyerMooreNocase(const uint8_t *x, uint16_t m, const uint8_t *y, int32_ return NULL; } +typedef struct SpmBmCtx_ { + BmCtx *bm_ctx; + uint8_t *needle; + uint16_t needle_len; + int nocase; +} SpmBmCtx; + +static SpmCtx *BMInitCtx(const uint8_t *needle, uint16_t needle_len, int nocase, + SpmGlobalThreadCtx *global_thread_ctx) +{ + SpmCtx *ctx = SCMalloc(sizeof(SpmCtx)); + if (ctx == NULL) { + SCLogDebug("Unable to alloc SpmCtx."); + return NULL; + } + memset(ctx, 0, sizeof(*ctx)); + ctx->matcher = SPM_BM; + + SpmBmCtx *sctx = SCMalloc(sizeof(SpmBmCtx)); + if (sctx == NULL) { + SCLogDebug("Unable to alloc SpmBmCtx."); + SCFree(ctx); + return NULL; + } + memset(sctx, 0, sizeof(*sctx)); + + sctx->needle = SCMalloc(needle_len); + if (sctx->needle == NULL) { + SCLogDebug("Unable to alloc string."); + SCFree(sctx); + SCFree(ctx); + return NULL; + } + memcpy(sctx->needle, needle, needle_len); + sctx->needle_len = needle_len; + + if (nocase) { + sctx->bm_ctx = BoyerMooreNocaseCtxInit(sctx->needle, sctx->needle_len); + sctx->nocase = 1; + } else { + sctx->bm_ctx = BoyerMooreCtxInit(sctx->needle, sctx->needle_len); + sctx->nocase = 0; + } + + ctx->ctx = sctx; + return ctx; +} + +static void BMDestroyCtx(SpmCtx *ctx) +{ + if (ctx == NULL) { + return; + } + + SpmBmCtx *sctx = ctx->ctx; + if (sctx != NULL) { + BoyerMooreCtxDeInit(sctx->bm_ctx); + if (sctx->needle != NULL) { + SCFree(sctx->needle); + } + SCFree(sctx); + } + + SCFree(ctx); +} + +static uint8_t *BMScan(const SpmCtx *ctx, SpmThreadCtx *thread_ctx, + const uint8_t *haystack, uint16_t haystack_len) +{ + const SpmBmCtx *sctx = ctx->ctx; + + if (sctx->nocase) { + return BoyerMooreNocase(sctx->needle, sctx->needle_len, haystack, + haystack_len, sctx->bm_ctx); + } else { + return BoyerMoore(sctx->needle, sctx->needle_len, haystack, + haystack_len, sctx->bm_ctx); + } +} + +static SpmGlobalThreadCtx *BMInitGlobalThreadCtx(void) +{ + SpmGlobalThreadCtx *global_thread_ctx = SCMalloc(sizeof(SpmGlobalThreadCtx)); + if (global_thread_ctx == NULL) { + SCLogDebug("Unable to alloc SpmThreadCtx."); + return NULL; + } + memset(global_thread_ctx, 0, sizeof(*global_thread_ctx)); + global_thread_ctx->matcher = SPM_BM; + return global_thread_ctx; +} + +static void BMDestroyGlobalThreadCtx(SpmGlobalThreadCtx *global_thread_ctx) +{ + if (global_thread_ctx == NULL) { + return; + } + SCFree(global_thread_ctx); +} + +static void BMDestroyThreadCtx(SpmThreadCtx *thread_ctx) +{ + if (thread_ctx == NULL) { + return; + } + SCFree(thread_ctx); +} + +static SpmThreadCtx *BMMakeThreadCtx(const SpmGlobalThreadCtx *global_thread_ctx) { + SpmThreadCtx *thread_ctx = SCMalloc(sizeof(SpmThreadCtx)); + if (thread_ctx == NULL) { + SCLogDebug("Unable to alloc SpmThreadCtx."); + return NULL; + } + memset(thread_ctx, 0, sizeof(*thread_ctx)); + thread_ctx->matcher = SPM_BM; + return thread_ctx; +} + +void SpmBMRegister(void) +{ + spm_table[SPM_BM].name = "bm"; + spm_table[SPM_BM].InitGlobalThreadCtx = BMInitGlobalThreadCtx; + spm_table[SPM_BM].DestroyGlobalThreadCtx = BMDestroyGlobalThreadCtx; + spm_table[SPM_BM].MakeThreadCtx = BMMakeThreadCtx; + spm_table[SPM_BM].DestroyThreadCtx = BMDestroyThreadCtx; + spm_table[SPM_BM].InitCtx = BMInitCtx; + spm_table[SPM_BM].DestroyCtx = BMDestroyCtx; + spm_table[SPM_BM].Scan = BMScan; +} diff --git a/src/util-spm-bm.h b/src/util-spm-bm.h index 051a10243b..a1c6b4c315 100644 --- a/src/util-spm-bm.h +++ b/src/util-spm-bm.h @@ -45,5 +45,7 @@ uint8_t *BoyerMoore(const uint8_t *x, uint16_t m, const uint8_t *y, int32_t n, B uint8_t *BoyerMooreNocase(const uint8_t *x, uint16_t m, const uint8_t *y, int32_t n, BmCtx *bm_ctx); void BoyerMooreCtxDeInit(BmCtx *); +void SpmBMRegister(void); + #endif /* __UTIL_SPM_BM__ */ diff --git a/src/util-spm.c b/src/util-spm.c index 97c5587ec5..aa09b98bc8 100644 --- a/src/util-spm.c +++ b/src/util-spm.c @@ -59,11 +59,19 @@ * \brief Returns the single pattern matcher algorithm to be used, based on the * spm-algo setting in yaml. */ -uint16_t SinglePatternMatchDefaultMatcher(void) { +uint16_t SinglePatternMatchDefaultMatcher(void) +{ char *spm_algo; if ((ConfGet("spm-algo", &spm_algo)) == 1) { - if (strcmp("bm", spm_algo) == 0) { - return SPM_BM; + if (spm_algo != NULL) { + for (uint16_t i = 0; i < SPM_TABLE_SIZE; i++) { + if (spm_table[i].name == NULL) { + continue; + } + if (strcmp(spm_table[i].name, spm_algo) == 0) { + return i; + } + } } SCLogError(SC_ERR_INVALID_YAML_CONF_ENTRY, @@ -76,6 +84,56 @@ uint16_t SinglePatternMatchDefaultMatcher(void) { return SPM_BM; /* default to Boyer-Moore */ } +void SpmTableSetup(void) +{ + memset(spm_table, 0, sizeof(spm_table)); + + SpmBMRegister(); +} + +SpmGlobalThreadCtx *SpmInitGlobalThreadCtx(uint16_t matcher) +{ + return spm_table[matcher].InitGlobalThreadCtx(); +} + +void SpmDestroyGlobalThreadCtx(SpmGlobalThreadCtx *global_thread_ctx) +{ + uint16_t matcher = global_thread_ctx->matcher; + spm_table[matcher].DestroyGlobalThreadCtx(global_thread_ctx); +} + +SpmThreadCtx *SpmMakeThreadCtx(const SpmGlobalThreadCtx *global_thread_ctx) +{ + uint16_t matcher = global_thread_ctx->matcher; + return spm_table[matcher].MakeThreadCtx(global_thread_ctx); +} + +void SpmDestroyThreadCtx(SpmThreadCtx *thread_ctx) +{ + uint16_t matcher = thread_ctx->matcher; + spm_table[matcher].DestroyThreadCtx(thread_ctx); +} + +SpmCtx *SpmInitCtx(const uint8_t *needle, uint16_t needle_len, int nocase, + SpmGlobalThreadCtx *global_thread_ctx) +{ + uint16_t matcher = global_thread_ctx->matcher; + return spm_table[matcher].InitCtx(needle, needle_len, nocase, + global_thread_ctx); +} + +void SpmDestroyCtx(SpmCtx *ctx) +{ + spm_table[ctx->matcher].DestroyCtx(ctx); +} + +uint8_t *SpmScan(const SpmCtx *ctx, SpmThreadCtx *thread_ctx, + const uint8_t *haystack, uint16_t haystack_len) +{ + uint16_t matcher = ctx->matcher; + return spm_table[matcher].Scan(ctx, thread_ctx, haystack, haystack_len); +} + /** * Wrappers for building context and searching (Bs2Bm and boyermoore) * Use them if you cant store the context @@ -2338,6 +2396,213 @@ int UtilSpmNocaseSearchStatsTest07() return 1; } +/* Unit tests for new SPM API. */ + +#define SPM_NO_MATCH UINT32_MAX + +/* Helper structure describing a particular search. */ +typedef struct SpmTestData_ { + const char *needle; + uint16_t needle_len; + const char *haystack; + uint16_t haystack_len; + int nocase; + uint32_t match_offset; /* offset in haystack, or SPM_NO_MATCH. */ +} SpmTestData; + +/* Helper function to conduct a search with a particular SPM matcher. */ +static int SpmTestSearch(const SpmTestData *d, uint16_t matcher) +{ + int ret = 1; + SpmGlobalThreadCtx *global_thread_ctx = NULL; + SpmThreadCtx *thread_ctx = NULL; + SpmCtx *ctx = NULL; + uint8_t *found = NULL; + + global_thread_ctx = SpmInitGlobalThreadCtx(matcher); + if (global_thread_ctx == NULL) { + ret = 0; + goto exit; + } + + ctx = SpmInitCtx((const uint8_t *)d->needle, d->needle_len, d->nocase, + global_thread_ctx); + if (ctx == NULL) { + ret = 0; + goto exit; + } + + thread_ctx = SpmMakeThreadCtx(global_thread_ctx); + if (thread_ctx == NULL) { + ret = 0; + goto exit; + } + + found = SpmScan(ctx, thread_ctx, (const uint8_t *)d->haystack, + d->haystack_len); + if (found == NULL) { + if (d->match_offset != SPM_NO_MATCH) { + printf(" should have matched at %" PRIu32 " but didn't\n", + d->match_offset); + ret = 0; + } + } else { + uint32_t offset = (uint32_t)(found - (const uint8_t *)d->haystack); + if (offset != d->match_offset) { + printf(" should have matched at %" PRIu32 + " but matched at %" PRIu32 "\n", + d->match_offset, offset); + ret = 0; + } + } + +exit: + SpmDestroyCtx(ctx); + SpmDestroyThreadCtx(thread_ctx); + SpmDestroyGlobalThreadCtx(global_thread_ctx); + return ret; +} + +int SpmSearchTest01() { + SpmTableSetup(); + printf("\n"); + + /* Each of the following tests will be run against every registered SPM + * algorithm. */ + + static const SpmTestData data[] = { + /* Some trivial single-character case/nocase tests */ + {"a", 1, "a", 1, 0, 0}, + {"a", 1, "A", 1, 1, 0}, + {"A", 1, "A", 1, 0, 0}, + {"A", 1, "a", 1, 1, 0}, + {"a", 1, "A", 1, 0, SPM_NO_MATCH}, + {"A", 1, "a", 1, 0, SPM_NO_MATCH}, + /* Nulls and odd characters */ + {"\x00", 1, "test\x00test", 9, 0, 4}, + {"\x00", 1, "testtest", 8, 0, SPM_NO_MATCH}, + {"\n", 1, "new line\n", 9, 0, 8}, + {"\n", 1, "new line\x00\n", 10, 0, 9}, + {"\xff", 1, "abcdef\xff", 7, 0, 6}, + {"\xff", 1, "abcdef\xff", 7, 1, 6}, + {"$", 1, "dollar$", 7, 0, 6}, + {"^", 1, "caret^", 6, 0, 5}, + /* Longer literals */ + {"Suricata", 8, "This is a Suricata test", 23, 0, 10}, + {"Suricata", 8, "This is a suricata test", 23, 1, 10}, + {"Suricata", 8, "This is a suriCATA test", 23, 1, 10}, + {"suricata", 8, "This is a Suricata test", 23, 0, SPM_NO_MATCH}, + {"Suricata", 8, "This is a Suricat_ test", 23, 0, SPM_NO_MATCH}, + {"Suricata", 8, "This is a _uricata test", 23, 0, SPM_NO_MATCH}, + /* First occurrence with the correct case should match */ + {"foo", 3, "foofoofoo", 9, 0, 0}, + {"foo", 3, "_foofoofoo", 9, 0, 1}, + {"FOO", 3, "foofoofoo", 9, 1, 0}, + {"FOO", 3, "_foofoofoo", 9, 1, 1}, + {"FOO", 3, "foo Foo FOo fOo foO FOO", 23, 0, 20}, + {"foo", 3, "Foo FOo fOo foO FOO foo", 23, 0, 20}, + }; + + int ret = 1; + + uint16_t matcher; + for (matcher = 0; matcher < SPM_TABLE_SIZE; matcher++) { + const SpmTableElmt *m = &spm_table[matcher]; + if (m->name == NULL) { + continue; + } + printf("matcher: %s\n", m->name); + + uint32_t i; + for (i = 0; i < sizeof(data)/sizeof(data[0]); i++) { + const SpmTestData *d = &data[i]; + if (SpmTestSearch(d, matcher) == 0) { + printf(" test %" PRIu32 ": fail\n", i); + ret = 0; + } + } + printf(" %" PRIu32 " tests passed\n", i); + } + + return ret; +} + +int SpmSearchTest02() { + SpmTableSetup(); + printf("\n"); + + /* Test that we can find needles of various lengths at various alignments + * in the haystack. Note that these are passed to strlen. */ + + static const char* needles[] = { + /* Single bytes */ + "a", "b", "c", ":", "/", "\x7f", "\xff", + /* Repeats */ + "aa", "aaa", "aaaaaaaaaaaaaaaaaaaaaaa", + /* Longer literals */ + "suricata", "meerkat", "aardvark", "raptor", "marmot", "lemming", + /* Mixed case */ + "Suricata", "CAPS LOCK", "mIxEd cAsE", + }; + + int ret = 1; + + uint16_t matcher; + for (matcher = 0; matcher < SPM_TABLE_SIZE; matcher++) { + const SpmTableElmt *m = &spm_table[matcher]; + if (m->name == NULL) { + continue; + } + printf("matcher: %s\n", m->name); + + SpmTestData d; + + uint32_t i; + for (i = 0; i < sizeof(needles) / sizeof(needles[0]); i++) { + const char *needle = needles[i]; + uint16_t prefix; + for (prefix = 0; prefix < 32; prefix++) { + d.needle = needle; + d.needle_len = strlen(needle); + uint16_t haystack_len = prefix + d.needle_len; + char *haystack = SCMalloc(haystack_len); + if (haystack == NULL) { + printf("alloc failure\n"); + return 0; + } + memset(haystack, ' ', haystack_len); + memcpy(haystack + prefix, d.needle, d.needle_len); + d.haystack = haystack; + d.haystack_len = haystack_len; + d.nocase = 0; + d.match_offset = prefix; + + /* Case-sensitive scan */ + if (SpmTestSearch(&d, matcher) == 0) { + printf(" test %" PRIu32 ": fail (case-sensitive)\n", i); + ret = 0; + } + + /* Case-insensitive scan */ + d.nocase = 1; + uint16_t j; + for (j = 0; j < haystack_len; j++) { + haystack[j] = toupper(haystack[j]); + } + if (SpmTestSearch(&d, matcher) == 0) { + printf(" test %" PRIu32 ": fail (case-insensitive)\n", i); + ret = 0; + } + + SCFree(haystack); + } + } + printf(" %" PRIu32 " tests passed\n", i); + } + + return ret; +} + #endif /* Register unittests */ @@ -2378,6 +2643,10 @@ void UtilSpmSearchRegistertests(void) UtRegisterTest("UtilSpmSearchOffsetsNocaseTest01", UtilSpmSearchOffsetsNocaseTest01); + /* new SPM API */ + UtRegisterTest("SpmSearchTest01", SpmSearchTest01); + UtRegisterTest("SpmSearchTest02", SpmSearchTest02); + #ifdef ENABLE_SEARCH_STATS /* Give some stats searching given a prepared context (look at the wrappers) */ UtRegisterTest("UtilSpmSearchStatsTest01", UtilSpmSearchStatsTest01); diff --git a/src/util-spm.h b/src/util-spm.h index 11149867a1..310285695c 100644 --- a/src/util-spm.h +++ b/src/util-spm.h @@ -31,10 +31,65 @@ enum { SPM_BM, /* Boyer-Moore */ /* Other SPM matchers will go here. */ + SPM_TABLE_SIZE }; uint16_t SinglePatternMatchDefaultMatcher(void); +/** Structure holding an immutable "built" SPM matcher (such as the Boyer-Moore + * tables, Hyperscan database etc) that is passed to the Scan call. */ +typedef struct SpmCtx_ { + uint16_t matcher; + void *ctx; +} SpmCtx; + +/** Structure holding a global prototype for per-thread scratch space, passed + * to each InitCtx call. */ +typedef struct SpmGlobalThreadCtx_ { + uint16_t matcher; + void *ctx; +} SpmGlobalThreadCtx; + +/** Structure holding some mutable per-thread space for use by a matcher at + * scan time. Constructed from SpmGlobalThreadCtx by the MakeThreadCtx call. */ +typedef struct SpmThreadCtx_ { + uint16_t matcher; + void *ctx; +} SpmThreadCtx; + +typedef struct SpmTableElmt_ { + const char *name; + SpmGlobalThreadCtx *(*InitGlobalThreadCtx)(void); + void (*DestroyGlobalThreadCtx)(SpmGlobalThreadCtx *g_thread_ctx); + SpmThreadCtx *(*MakeThreadCtx)(const SpmGlobalThreadCtx *g_thread_ctx); + void (*DestroyThreadCtx)(SpmThreadCtx *thread_ctx); + SpmCtx *(*InitCtx)(const uint8_t *needle, uint16_t needle_len, int nocase, + SpmGlobalThreadCtx *g_thread_ctx); + void (*DestroyCtx)(SpmCtx *); + uint8_t *(*Scan)(const SpmCtx *ctx, SpmThreadCtx *thread_ctx, + const uint8_t *haystack, uint16_t haystack_len); +} SpmTableElmt; + +SpmTableElmt spm_table[SPM_TABLE_SIZE]; + +void SpmTableSetup(void); + +SpmGlobalThreadCtx *SpmInitGlobalThreadCtx(uint16_t matcher); + +void SpmDestroyGlobalThreadCtx(SpmGlobalThreadCtx *g_thread_ctx); + +SpmThreadCtx *SpmMakeThreadCtx(const SpmGlobalThreadCtx *g_thread_ctx); + +void SpmDestroyThreadCtx(SpmThreadCtx *thread_ctx); + +SpmCtx *SpmInitCtx(const uint8_t *needle, uint16_t needle_len, int nocase, + SpmGlobalThreadCtx *g_thread_ctx); + +void SpmDestroyCtx(SpmCtx *ctx); + +uint8_t *SpmScan(const SpmCtx *ctx, SpmThreadCtx *thread_ctx, + const uint8_t *haystack, uint16_t haystack_len); + /** Default algorithm to use: Boyer Moore */ uint8_t *Bs2bmSearch(const uint8_t *text, uint32_t textlen, const uint8_t *needle, uint16_t needlelen); uint8_t *Bs2bmNocaseSearch(const uint8_t *text, uint32_t textlen, const uint8_t *needle, uint16_t needlelen);