From cf9678d926f4e09d962ac1dee1cd808786ccf8cb Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 19 Dec 2016 11:25:27 +0100 Subject: [PATCH] detect: global registery for keyword thread data Some keywords need a scratch space where they can do store the results of expensive operations that remain valid for the time of a packets journey through the detection engine. An example is the reconstructed 'http_header' field, that is needed in MPM, and then for each rule that manually inspects it. Storing this data in the flow is a waste, and reconstructing multiple times on demand as well. This API allows for registering a keyword with an init and free function. It it mean to be used an initialization time, when the keyword is registered. --- src/detect-engine.c | 130 +++++++++++++++++++++++++++++++++++++++++++- src/detect-engine.h | 4 ++ src/detect.c | 1 + src/detect.h | 12 +++- 4 files changed, 144 insertions(+), 3 deletions(-) diff --git a/src/detect-engine.c b/src/detect-engine.c index c19d3f0ef8..d6a412f21e 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -91,7 +91,8 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForReload( static int DetectEngineCtxLoadConf(DetectEngineCtx *); -static DetectEngineMasterCtx g_master_de_ctx = { SCMUTEX_INITIALIZER, 0, NULL, NULL, TENANT_SELECTOR_UNKNOWN, NULL,}; +static DetectEngineMasterCtx g_master_de_ctx = { SCMUTEX_INITIALIZER, + 0, NULL, NULL, TENANT_SELECTOR_UNKNOWN, NULL, NULL, 0}; static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len); static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len); @@ -1387,6 +1388,62 @@ void DetectEngineResetMaxSigId(DetectEngineCtx *de_ctx) de_ctx->signum = 0; } +static int DetectEngineThreadCtxInitGlobalKeywords(DetectEngineThreadCtx *det_ctx) +{ + DetectEngineMasterCtx *master = &g_master_de_ctx; + SCMutexLock(&master->lock); + + if (master->keyword_id > 0) { + det_ctx->global_keyword_ctxs_array = SCCalloc(master->keyword_id, sizeof(void *)); + if (det_ctx->global_keyword_ctxs_array == NULL) { + SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx"); + goto error; + } + det_ctx->global_keyword_ctxs_size = master->keyword_id; + + DetectEngineThreadKeywordCtxItem *item = master->keyword_list; + while (item) { + det_ctx->global_keyword_ctxs_array[item->id] = item->InitFunc(item->data); + if (det_ctx->global_keyword_ctxs_array[item->id] == NULL) { + SCLogError(SC_ERR_DETECT_PREPARE, "setting up thread local detect ctx " + "for keyword \"%s\" failed", item->name); + goto error; + } + item = item->next; + } + } + SCMutexUnlock(&master->lock); + return TM_ECODE_OK; +error: + SCMutexUnlock(&master->lock); + return TM_ECODE_FAILED; +} + +static void DetectEngineThreadCtxDeinitGlobalKeywords(DetectEngineThreadCtx *det_ctx) +{ + if (det_ctx->global_keyword_ctxs_array == NULL || + det_ctx->global_keyword_ctxs_size == 0) { + return; + } + + DetectEngineMasterCtx *master = &g_master_de_ctx; + SCMutexLock(&master->lock); + + if (master->keyword_id > 0) { + DetectEngineThreadKeywordCtxItem *item = master->keyword_list; + while (item) { + if (det_ctx->global_keyword_ctxs_array[item->id] != NULL) + item->FreeFunc(det_ctx->global_keyword_ctxs_array[item->id]); + + item = item->next; + } + det_ctx->global_keyword_ctxs_size = 0; + SCFree(det_ctx->global_keyword_ctxs_array); + det_ctx->global_keyword_ctxs_array = NULL; + } + SCMutexUnlock(&master->lock); +} + static int DetectEngineThreadCtxInitKeywords(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx) { if (de_ctx->keyword_id > 0) { @@ -1605,6 +1662,7 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx * } DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx); + DetectEngineThreadCtxInitGlobalKeywords(det_ctx); #ifdef PROFILING SCProfilingRuleThreadSetup(de_ctx->profile_ctx, det_ctx); SCProfilingKeywordThreadSetup(de_ctx->profile_keyword_ctx, det_ctx); @@ -1823,6 +1881,7 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) SCFree(det_ctx->base64_decoded); } + DetectEngineThreadCtxDeinitGlobalKeywords(det_ctx); if (det_ctx->de_ctx != NULL) { DetectEngineThreadCtxDeinitKeywords(det_ctx->de_ctx, det_ctx); #ifdef UNITTESTS @@ -1924,6 +1983,75 @@ void *DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id) return det_ctx->keyword_ctxs_array[id]; } + +/** \brief Register Thread keyword context Funcs (Global) + * + * IDs stay static over reloads and between tenants + * + * \param name keyword name for error printing + * \param InitFunc function ptr + * \param FreeFunc function ptr + * + * \retval id for retrieval of ctx at runtime + * \retval -1 on error + */ +int DetectRegisterThreadCtxGlobalFuncs(const char *name, + void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *)) +{ + int id; + BUG_ON(InitFunc == NULL || FreeFunc == NULL); + + DetectEngineMasterCtx *master = &g_master_de_ctx; + SCMutexLock(&master->lock); + + /* if already registered, return existing id */ + DetectEngineThreadKeywordCtxItem *item = master->keyword_list; + while (item != NULL) { + if (strcmp(name, item->name) == 0) { + id = item->id; + SCMutexUnlock(&master->lock); + return id; + } + + item = item->next; + } + + item = SCCalloc(1, sizeof(*item)); + if (unlikely(item == NULL)) + return -1; + + item->InitFunc = InitFunc; + item->FreeFunc = FreeFunc; + item->name = name; + item->data = data; + + item->next = master->keyword_list; + master->keyword_list = item; + item->id = master->keyword_id++; + + id = item->id; + SCMutexUnlock(&master->lock); + return id; +} + +/** \brief Retrieve thread local keyword ctx by id + * + * \param det_ctx detection engine thread ctx to retrieve the ctx from + * \param id id of the ctx returned by DetectRegisterThreadCtxInitFunc at + * keyword init. + * + * \retval ctx or NULL on error + */ +void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id) +{ + if (id < 0 || id > det_ctx->global_keyword_ctxs_size || + det_ctx->global_keyword_ctxs_array == NULL) { + return NULL; + } + + return det_ctx->global_keyword_ctxs_array[id]; +} + /** \brief Check if detection is enabled * \retval bool true or false */ int DetectEngineEnabled(void) diff --git a/src/detect-engine.h b/src/detect-engine.h index a29db7a564..9c1c1614a5 100644 --- a/src/detect-engine.h +++ b/src/detect-engine.h @@ -53,6 +53,10 @@ DetectEngineCtx *DetectEngineCtxInit(void); DetectEngineCtx *DetectEngineCtxInitMinimal(void); void DetectEngineCtxFree(DetectEngineCtx *); +int DetectRegisterThreadCtxGlobalFuncs(const char *name, + void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *)); +void *DetectThreadCtxGetGlobalKeywordThreadCtx(DetectEngineThreadCtx *det_ctx, int id); + TmEcode DetectEngineThreadCtxInit(ThreadVars *, void *, void **); TmEcode DetectEngineThreadCtxDeinit(ThreadVars *, void *); //inline uint32_t DetectEngineGetMaxSigId(DetectEngineCtx *); diff --git a/src/detect.c b/src/detect.c index cb21cbf9e3..0bb8167902 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1049,6 +1049,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh SCLogDebug("pcap_cnt %"PRIu64, p->pcap_cnt); + det_ctx->ticker++; p->alerts.cnt = 0; det_ctx->filestore_cnt = 0; det_ctx->smsg = NULL; diff --git a/src/detect.h b/src/detect.h index 86895a5f3b..c1321914dc 100644 --- a/src/detect.h +++ b/src/detect.h @@ -778,6 +778,9 @@ typedef struct DetectEngineThreadCtx_ { * on this beeing the first member */ uint32_t tenant_id; + /** ticker that is incremented once per packet. */ + uint64_t ticker; + /* the thread to which this detection engine thread belongs */ ThreadVars *tv; @@ -900,10 +903,12 @@ typedef struct DetectEngineThreadCtx_ { } filestore[DETECT_FILESTORE_MAX]; DetectEngineCtx *de_ctx; - /** store for keyword contexts that need a per thread storage because of - * thread safety issues */ + /** store for keyword contexts that need a per thread storage. Per de_ctx. */ void **keyword_ctxs_array; int keyword_ctxs_size; + /** store for keyword contexts that need a per thread storage. Global. */ + int global_keyword_ctxs_size; + void **global_keyword_ctxs_array; uint8_t *base64_decoded; int base64_decoded_len; @@ -1146,6 +1151,9 @@ typedef struct DetectEngineMasterCtx_ { * structures. */ DetectEngineTenantMapping *tenant_mapping_list; + /** list of keywords that need thread local ctxs */ + DetectEngineThreadKeywordCtxItem *keyword_list; + int keyword_id; } DetectEngineMasterCtx; /** \brief Signature loader statistics */