From e19c41a80726f27672f9ba889ab9b0444cc09399 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 9 Apr 2015 18:20:28 +0200 Subject: [PATCH] multi-detect: hash lookup for tenants Use hash for storing and looking up det_ctxs. --- src/detect-engine.c | 53 ++++++++++++++++++++++++++++----------------- src/detect.c | 13 ++++++++--- src/detect.h | 5 +++++ 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/src/detect-engine.c b/src/detect-engine.c index d7d13d0cfd..db2c71bff3 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -1422,6 +1422,7 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForReload( return NULL; memset(det_ctx, 0, sizeof(DetectEngineThreadCtx)); + det_ctx->tenant_id = new_de_ctx->tenant_id; det_ctx->tv = tv; det_ctx->de_ctx = DetectEngineReference(new_de_ctx); if (det_ctx->de_ctx == NULL) { @@ -1454,19 +1455,6 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForReload( void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) { - if (det_ctx->mt_det_ctxs != NULL) { - uint32_t x; - for (x = 0; x < det_ctx->mt_det_ctxs_cnt; x++) { - if (det_ctx->mt_det_ctxs[x] == NULL) - continue; - - DetectEngineThreadCtxFree(det_ctx->mt_det_ctxs[x]); - det_ctx->mt_det_ctxs[x] = NULL; - } - SCFree(det_ctx->mt_det_ctxs); - det_ctx->mt_det_ctxs = NULL; - } - if (det_ctx->tenant_array != NULL) { SCFree(det_ctx->tenant_array); det_ctx->tenant_array = NULL; @@ -1558,6 +1546,10 @@ TmEcode DetectEngineThreadCtxDeinit(ThreadVars *tv, void *data) return TM_ECODE_OK; } + if (det_ctx->mt_det_ctxs_hash != NULL) { + HashTableFree(det_ctx->mt_det_ctxs_hash); + det_ctx->mt_det_ctxs_hash = NULL; + } DetectEngineThreadCtxFree(det_ctx); return TM_ECODE_OK; @@ -2235,6 +2227,24 @@ int DetectEngineReload(const char *filename) return 0; } +static uint32_t TenantIdHash(HashTable *h, void *data, uint16_t data_len) +{ + DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data; + return det_ctx->tenant_id % h->array_size; +} + +static char TenantIdCompare(void *d1, uint16_t d1_len, void *d2, uint16_t d2_len) +{ + DetectEngineThreadCtx *det1 = (DetectEngineThreadCtx *)d1; + DetectEngineThreadCtx *det2 = (DetectEngineThreadCtx *)d2; + return (det1->tenant_id == det2->tenant_id); +} + +static void TenantIdFree(void *d) +{ + DetectEngineThreadCtxFree(d); +} + /** NOTE: master MUST be locked before calling this */ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForMT(ThreadVars *tv) { @@ -2244,7 +2254,6 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForMT(ThreadVars *tv) uint32_t map_cnt = 0; int max_tenant_id = 0; DetectEngineCtx *list = master->list; - DetectEngineThreadCtx **tenant_det_ctxs = NULL; if (master->tenant_selector == TENANT_SELECTOR_UNKNOWN) { SCLogError(SC_ERR_MT_NO_SELECTOR, "no tenant selector set: " @@ -2252,13 +2261,18 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForMT(ThreadVars *tv) return NULL; } + uint32_t tcnt = 0; while (list) { if (list->tenant_id > max_tenant_id) max_tenant_id = list->tenant_id; list = list->next; + tcnt++; } + HashTable *mt_det_ctxs_hash = HashTableInit(tcnt * 2, TenantIdHash, TenantIdCompare, TenantIdFree); + BUG_ON(mt_det_ctxs_hash == NULL); + if (max_tenant_id == 0) { SCLogInfo("no tenants left, or none registered yet"); } else { @@ -2290,15 +2304,14 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForMT(ThreadVars *tv) } - tenant_det_ctxs = SCCalloc(max_tenant_id, sizeof(DetectEngineThreadCtx *)); - BUG_ON(tenant_det_ctxs == NULL); - + /* set up hash for tenant lookup */ list = master->list; while (list) { if (list->tenant_id != 0) { - tenant_det_ctxs[list->tenant_id] = DetectEngineThreadCtxInitForReload(tv, list); - if (tenant_det_ctxs[list->tenant_id] == NULL) + DetectEngineThreadCtx *mt_det_ctx = DetectEngineThreadCtxInitForReload(tv, list); + if (mt_det_ctx == NULL) goto error; + BUG_ON(HashTableAdd(mt_det_ctxs_hash, mt_det_ctx, 0) != 0); } list = list->next; } @@ -2308,6 +2321,7 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForMT(ThreadVars *tv) if (det_ctx == NULL) { goto error; } + det_ctx->mt_det_ctxs_hash = mt_det_ctxs_hash; /* first register the counter. In delayed detect mode we exit right after if the * rules haven't been loaded yet. */ @@ -2326,7 +2340,6 @@ static DetectEngineThreadCtx *DetectEngineThreadCtxInitForMT(ThreadVars *tv) det_ctx->counter_fnonmpm_list = counter_fnonmpm_list; det_ctx->counter_match_list = counter_match_list; #endif - det_ctx->mt_det_ctxs = tenant_det_ctxs; det_ctx->mt_det_ctxs_cnt = max_tenant_id; det_ctx->tenant_array = map_array; diff --git a/src/detect.c b/src/detect.c index b3f9339ffe..7d608a0a24 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1926,6 +1926,14 @@ void DetectSignatureApplyActions(Packet *p, const Signature *s) /* tm module api functions */ +static DetectEngineThreadCtx *GetTenantById(HashTable *h, uint32_t id) +{ + /* technically we need to pass a DetectEngineThreadCtx struct with the + * tentant_id member. But as that member is the first in the struct, we + * can use the id directly. */ + return HashTableLookup(h, &id, 0); +} + /** \brief Detection engine thread wrapper. * \param tv thread vars * \param p packet to inspect @@ -1979,15 +1987,14 @@ TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQue if (det_ctx->TenantGetId != NULL) { /* in MT mode, but no tenants registered yet */ - if (det_ctx->mt_det_ctxs == 0) { + if (det_ctx->mt_det_ctxs_cnt == 0) { return TM_ECODE_OK; } uint32_t tenant_id = det_ctx->TenantGetId(det_ctx, p); if (tenant_id > 0 && tenant_id < det_ctx->mt_det_ctxs_cnt) { p->tenant_id = tenant_id; - - det_ctx = det_ctx->mt_det_ctxs[tenant_id]; + det_ctx = GetTenantById(det_ctx->mt_det_ctxs_hash, tenant_id); if (det_ctx == NULL) return TM_ECODE_OK; de_ctx = det_ctx->de_ctx; diff --git a/src/detect.h b/src/detect.h index 985a8cf59a..2ebfb43f96 100644 --- a/src/detect.h +++ b/src/detect.h @@ -760,6 +760,10 @@ typedef struct FiledataReassembledBody_ { * Detection engine thread data. */ typedef struct DetectEngineThreadCtx_ { + /** \note multi-tenant hash lookup code from Detect() *depends* + * on this beeing the first member */ + uint32_t tenant_id; + /* the thread to which this detection engine thread belongs */ ThreadVars *tv; @@ -768,6 +772,7 @@ typedef struct DetectEngineThreadCtx_ { uint32_t mt_det_ctxs_cnt; struct DetectEngineThreadCtx_ **mt_det_ctxs; + HashTable *mt_det_ctxs_hash; struct DetectEngineTenantMapping_ *tenant_array; uint32_t tenant_array_size;