diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 1faa3fe192..776622ab6e 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -933,24 +933,24 @@ static void PopulateMpmHelperAddPatternToPktCtx(MpmCtx *mpm_ctx, MpmAddPatternCI(mpm_ctx, cd->content + cd->fp_chop_offset, cd->fp_chop_len, pat_offset, pat_depth, - cd->id, s->num, flags); + cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID); } else { MpmAddPatternCI(mpm_ctx, cd->content, cd->content_len, pat_offset, pat_depth, - cd->id, s->num, flags); + cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID); } } else { if (chop) { MpmAddPatternCS(mpm_ctx, cd->content + cd->fp_chop_offset, cd->fp_chop_len, pat_offset, pat_depth, - cd->id, s->num, flags); + cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID); } else { MpmAddPatternCS(mpm_ctx, cd->content, cd->content_len, pat_offset, pat_depth, - cd->id, s->num, flags); + cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID); } } diff --git a/src/util-mpm-ac-bs.c b/src/util-mpm-ac-bs.c index 6883f56724..f8e313a175 100644 --- a/src/util-mpm-ac-bs.c +++ b/src/util-mpm-ac-bs.c @@ -77,8 +77,6 @@ void SCACBSRegisterTests(void); /* a placeholder to denote a failure transition in the goto table */ #define SC_AC_BS_FAIL (-1) -/* size of the hash table used to speed up pattern insertions initially */ -#define INIT_HASH_SIZE 65536 #define STATE_QUEUE_CONTAINER_SIZE 65536 @@ -132,276 +130,6 @@ static void SCACBSGetConfig() return; } -/** - * \internal - * \brief Creates a hash of the pattern. We use it for the hashing process - * during the initial pattern insertion time, to cull duplicate sigs. - * - * \param pat Pointer to the pattern. - * \param patlen Pattern length. - * - * \retval hash A 32 bit unsigned hash. - */ -static inline uint32_t SCACBSInitHashRaw(uint8_t *pat, uint16_t patlen) -{ - uint32_t hash = patlen * pat[0]; - if (patlen > 1) - hash += pat[1]; - - return (hash % INIT_HASH_SIZE); -} - -/** - * \internal - * \brief Looks up a pattern. We use it for the hashing process during the - * the initial pattern insertion time, to cull duplicate sigs. - * - * \param ctx Pointer to the AC ctx. - * \param pat Pointer to the pattern. - * \param patlen Pattern length. - * \param flags Flags. We don't need this. - * - * \retval hash A 32 bit unsigned hash. - */ -static inline SCACBSPattern *SCACBSInitHashLookup(SCACBSCtx *ctx, uint8_t *pat, - uint16_t patlen, char flags, - uint32_t pid) -{ - uint32_t hash = SCACBSInitHashRaw(pat, patlen); - - if (ctx->init_hash == NULL) { - return NULL; - } - - SCACBSPattern *t = ctx->init_hash[hash]; - for ( ; t != NULL; t = t->next) { - if (t->id == pid) - return t; - } - - return NULL; -} - -/** - * \internal - * \brief Allocs a new pattern instance. - * - * \param mpm_ctx Pointer to the mpm context. - * - * \retval p Pointer to the newly created pattern. - */ -static inline SCACBSPattern *SCACBSAllocPattern(MpmCtx *mpm_ctx) -{ - SCACBSPattern *p = SCMalloc(sizeof(SCACBSPattern)); - if (unlikely(p == NULL)) { - exit(EXIT_FAILURE); - } - memset(p, 0, sizeof(SCACBSPattern)); - - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += sizeof(SCACBSPattern); - - return p; -} - -/** - * \internal - * \brief Used to free SCACBSPattern instances. - * - * \param mpm_ctx Pointer to the mpm context. - * \param p Pointer to the SCACBSPattern instance to be freed. - */ -static inline void SCACBSFreePattern(MpmCtx *mpm_ctx, SCACBSPattern *p) -{ - if (p != NULL && p->cs != NULL && p->cs != p->ci) { - SCFree(p->cs); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL && p->ci != NULL) { - SCFree(p->ci); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL && p->original_pat != NULL) { - SCFree(p->original_pat); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL) { - SCFree(p); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= sizeof(SCACBSPattern); - } - return; -} - -static inline uint32_t SCACBSInitHash(SCACBSPattern *p) -{ - uint32_t hash = p->len * p->original_pat[0]; - if (p->len > 1) - hash += p->original_pat[1]; - - return (hash % INIT_HASH_SIZE); -} - -static inline int SCACBSInitHashAdd(SCACBSCtx *ctx, SCACBSPattern *p) -{ - uint32_t hash = SCACBSInitHash(p); - - if (ctx->init_hash == NULL) { - return 0; - } - - if (ctx->init_hash[hash] == NULL) { - ctx->init_hash[hash] = p; - return 0; - } - - SCACBSPattern *tt = NULL; - SCACBSPattern *t = ctx->init_hash[hash]; - - /* get the list tail */ - do { - tt = t; - t = t->next; - } while (t != NULL); - - tt->next = p; - - return 0; -} - -/** - * \internal - * \brief Add a pattern to the mpm-ac context. - * - * \param mpm_ctx Mpm context. - * \param pat Pointer to the pattern. - * \param patlen Length of the pattern. - * \param pid Pattern id - * \param sid Signature id (internal id). - * \param flags Pattern's MPM_PATTERN_* flags. - * - * \retval 0 On success. - * \retval -1 On failure. - */ -static int SCACBSAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, - uint16_t offset, uint16_t depth, uint32_t pid, - SigIntId sid, uint8_t flags) -{ - SCACBSCtx *ctx = (SCACBSCtx *)mpm_ctx->ctx; - - SCLogDebug("Adding pattern for ctx %p, patlen %"PRIu16" and pid %" PRIu32, - ctx, patlen, pid); - - if (patlen == 0) { - SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0"); - return 0; - } - - /* check if we have already inserted this pattern */ - SCACBSPattern *p = SCACBSInitHashLookup(ctx, pat, patlen, flags, pid); - if (p == NULL) { - SCLogDebug("Allocing new pattern"); - - /* p will never be NULL */ - p = SCACBSAllocPattern(mpm_ctx); - - p->len = patlen; - p->flags = flags; - p->id = pid; - - p->original_pat = SCMalloc(patlen); - if (p->original_pat == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy(p->original_pat, pat, patlen); - - p->ci = SCMalloc(patlen); - if (p->ci == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy_tolower(p->ci, pat, patlen); - - /* setup the case sensitive part of the pattern */ - if (p->flags & MPM_PATTERN_FLAG_NOCASE) { - /* nocase means no difference between cs and ci */ - p->cs = p->ci; - } else { - if (memcmp(p->ci, pat, p->len) == 0) { - /* no diff between cs and ci: pat is lowercase */ - p->cs = p->ci; - } else { - p->cs = SCMalloc(patlen); - if (p->cs == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy(p->cs, pat, patlen); - } - } - - /* put in the pattern hash */ - SCACBSInitHashAdd(ctx, p); - - //if (mpm_ctx->pattern_cnt == 65535) { - // SCLogError(SC_ERR_AHO_CORASICK, "Max search words reached. Can't " - // "insert anymore. Exiting"); - // exit(EXIT_FAILURE); - //} - mpm_ctx->pattern_cnt++; - - if (mpm_ctx->maxlen < patlen) - mpm_ctx->maxlen = patlen; - - if (mpm_ctx->minlen == 0) { - mpm_ctx->minlen = patlen; - } else { - if (mpm_ctx->minlen > patlen) - mpm_ctx->minlen = patlen; - } - - /* we need the max pat id */ - if (pid > ctx->max_pat_id) - ctx->max_pat_id = pid; - - p->sids_size = 1; - p->sids = SCMalloc(p->sids_size * sizeof(SigIntId)); - BUG_ON(p->sids == NULL); - p->sids[0] = sid; - } else { - /* TODO figure out how we can be called multiple times for the same CTX with the same sid */ - - int found = 0; - uint32_t x = 0; - for (x = 0; x < p->sids_size; x++) { - if (p->sids[x] == sid) { - found = 1; - break; - } - } - if (!found) { - SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1))); - BUG_ON(sids == NULL); - p->sids = sids; - p->sids[p->sids_size] = sid; - p->sids_size++; - } - } - - return 0; - -error: - SCACBSFreePattern(mpm_ctx, p); - return -1; -} - /** * \internal * \brief Initialize a new state in the goto and output tables. @@ -1167,24 +895,24 @@ int SCACBSPreparePatterns(MpmCtx *mpm_ctx) { SCACBSCtx *ctx = (SCACBSCtx *)mpm_ctx->ctx; - if (mpm_ctx->pattern_cnt == 0 || ctx->init_hash == NULL) { + if (mpm_ctx->pattern_cnt == 0 || mpm_ctx->init_hash == NULL) { SCLogDebug("no patterns supplied to this mpm_ctx"); return 0; } /* alloc the pattern array */ - ctx->parray = (SCACBSPattern **)SCMalloc(mpm_ctx->pattern_cnt * - sizeof(SCACBSPattern *)); + ctx->parray = (MpmPattern **)SCMalloc(mpm_ctx->pattern_cnt * + sizeof(MpmPattern *)); if (ctx->parray == NULL) goto error; - memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(SCACBSPattern *)); + memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(SCACBSPattern *)); + mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); /* populate it with the patterns in the hash */ uint32_t i = 0, p = 0; - for (i = 0; i < INIT_HASH_SIZE; i++) { - SCACBSPattern *node = ctx->init_hash[i], *nnode = NULL; + for (i = 0; i < MPM_INIT_HASH_SIZE; i++) { + MpmPattern *node = mpm_ctx->init_hash[i], *nnode = NULL; while(node != NULL) { nnode = node->next; node->next = NULL; @@ -1194,19 +922,19 @@ int SCACBSPreparePatterns(MpmCtx *mpm_ctx) } /* we no longer need the hash, so free it's memory */ - SCFree(ctx->init_hash); - ctx->init_hash = NULL; + SCFree(mpm_ctx->init_hash); + mpm_ctx->init_hash = NULL; /* the memory consumed by a single state in our goto table */ ctx->single_state_size = sizeof(int32_t) * 256; /* handle no case patterns */ - ctx->pid_pat_list = SCMalloc((ctx->max_pat_id + 1)* sizeof(SCACBSPatternList)); + ctx->pid_pat_list = SCMalloc((mpm_ctx->max_pat_id + 1)* sizeof(SCACBSPatternList)); if (ctx->pid_pat_list == NULL) { SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory"); exit(EXIT_FAILURE); } - memset(ctx->pid_pat_list, 0, (ctx->max_pat_id + 1) * sizeof(SCACBSPatternList)); + memset(ctx->pid_pat_list, 0, (mpm_ctx->max_pat_id + 1) * sizeof(SCACBSPatternList)); for (i = 0; i < mpm_ctx->pattern_cnt; i++) { if (!(ctx->parray[i]->flags & MPM_PATTERN_FLAG_NOCASE)) { @@ -1231,15 +959,15 @@ int SCACBSPreparePatterns(MpmCtx *mpm_ctx) /* free all the stored patterns. Should save us a good 100-200 mbs */ for (i = 0; i < mpm_ctx->pattern_cnt; i++) { if (ctx->parray[i] != NULL) { - SCACBSFreePattern(mpm_ctx, ctx->parray[i]); + MpmFreePattern(mpm_ctx, ctx->parray[i]); } } SCFree(ctx->parray); ctx->parray = NULL; mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(SCACBSPattern *)); + mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); - ctx->pattern_id_bitarray_size = (ctx->max_pat_id / 8) + 1; + ctx->pattern_id_bitarray_size = (mpm_ctx->max_pat_id / 8) + 1; return 0; @@ -1291,12 +1019,11 @@ void SCACBSInitCtx(MpmCtx *mpm_ctx) mpm_ctx->memory_size += sizeof(SCACBSCtx); /* initialize the hash we use to speed up pattern insertions */ - SCACBSCtx *ctx = (SCACBSCtx *)mpm_ctx->ctx; - ctx->init_hash = SCMalloc(sizeof(SCACBSPattern *) * INIT_HASH_SIZE); - if (ctx->init_hash == NULL) { + mpm_ctx->init_hash = SCMalloc(sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE); + if (mpm_ctx->init_hash == NULL) { exit(EXIT_FAILURE); } - memset(ctx->init_hash, 0, sizeof(SCACBSPattern *) * INIT_HASH_SIZE); + memset(mpm_ctx->init_hash, 0, sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE); /* get conf values for AC from our yaml file. We have no conf values for * now. We will certainly need this, as we develop the algo */ @@ -1336,25 +1063,25 @@ void SCACBSDestroyCtx(MpmCtx *mpm_ctx) if (ctx == NULL) return; - if (ctx->init_hash != NULL) { - SCFree(ctx->init_hash); - ctx->init_hash = NULL; + if (mpm_ctx->init_hash != NULL) { + SCFree(mpm_ctx->init_hash); + mpm_ctx->init_hash = NULL; mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (INIT_HASH_SIZE * sizeof(SCACBSPattern *)); + mpm_ctx->memory_size -= (MPM_INIT_HASH_SIZE * sizeof(MpmPattern *)); } if (ctx->parray != NULL) { uint32_t i; for (i = 0; i < mpm_ctx->pattern_cnt; i++) { if (ctx->parray[i] != NULL) { - SCACBSFreePattern(mpm_ctx, ctx->parray[i]); + MpmFreePattern(mpm_ctx, ctx->parray[i]); } } SCFree(ctx->parray); ctx->parray = NULL; mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(SCACBSPattern *)); + mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); } if (ctx->state_table_u16 != NULL) { @@ -1385,7 +1112,7 @@ void SCACBSDestroyCtx(MpmCtx *mpm_ctx) if (ctx->pid_pat_list != NULL) { uint32_t i; - for (i = 0; i < (ctx->max_pat_id + 1); i++) { + for (i = 0; i < (mpm_ctx->max_pat_id + 1); i++) { if (ctx->pid_pat_list[i].cs != NULL) SCFree(ctx->pid_pat_list[i].cs); if (ctx->pid_pat_list[i].sids != NULL) @@ -1629,7 +1356,7 @@ int SCACBSAddPatternCI(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, SigIntId sid, uint8_t flags) { flags |= MPM_PATTERN_FLAG_NOCASE; - return SCACBSAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); + return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); } /** @@ -1653,7 +1380,7 @@ int SCACBSAddPatternCS(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, SigIntId sid, uint8_t flags) { - return SCACBSAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); + return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); } void SCACBSPrintSearchStats(MpmThreadCtx *mpm_thread_ctx) @@ -1679,8 +1406,8 @@ void SCACBSPrintInfo(MpmCtx *mpm_ctx) printf(" Sizeof:\n"); printf(" MpmCtx %" PRIuMAX "\n", (uintmax_t)sizeof(MpmCtx)); printf(" SCACBSCtx: %" PRIuMAX "\n", (uintmax_t)sizeof(SCACBSCtx)); - printf(" SCACBSPattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACBSPattern)); - printf(" SCACBSPattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACBSPattern)); + printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern)); + printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern)); printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt); printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen); printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen); diff --git a/src/util-mpm-ac-bs.h b/src/util-mpm-ac-bs.h index 42d254c585..80ac0f4b06 100644 --- a/src/util-mpm-ac-bs.h +++ b/src/util-mpm-ac-bs.h @@ -22,30 +22,11 @@ * */ +#include "util-mpm.h" + #define SC_AC_BS_STATE_TYPE_U16 uint16_t #define SC_AC_BS_STATE_TYPE_U32 uint32_t -typedef struct SCACBSPattern_ { - /* length of the pattern */ - uint16_t len; - /* flags decribing the pattern */ - uint8_t flags; - /* holds the original pattern that was added */ - uint8_t *original_pat; - /* case sensitive */ - uint8_t *cs; - /* case INsensitive */ - uint8_t *ci; - /* pattern id */ - uint32_t id; - - /* sid(s) for this pattern */ - uint32_t sids_size; - SigIntId *sids; - - struct SCACBSPattern_ *next; -} SCACBSPattern; - typedef struct SCACBSPatternList_ { uint8_t *cs; uint16_t patlen; @@ -63,16 +44,13 @@ typedef struct SCACBSOutputTable_ { } SCACBSOutputTable; typedef struct SCACBSCtx_ { - /* hash used during ctx initialization */ - SCACBSPattern **init_hash; - /* pattern arrays. We need this only during the goto table creation phase */ - SCACBSPattern **parray; + MpmPattern **parray; /* no of states used by ac */ uint32_t state_count; - uint16_t pattern_id_bitarray_size; + uint32_t pattern_id_bitarray_size; /* the all important memory hungry state_table */ SC_AC_BS_STATE_TYPE_U16 (*state_table_u16)[256]; @@ -91,7 +69,6 @@ typedef struct SCACBSCtx_ { /* the size of each state */ uint16_t single_state_size; - uint32_t max_pat_id; } SCACBSCtx; typedef struct SCACBSThreadCtx_ { diff --git a/src/util-mpm-ac-tile.c b/src/util-mpm-ac-tile.c index 663eb66071..72b2936962 100644 --- a/src/util-mpm-ac-tile.c +++ b/src/util-mpm-ac-tile.c @@ -143,8 +143,6 @@ static void SCACTileDestroyInitCtx(MpmCtx *mpm_ctx); /* a placeholder to denote a failure transition in the goto table */ #define SC_AC_TILE_FAIL (-1) -/* size of the hash table used to speed up pattern insertions initially */ -#define INIT_HASH_SIZE 65536 #define STATE_QUEUE_CONTAINER_SIZE 65536 @@ -167,193 +165,6 @@ static void SCACTileGetConfig() { } -/** - * \internal - * \brief Compares 2 patterns. We use it for the hashing process during the - * the initial pattern insertion time, to cull duplicate sigs. - * - * \param p Pointer to the first pattern(SCACTilePattern). - * \param pat Pointer to the second pattern(raw pattern array). - * \param patlen Pattern length. - * \param flags Flags. We don't need this. - * - * \retval hash A 32 bit unsigned hash. - */ -static inline int SCACTileCmpPattern(SCACTilePattern *p, uint8_t *pat, - uint16_t patlen, char flags) -{ - if (p->len != patlen) - return 0; - - if (p->flags != flags) - return 0; - - if (flags & MPM_PATTERN_FLAG_NOCASE) { - // Case insensitive - if (SCMemcmpLowercase(p->cs, pat, patlen) != 0) - return 0; - - } else { - // Case sensitive - if (SCMemcmp(p->cs, pat, patlen) != 0) - return 0; - } - - return 1; -} - -/** - * \internal - * \brief Creates a hash of the pattern. We use it for the hashing process - * during the initial pattern insertion time, to cull duplicate sigs. - * - * \param pat Pointer to the pattern. - * \param patlen Pattern length. - * - * \retval hash A 32 bit unsigned hash. - */ -static inline uint32_t SCACTileInitHashRaw(uint8_t *pat, uint16_t patlen) -{ - uint32_t hash = patlen * pat[0]; - if (patlen > 1) - hash += pat[1]; - - return (hash % INIT_HASH_SIZE); -} - -/** - * \internal - * \brief Looks up a pattern. We use it for the hashing process during the - * the initial pattern insertion time, to cull duplicate sigs. - * - * \param ctx Pointer to the AC ctx. - * \param pat Pointer to the pattern. - * \param patlen Pattern length. - * \param flags Flags. We don't need this. - * - * \retval hash A 32 bit unsigned hash. - */ -static inline SCACTilePattern *SCACTileInitHashLookup(SCACTileCtx *ctx, - uint8_t *pat, - uint16_t patlen, - char flags, - uint32_t pid) -{ - uint32_t hash = SCACTileInitHashRaw(pat, patlen); - - if (ctx->init_hash == NULL) { - return NULL; - } - - SCACTilePattern *t = ctx->init_hash[hash]; - for ( ; t != NULL; t = t->next) { - if (t->id == pid) { - return t; - } - } - - return NULL; -} - -/** - * \internal - * \brief Allocs a new pattern instance. - * - * \param mpm_ctx Pointer to the mpm context. - * - * \retval p Pointer to the newly created pattern. - */ -static inline SCACTilePattern *SCACTileAllocPattern(MpmCtx *mpm_ctx) -{ - SCACTilePattern *p = SCMalloc(sizeof(SCACTilePattern)); - if (unlikely(p == NULL)) { - exit(EXIT_FAILURE); - } - memset(p, 0, sizeof(SCACTilePattern)); - - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += sizeof(SCACTilePattern); - - return p; -} - -/** - * \internal - * \brief Used to free SCACTilePattern instances. - * - * \param mpm_ctx Pointer to the mpm context. - * \param p Pointer to the SCACTilePattern instance to be freed. - * \param free Free the above pointer or not. - */ -static void SCACTileFreePattern(MpmCtx *mpm_ctx, SCACTilePattern *p) -{ - if (p != NULL && p->cs != NULL && p->cs != p->ci) { - SCFree(p->cs); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL && p->ci != NULL) { - SCFree(p->ci); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL && p->original_pat != NULL) { - SCFree(p->original_pat); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL && p->sids != NULL) { - SCFree(p->sids); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (p->sids_size * sizeof(SigIntId)); - } - - if (p != NULL) { - SCFree(p); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= sizeof(SCACTilePattern); - } -} - -static inline uint32_t SCACTileInitHash(SCACTilePattern *p) -{ - uint32_t hash = p->len * p->original_pat[0]; - if (p->len > 1) - hash += p->original_pat[1]; - - return (hash % INIT_HASH_SIZE); -} - -static inline int SCACTileInitHashAdd(SCACTileCtx *ctx, SCACTilePattern *p) -{ - uint32_t hash = SCACTileInitHash(p); - - if (ctx->init_hash == NULL) { - return 0; - } - - if (ctx->init_hash[hash] == NULL) { - ctx->init_hash[hash] = p; - return 0; - } - - SCACTilePattern *tt = NULL; - SCACTilePattern *t = ctx->init_hash[hash]; - - /* get the list tail */ - do { - tt = t; - t = t->next; - } while (t != NULL); - - tt->next = p; - - return 0; -} - /** * \internal @@ -362,7 +173,7 @@ static inline int SCACTileInitHashAdd(SCACTileCtx *ctx, SCACTilePattern *p) * characters, so could just set to 1 instead of counting. */ static inline void SCACTileHistogramAlphabet(SCACTileCtx *ctx, - SCACTilePattern *p) + MpmPattern *p) { for (int i = 0; i < p->len; i++) { ctx->alpha_hist[p->ci[i]]++; @@ -413,137 +224,6 @@ static void SCACTileInitTranslateTable(SCACTileCtx *ctx) ctx->alphabet_storage = 256; } -/** - * \internal - * \brief Add a pattern to the mpm-ac context. - * - * \param mpm_ctx Mpm context. - * \param pat Pointer to the pattern. - * \param patlen Length of the pattern. - * \param pid Pattern id - * \param sid Signature id (internal id). - * \param flags Pattern's MPM_PATTERN_* flags. - * - * \retval 0 On success. - * \retval -1 On failure. - */ -static int SCACTileAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, - uint16_t offset, uint16_t depth, uint32_t pid, - uint32_t sid, uint8_t flags) -{ - SCACTileSearchCtx *search_ctx = (SCACTileSearchCtx *)mpm_ctx->ctx; - SCACTileCtx *ctx = search_ctx->init_ctx; - - SCLogDebug("Adding pattern for ctx %p, patlen %"PRIu16" and pid %" PRIu32, - ctx, patlen, pid); - - if (patlen == 0) { - SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0"); - return 0; - } - - /* Check if we have already inserted this pattern. */ - SCACTilePattern *p = SCACTileInitHashLookup(ctx, pat, patlen, flags, pid); - if (p == NULL) { - SCLogDebug("Allocing new pattern"); - - /* p will never be NULL */ - p = SCACTileAllocPattern(mpm_ctx); - - p->len = patlen; - p->flags = flags; - p->id = pid; - - p->original_pat = SCMalloc(patlen); - if (p->original_pat == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy(p->original_pat, pat, patlen); - - p->ci = SCMalloc(patlen); - if (p->ci == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy_tolower(p->ci, pat, patlen); - - /* setup the case sensitive part of the pattern */ - if (p->flags & MPM_PATTERN_FLAG_NOCASE) { - /* nocase means no difference between cs and ci */ - p->cs = p->ci; - } else { - if (memcmp(p->ci, pat, p->len) == 0) { - /* no diff between cs and ci: pat is lowercase */ - p->cs = p->ci; - } else { - p->cs = SCMalloc(patlen); - if (p->cs == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy(p->cs, pat, patlen); - } - } - - /* put in the pattern hash */ - SCACTileInitHashAdd(ctx, p); - /* Count alphabet usages */ - SCACTileHistogramAlphabet(ctx, p); - - //if (mpm_ctx->pattern_cnt == 65535) { - // SCLogError(SC_ERR_AHO_CORASICK, "Max search words reached. Can't " - // "insert anymore. Exiting"); - // exit(EXIT_FAILURE); - //} - mpm_ctx->pattern_cnt++; - - if (mpm_ctx->maxlen < patlen) - mpm_ctx->maxlen = patlen; - - if (mpm_ctx->minlen == 0) { - mpm_ctx->minlen = patlen; - } else { - if (mpm_ctx->minlen > patlen) - mpm_ctx->minlen = patlen; - } - - p->sids_size = 1; - p->sids = SCMalloc(p->sids_size * sizeof(SigIntId)); - BUG_ON(p->sids == NULL); - p->sids[0] = sid; - //SCLogInfo("MPM added %u:%u", pid, sid); - } else { - /* TODO figure out how we can be called multiple times for the same CTX with the same sid */ - - int found = 0; - uint32_t x = 0; - for (x = 0; x < p->sids_size; x++) { - if (p->sids[x] == sid) { - found = 1; - break; - } - } - if (!found) { - SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1))); - BUG_ON(sids == NULL); - p->sids = sids; - p->sids[p->sids_size] = sid; - p->sids_size++; - //SCLogInfo("p->sids_size %u", p->sids_size); - //SCLogInfo("MPM added %u:%u (append)", pid, sid); - } else { - //SCLogInfo("rule %u already part of pid %u", sid, pid); - } - } - - return 0; - -error: - SCACTileFreePattern(mpm_ctx, p); - return -1; -} - static void SCACTileReallocOutputTable(SCACTileCtx *ctx, int new_state_count) { @@ -1198,33 +878,34 @@ int SCACTilePreparePatterns(MpmCtx *mpm_ctx) return 0; } SCACTileCtx *ctx = search_ctx->init_ctx; - if (ctx->init_hash == NULL) { + if (mpm_ctx->init_hash == NULL) { SCLogDebug("no patterns supplied to this mpm_ctx"); return 0; } /* alloc the pattern array */ - ctx->parray = (SCACTilePattern **)SCMalloc(mpm_ctx->pattern_cnt * - sizeof(SCACTilePattern *)); + ctx->parray = (MpmPattern **)SCMalloc(mpm_ctx->pattern_cnt * + sizeof(MpmPattern *)); if (ctx->parray == NULL) goto error; - memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(SCACTilePattern *)); + memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); /* populate it with the patterns in the hash */ uint32_t i = 0, p = 0; - for (i = 0; i < INIT_HASH_SIZE; i++) { - SCACTilePattern *node = ctx->init_hash[i], *nnode = NULL; + for (i = 0; i < MPM_INIT_HASH_SIZE; i++) { + MpmPattern *node = mpm_ctx->init_hash[i], *nnode = NULL; while(node != NULL) { nnode = node->next; node->next = NULL; ctx->parray[p++] = node; + SCACTileHistogramAlphabet(ctx, node); node = nnode; } } /* we no longer need the hash, so free it's memory */ - SCFree(ctx->init_hash); - ctx->init_hash = NULL; + SCFree(mpm_ctx->init_hash); + mpm_ctx->init_hash = NULL; /* Handle case patterns by storing a copy of the pattern to compare * to each possible match (no-case). @@ -1336,12 +1017,11 @@ void SCACTileInitCtx(MpmCtx *mpm_ctx) mpm_ctx->memory_size += sizeof(SCACTileCtx); /* initialize the hash we use to speed up pattern insertions */ - SCACTileCtx *ctx = search_ctx->init_ctx; - ctx->init_hash = SCMalloc(sizeof(SCACTilePattern *) * INIT_HASH_SIZE); - if (ctx->init_hash == NULL) { + mpm_ctx->init_hash = SCMalloc(sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE); + if (mpm_ctx->init_hash == NULL) { exit(EXIT_FAILURE); } - memset(ctx->init_hash, 0, sizeof(SCACTilePattern *) * INIT_HASH_SIZE); + memset(mpm_ctx->init_hash, 0, sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE); /* get conf values for AC from our yaml file. We have no conf values for * now. We will certainly need this, as we develop the algo */ @@ -1374,16 +1054,16 @@ static void SCACTileDestroyInitCtx(MpmCtx *mpm_ctx) if (ctx == NULL) return; - if (ctx->init_hash != NULL) { - SCFree(ctx->init_hash); - ctx->init_hash = NULL; + if (mpm_ctx->init_hash != NULL) { + SCFree(mpm_ctx->init_hash); + mpm_ctx->init_hash = NULL; } if (ctx->parray != NULL) { uint32_t i; for (i = 0; i < mpm_ctx->pattern_cnt; i++) { if (ctx->parray[i] != NULL) { - SCACTileFreePattern(mpm_ctx, ctx->parray[i]); + MpmFreePattern(mpm_ctx, ctx->parray[i]); } } @@ -1739,7 +1419,7 @@ int SCACTileAddPatternCI(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, SigIntId sid, uint8_t flags) { flags |= MPM_PATTERN_FLAG_NOCASE; - return SCACTileAddPattern(mpm_ctx, pat, patlen, offset, depth, + return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); } @@ -1764,7 +1444,7 @@ int SCACTileAddPatternCS(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, SigIntId sid, uint8_t flags) { - return SCACTileAddPattern(mpm_ctx, pat, patlen, offset, depth, + return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); } @@ -1789,8 +1469,8 @@ void SCACTilePrintInfo(MpmCtx *mpm_ctx) printf(" Sizeof:\n"); printf(" MpmCtx %" PRIuMAX "\n", (uintmax_t)sizeof(MpmCtx)); printf(" SCACTileCtx: %" PRIuMAX "\n", (uintmax_t)sizeof(SCACTileCtx)); - printf(" SCACTilePattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACTilePattern)); - printf(" SCACTilePattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACTilePattern)); + printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern)); + printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern)); printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt); printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen); printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen); diff --git a/src/util-mpm-ac-tile.h b/src/util-mpm-ac-tile.h index 62e3649fab..5f47f23018 100644 --- a/src/util-mpm-ac-tile.h +++ b/src/util-mpm-ac-tile.h @@ -26,27 +26,6 @@ #ifndef __UTIL_MPM_AC_TILE__H__ #define __UTIL_MPM_AC_TILE__H__ -typedef struct SCACTilePattern_ { - /* length of the pattern */ - uint16_t len; - /* flags decribing the pattern */ - uint8_t flags; - /* holds the original pattern that was added */ - uint8_t *original_pat; - /* case sensitive */ - uint8_t *cs; - /* case INsensitive */ - uint8_t *ci; - /* pattern id */ - uint32_t id; - - /* sid(s) for this pattern */ - uint32_t sids_size; - SigIntId *sids; - - struct SCACTilePattern_ *next; -} SCACTilePattern; - typedef struct SCACTilePatternList_ { uint8_t *cs; uint16_t patlen; @@ -98,12 +77,9 @@ typedef struct SCACTileCtx_ { /* Indexed by MpmPatternIndex */ SCACTilePatternList *pattern_list; - /* hash used during ctx initialization */ - SCACTilePattern **init_hash; - /* pattern arrays. We need this only during the goto table creation phase */ - SCACTilePattern **parray; + MpmPattern **parray; /* goto_table, failure table and output table. Needed to create * state_table. Will be freed, once we have created the diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c index 2dca134f51..fe37af19ba 100644 --- a/src/util-mpm-ac.c +++ b/src/util-mpm-ac.c @@ -86,8 +86,6 @@ void SCACRegisterTests(void); /* a placeholder to denote a failure transition in the goto table */ #define SC_AC_FAIL (-1) -/* size of the hash table used to speed up pattern insertions initially */ -#define INIT_HASH_SIZE 65536 #define STATE_QUEUE_CONTAINER_SIZE 65536 @@ -118,286 +116,6 @@ static void SCACGetConfig() return; } -/** - * \internal - * \brief Creates a hash of the pattern. We use it for the hashing process - * during the initial pattern insertion time, to cull duplicate sigs. - * - * \param pat Pointer to the pattern. - * \param patlen Pattern length. - * - * \retval hash A 32 bit unsigned hash. - */ -static inline uint32_t SCACInitHashRaw(uint8_t *pat, uint16_t patlen) -{ - uint32_t hash = patlen * pat[0]; - if (patlen > 1) - hash += pat[1]; - - return (hash % INIT_HASH_SIZE); -} - -/** - * \internal - * \brief Looks up a pattern. We use it for the hashing process during the - * the initial pattern insertion time, to cull duplicate sigs. - * - * \param ctx Pointer to the AC ctx. - * \param pat Pointer to the pattern. - * \param patlen Pattern length. - * \param flags Flags. We don't need this. - * - * \retval hash A 32 bit unsigned hash. - */ -static inline SCACPattern *SCACInitHashLookup(SCACCtx *ctx, uint8_t *pat, - uint16_t patlen, char flags, - uint32_t pid) -{ - uint32_t hash = SCACInitHashRaw(pat, patlen); - - if (ctx->init_hash == NULL) { - return NULL; - } - - SCACPattern *t = ctx->init_hash[hash]; - for ( ; t != NULL; t = t->next) { - if (t->id == pid) - return t; - } - - return NULL; -} - -/** - * \internal - * \brief Allocs a new pattern instance. - * - * \param mpm_ctx Pointer to the mpm context. - * - * \retval p Pointer to the newly created pattern. - */ -static inline SCACPattern *SCACAllocPattern(MpmCtx *mpm_ctx) -{ - SCACPattern *p = SCMalloc(sizeof(SCACPattern)); - if (unlikely(p == NULL)) { - exit(EXIT_FAILURE); - } - memset(p, 0, sizeof(SCACPattern)); - - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += sizeof(SCACPattern); - - return p; -} - -/** - * \internal - * \brief Used to free SCACPattern instances. - * - * \param mpm_ctx Pointer to the mpm context. - * \param p Pointer to the SCACPattern instance to be freed. - * \param free Free the above pointer or not. - */ -static inline void SCACFreePattern(MpmCtx *mpm_ctx, SCACPattern *p) -{ - if (p != NULL && p->cs != NULL && p->cs != p->ci) { - SCFree(p->cs); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL && p->ci != NULL) { - SCFree(p->ci); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL && p->original_pat != NULL) { - SCFree(p->original_pat); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= p->len; - } - - if (p != NULL && p->sids != NULL) { - SCFree(p->sids); - } - - if (p != NULL) { - SCFree(p); - mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= sizeof(SCACPattern); - } - return; -} - -static inline uint32_t SCACInitHash(SCACPattern *p) -{ - uint32_t hash = p->len * p->original_pat[0]; - if (p->len > 1) - hash += p->original_pat[1]; - - return (hash % INIT_HASH_SIZE); -} - -static inline int SCACInitHashAdd(SCACCtx *ctx, SCACPattern *p) -{ - uint32_t hash = SCACInitHash(p); - - if (ctx->init_hash == NULL) { - return 0; - } - - if (ctx->init_hash[hash] == NULL) { - ctx->init_hash[hash] = p; - return 0; - } - - SCACPattern *tt = NULL; - SCACPattern *t = ctx->init_hash[hash]; - - /* get the list tail */ - do { - tt = t; - t = t->next; - } while (t != NULL); - - tt->next = p; - - return 0; -} - -/** - * \internal - * \brief Add a pattern to the mpm-ac context. - * - * \param mpm_ctx Mpm context. - * \param pat Pointer to the pattern. - * \param patlen Length of the pattern. - * \param pid Pattern id - * \param sid Signature id (internal id). - * \param flags Pattern's MPM_PATTERN_* flags. - * - * \retval 0 On success. - * \retval -1 On failure. - */ -static int SCACAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, - uint16_t offset, uint16_t depth, uint32_t pid, - SigIntId sid, uint8_t flags) -{ - SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx; - - SCLogDebug("Adding pattern for ctx %p, patlen %"PRIu16" and pid %" PRIu32, - ctx, patlen, pid); - - if (patlen == 0) { - SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0"); - return 0; - } - - /* check if we have already inserted this pattern */ - SCACPattern *p = SCACInitHashLookup(ctx, pat, patlen, flags, pid); - if (p == NULL) { - SCLogDebug("Allocing new pattern"); - - /* p will never be NULL */ - p = SCACAllocPattern(mpm_ctx); - - p->len = patlen; - p->flags = flags; - p->id = pid; - - p->original_pat = SCMalloc(patlen); - if (p->original_pat == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy(p->original_pat, pat, patlen); - - p->ci = SCMalloc(patlen); - if (p->ci == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy_tolower(p->ci, pat, patlen); - - /* setup the case sensitive part of the pattern */ - if (p->flags & MPM_PATTERN_FLAG_NOCASE) { - /* nocase means no difference between cs and ci */ - p->cs = p->ci; - } else { - if (memcmp(p->ci, pat, p->len) == 0) { - /* no diff between cs and ci: pat is lowercase */ - p->cs = p->ci; - } else { - p->cs = SCMalloc(patlen); - if (p->cs == NULL) - goto error; - mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += patlen; - memcpy(p->cs, pat, patlen); - } - } - - /* put in the pattern hash */ - SCACInitHashAdd(ctx, p); - - //if (mpm_ctx->pattern_cnt == 65535) { - // SCLogError(SC_ERR_AHO_CORASICK, "Max search words reached. Can't " - // "insert anymore. Exiting"); - // exit(EXIT_FAILURE); - //} - mpm_ctx->pattern_cnt++; - - if (mpm_ctx->maxlen < patlen) - mpm_ctx->maxlen = patlen; - - if (mpm_ctx->minlen == 0) { - mpm_ctx->minlen = patlen; - } else { - if (mpm_ctx->minlen > patlen) - mpm_ctx->minlen = patlen; - } - - /* we need the max pat id */ - if (pid > ctx->max_pat_id) - ctx->max_pat_id = pid; - - p->sids_size = 1; - p->sids = SCMalloc(p->sids_size * sizeof(SigIntId)); - BUG_ON(p->sids == NULL); - p->sids[0] = sid; - //SCLogInfo("MPM added %u:%u", pid, sid); - } else { - /* TODO figure out how we can be called multiple times for the same CTX with the same sid */ - - int found = 0; - uint32_t x = 0; - for (x = 0; x < p->sids_size; x++) { - if (p->sids[x] == sid) { - found = 1; - break; - } - } - if (!found) { - SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1))); - BUG_ON(sids == NULL); - p->sids = sids; - p->sids[p->sids_size] = sid; - p->sids_size++; - //SCLogInfo("p->sids_size %u", p->sids_size); - //SCLogInfo("MPM added %u:%u (append)", pid, sid); - } else { - //SCLogInfo("rule %u already part of pid %u", sid, pid); - } - } - - return 0; - -error: - SCACFreePattern(mpm_ctx, p); - return -1; -} - /** * \internal * \brief Initialize a new state in the goto and output tables. @@ -1036,24 +754,24 @@ int SCACPreparePatterns(MpmCtx *mpm_ctx) { SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx; - if (mpm_ctx->pattern_cnt == 0 || ctx->init_hash == NULL) { + if (mpm_ctx->pattern_cnt == 0 || mpm_ctx->init_hash == NULL) { SCLogDebug("no patterns supplied to this mpm_ctx"); return 0; } /* alloc the pattern array */ - ctx->parray = (SCACPattern **)SCMalloc(mpm_ctx->pattern_cnt * - sizeof(SCACPattern *)); + ctx->parray = (MpmPattern **)SCMalloc(mpm_ctx->pattern_cnt * + sizeof(MpmPattern *)); if (ctx->parray == NULL) goto error; - memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(SCACPattern *)); + memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); mpm_ctx->memory_cnt++; - mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(SCACPattern *)); + mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); /* populate it with the patterns in the hash */ uint32_t i = 0, p = 0; - for (i = 0; i < INIT_HASH_SIZE; i++) { - SCACPattern *node = ctx->init_hash[i], *nnode = NULL; + for (i = 0; i < MPM_INIT_HASH_SIZE; i++) { + MpmPattern *node = mpm_ctx->init_hash[i], *nnode = NULL; while(node != NULL) { nnode = node->next; node->next = NULL; @@ -1063,19 +781,19 @@ int SCACPreparePatterns(MpmCtx *mpm_ctx) } /* we no longer need the hash, so free it's memory */ - SCFree(ctx->init_hash); - ctx->init_hash = NULL; + SCFree(mpm_ctx->init_hash); + mpm_ctx->init_hash = NULL; /* the memory consumed by a single state in our goto table */ ctx->single_state_size = sizeof(int32_t) * 256; /* handle no case patterns */ - ctx->pid_pat_list = SCMalloc((ctx->max_pat_id + 1)* sizeof(SCACPatternList)); + ctx->pid_pat_list = SCMalloc((mpm_ctx->max_pat_id + 1)* sizeof(SCACPatternList)); if (ctx->pid_pat_list == NULL) { SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory"); exit(EXIT_FAILURE); } - memset(ctx->pid_pat_list, 0, (ctx->max_pat_id + 1) * sizeof(SCACPatternList)); + memset(ctx->pid_pat_list, 0, (mpm_ctx->max_pat_id + 1) * sizeof(SCACPatternList)); for (i = 0; i < mpm_ctx->pattern_cnt; i++) { if (!(ctx->parray[i]->flags & MPM_PATTERN_FLAG_NOCASE)) { @@ -1123,15 +841,16 @@ int SCACPreparePatterns(MpmCtx *mpm_ctx) /* free all the stored patterns. Should save us a good 100-200 mbs */ for (i = 0; i < mpm_ctx->pattern_cnt; i++) { if (ctx->parray[i] != NULL) { - SCACFreePattern(mpm_ctx, ctx->parray[i]); + MpmFreePattern(mpm_ctx, ctx->parray[i]); } } SCFree(ctx->parray); ctx->parray = NULL; mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(SCACPattern *)); + mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); - ctx->pattern_id_bitarray_size = (ctx->max_pat_id / 8) + 1; + ctx->pattern_id_bitarray_size = (mpm_ctx->max_pat_id / 8) + 1; + SCLogDebug("ctx->pattern_id_bitarray_size %u", ctx->pattern_id_bitarray_size); return 0; @@ -1181,12 +900,11 @@ void SCACInitCtx(MpmCtx *mpm_ctx) mpm_ctx->memory_size += sizeof(SCACCtx); /* initialize the hash we use to speed up pattern insertions */ - SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx; - ctx->init_hash = SCMalloc(sizeof(SCACPattern *) * INIT_HASH_SIZE); - if (ctx->init_hash == NULL) { + mpm_ctx->init_hash = SCMalloc(sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE); + if (mpm_ctx->init_hash == NULL) { exit(EXIT_FAILURE); } - memset(ctx->init_hash, 0, sizeof(SCACPattern *) * INIT_HASH_SIZE); + memset(mpm_ctx->init_hash, 0, sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE); /* get conf values for AC from our yaml file. We have no conf values for * now. We will certainly need this, as we develop the algo */ @@ -1226,25 +944,25 @@ void SCACDestroyCtx(MpmCtx *mpm_ctx) if (ctx == NULL) return; - if (ctx->init_hash != NULL) { - SCFree(ctx->init_hash); - ctx->init_hash = NULL; + if (mpm_ctx->init_hash != NULL) { + SCFree(mpm_ctx->init_hash); + mpm_ctx->init_hash = NULL; mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (INIT_HASH_SIZE * sizeof(SCACPattern *)); + mpm_ctx->memory_size -= (MPM_INIT_HASH_SIZE * sizeof(MpmPattern *)); } if (ctx->parray != NULL) { uint32_t i; for (i = 0; i < mpm_ctx->pattern_cnt; i++) { if (ctx->parray[i] != NULL) { - SCACFreePattern(mpm_ctx, ctx->parray[i]); + MpmFreePattern(mpm_ctx, ctx->parray[i]); } } SCFree(ctx->parray); ctx->parray = NULL; mpm_ctx->memory_cnt--; - mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(SCACPattern *)); + mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(MpmPattern *)); } if (ctx->state_table_u16 != NULL) { @@ -1276,7 +994,7 @@ void SCACDestroyCtx(MpmCtx *mpm_ctx) if (ctx->pid_pat_list != NULL) { uint32_t i; - for (i = 0; i < (ctx->max_pat_id + 1); i++) { + for (i = 0; i < (mpm_ctx->max_pat_id + 1); i++) { if (ctx->pid_pat_list[i].cs != NULL) SCFree(ctx->pid_pat_list[i].cs); if (ctx->pid_pat_list[i].sids != NULL) @@ -1425,7 +1143,7 @@ int SCACAddPatternCI(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, SigIntId sid, uint8_t flags) { flags |= MPM_PATTERN_FLAG_NOCASE; - return SCACAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); + return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); } /** @@ -1449,7 +1167,7 @@ int SCACAddPatternCS(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, SigIntId sid, uint8_t flags) { - return SCACAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); + return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags); } void SCACPrintSearchStats(MpmThreadCtx *mpm_thread_ctx) @@ -1475,8 +1193,8 @@ void SCACPrintInfo(MpmCtx *mpm_ctx) printf(" Sizeof:\n"); printf(" MpmCtx %" PRIuMAX "\n", (uintmax_t)sizeof(MpmCtx)); printf(" SCACCtx: %" PRIuMAX "\n", (uintmax_t)sizeof(SCACCtx)); - printf(" SCACPattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACPattern)); - printf(" SCACPattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACPattern)); + printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern)); + printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern)); printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt); printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen); printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen); diff --git a/src/util-mpm-ac.h b/src/util-mpm-ac.h index 394ad6781f..97ae48e8d7 100644 --- a/src/util-mpm-ac.h +++ b/src/util-mpm-ac.h @@ -38,27 +38,6 @@ #include "flow.h" #endif /* __SC_CUDA_SUPPORT__ */ -typedef struct SCACPattern_ { - /* length of the pattern */ - uint16_t len; - /* flags decribing the pattern */ - uint8_t flags; - /* holds the original pattern that was added */ - uint8_t *original_pat; - /* case sensitive */ - uint8_t *cs; - /* case INsensitive */ - uint8_t *ci; - /* pattern id */ - uint32_t id; - - /* sid(s) for this pattern */ - uint32_t sids_size; - SigIntId *sids; - - struct SCACPattern_ *next; -} SCACPattern; - typedef struct SCACPatternList_ { uint8_t *cs; uint16_t patlen; @@ -76,11 +55,8 @@ typedef struct SCACOutputTable_ { } SCACOutputTable; typedef struct SCACCtx_ { - /* hash used during ctx initialization */ - SCACPattern **init_hash; - /* pattern arrays. We need this only during the goto table creation phase */ - SCACPattern **parray; + MpmPattern **parray; /* no of states used by ac */ uint32_t state_count; @@ -101,7 +77,6 @@ typedef struct SCACCtx_ { /* the size of each state */ uint32_t single_state_size; - uint32_t max_pat_id; uint32_t allocated_state_count; diff --git a/src/util-mpm.c b/src/util-mpm.c index 27f5a86c07..9fbf6c2279 100644 --- a/src/util-mpm.c +++ b/src/util-mpm.c @@ -45,6 +45,7 @@ #include "util-cuda-handlers.h" #include "detect-engine-mpm.h" #endif +#include "util-memcpy.h" /** * \brief Register a new Mpm Context. @@ -556,6 +557,287 @@ int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen, } +/** + * \internal + * \brief Creates a hash of the pattern. We use it for the hashing process + * during the initial pattern insertion time, to cull duplicate sigs. + * + * \param pat Pointer to the pattern. + * \param patlen Pattern length. + * + * \retval hash A 32 bit unsigned hash. + */ +static inline uint32_t MpmInitHashRaw(uint8_t *pat, uint16_t patlen) +{ + uint32_t hash = patlen * pat[0]; + if (patlen > 1) + hash += pat[1]; + + return (hash % MPM_INIT_HASH_SIZE); +} + +/** + * \internal + * \brief Looks up a pattern. We use it for the hashing process during the + * the initial pattern insertion time, to cull duplicate sigs. + * + * \param ctx Pointer to the AC ctx. + * \param pat Pointer to the pattern. + * \param patlen Pattern length. + * \param flags Flags. We don't need this. + * + * \retval hash A 32 bit unsigned hash. + */ +static inline MpmPattern *MpmInitHashLookup(MpmCtx *ctx, uint8_t *pat, + uint16_t patlen, char flags, + uint32_t pid) +{ + uint32_t hash = MpmInitHashRaw(pat, patlen); + + if (ctx->init_hash == NULL) { + return NULL; + } + + MpmPattern *t = ctx->init_hash[hash]; + for ( ; t != NULL; t = t->next) { + if (!(flags & MPM_PATTERN_CTX_OWNS_ID)) { + if (t->id == pid) + return t; + } else { + if (t->len == patlen && + memcmp(pat, t->original_pat, patlen) == 0 && + t->flags == flags) + { + return t; + } + } + } + + return NULL; +} + +/** + * \internal + * \brief Allocs a new pattern instance. + * + * \param mpm_ctx Pointer to the mpm context. + * + * \retval p Pointer to the newly created pattern. + */ +static inline MpmPattern *MpmAllocPattern(MpmCtx *mpm_ctx) +{ + MpmPattern *p = SCMalloc(sizeof(MpmPattern)); + if (unlikely(p == NULL)) { + exit(EXIT_FAILURE); + } + memset(p, 0, sizeof(MpmPattern)); + + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += sizeof(MpmPattern); + + return p; +} + +/** + * \internal + * \brief Used to free MpmPattern instances. + * + * \param mpm_ctx Pointer to the mpm context. + * \param p Pointer to the MpmPattern instance to be freed. + */ +void MpmFreePattern(MpmCtx *mpm_ctx, MpmPattern *p) +{ + if (p != NULL && p->cs != NULL && p->cs != p->ci) { + SCFree(p->cs); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= p->len; + } + + if (p != NULL && p->ci != NULL) { + SCFree(p->ci); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= p->len; + } + + if (p != NULL && p->original_pat != NULL) { + SCFree(p->original_pat); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= p->len; + } + + if (p != NULL) { + SCFree(p); + mpm_ctx->memory_cnt--; + mpm_ctx->memory_size -= sizeof(MpmPattern); + } + return; +} + +static inline uint32_t MpmInitHash(MpmPattern *p) +{ + uint32_t hash = p->len * p->original_pat[0]; + if (p->len > 1) + hash += p->original_pat[1]; + + return (hash % MPM_INIT_HASH_SIZE); +} + +static inline int MpmInitHashAdd(MpmCtx *ctx, MpmPattern *p) +{ + uint32_t hash = MpmInitHash(p); + + if (ctx->init_hash == NULL) { + return 0; + } + + if (ctx->init_hash[hash] == NULL) { + ctx->init_hash[hash] = p; + return 0; + } + + MpmPattern *tt = NULL; + MpmPattern *t = ctx->init_hash[hash]; + + /* get the list tail */ + do { + tt = t; + t = t->next; + } while (t != NULL); + + tt->next = p; + + return 0; +} + +/** + * \internal + * \brief Add a pattern to the mpm-ac context. + * + * \param mpm_ctx Mpm context. + * \param pat Pointer to the pattern. + * \param patlen Length of the pattern. + * \param pid Pattern id + * \param sid Signature id (internal id). + * \param flags Pattern's MPM_PATTERN_* flags. + * + * \retval 0 On success. + * \retval -1 On failure. + */ +int MpmAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, + uint16_t offset, uint16_t depth, uint32_t pid, + SigIntId sid, uint8_t flags) +{ + SCLogDebug("Adding pattern for ctx %p, patlen %"PRIu16" and pid %" PRIu32, + mpm_ctx, patlen, pid); + + if (patlen == 0) { + SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0"); + return 0; + } + + if (flags & MPM_PATTERN_CTX_OWNS_ID) + pid = UINT_MAX; + + /* check if we have already inserted this pattern */ + MpmPattern *p = MpmInitHashLookup(mpm_ctx, pat, patlen, flags, pid); + if (p == NULL) { + SCLogDebug("Allocing new pattern"); + + /* p will never be NULL */ + p = MpmAllocPattern(mpm_ctx); + + p->len = patlen; + p->flags = flags; + if (flags & MPM_PATTERN_CTX_OWNS_ID) + p->id = mpm_ctx->max_pat_id++; + else + p->id = pid; + + p->original_pat = SCMalloc(patlen); + if (p->original_pat == NULL) + goto error; + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += patlen; + memcpy(p->original_pat, pat, patlen); + + p->ci = SCMalloc(patlen); + if (p->ci == NULL) + goto error; + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += patlen; + memcpy_tolower(p->ci, pat, patlen); + + /* setup the case sensitive part of the pattern */ + if (p->flags & MPM_PATTERN_FLAG_NOCASE) { + /* nocase means no difference between cs and ci */ + p->cs = p->ci; + } else { + if (memcmp(p->ci, pat, p->len) == 0) { + /* no diff between cs and ci: pat is lowercase */ + p->cs = p->ci; + } else { + p->cs = SCMalloc(patlen); + if (p->cs == NULL) + goto error; + mpm_ctx->memory_cnt++; + mpm_ctx->memory_size += patlen; + memcpy(p->cs, pat, patlen); + } + } + + /* put in the pattern hash */ + MpmInitHashAdd(mpm_ctx, p); + + mpm_ctx->pattern_cnt++; + + if (mpm_ctx->maxlen < patlen) + mpm_ctx->maxlen = patlen; + + if (mpm_ctx->minlen == 0) { + mpm_ctx->minlen = patlen; + } else { + if (mpm_ctx->minlen > patlen) + mpm_ctx->minlen = patlen; + } + + /* we need the max pat id */ + if (p->id > mpm_ctx->max_pat_id) + mpm_ctx->max_pat_id = p->id; + + p->sids_size = 1; + p->sids = SCMalloc(p->sids_size * sizeof(SigIntId)); + BUG_ON(p->sids == NULL); + p->sids[0] = sid; + } else { + /* we can be called multiple times for the same sid in the case + * of the 'single' modus. Here multiple rule groups share the + * same mpm ctx and might be adding the same pattern to the + * mpm_ctx */ + int found = 0; + uint32_t x = 0; + for (x = 0; x < p->sids_size; x++) { + if (p->sids[x] == sid) { + found = 1; + break; + } + } + + if (!found) { + SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1))); + BUG_ON(sids == NULL); + p->sids = sids; + p->sids[p->sids_size] = sid; + p->sids_size++; + } + } + + return 0; + +error: + MpmFreePattern(mpm_ctx, p); + return -1; +} + /************************************Unittests*********************************/ diff --git a/src/util-mpm.h b/src/util-mpm.h index 209a3fddfd..2cf85e5980 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -25,6 +25,9 @@ #define __UTIL_MPM_H__ #include "suricata-common.h" + +#define MPM_INIT_HASH_SIZE 65536 + enum { MPM_NOTSET = 0, @@ -76,6 +79,27 @@ typedef struct PatternMatcherQueue_ { } PatternMatcherQueue; +typedef struct MpmPattern_ { + /* length of the pattern */ + uint16_t len; + /* flags decribing the pattern */ + uint8_t flags; + /* holds the original pattern that was added */ + uint8_t *original_pat; + /* case sensitive */ + uint8_t *cs; + /* case INsensitive */ + uint8_t *ci; + /* pattern id */ + uint32_t id; + + /* sid(s) for this pattern */ + uint32_t sids_size; + SigIntId *sids; + + struct MpmPattern_ *next; +} MpmPattern; + typedef struct MpmCtx_ { void *ctx; uint16_t mpm_type; @@ -94,6 +118,11 @@ typedef struct MpmCtx_ { uint32_t memory_cnt; uint32_t memory_size; + + uint32_t max_pat_id; + + /* hash used during ctx initialization */ + MpmPattern **init_hash; } MpmCtx; /* if we want to retrieve an unique mpm context from the mpm context factory @@ -125,6 +154,9 @@ typedef struct MpmCtxFactoryContainer_ { #define MPM_PATTERN_FLAG_OFFSET 0x08 /** one byte pattern (used in b2g) */ #define MPM_PATTERN_ONE_BYTE 0x10 +/** the ctx uses it's own internal id instead of + * what is passed through the API */ +#define MPM_PATTERN_CTX_OWNS_ID 0x20 typedef struct MpmTableElmt_ { char *name; @@ -220,6 +252,12 @@ int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, SigIntId sid, uint8_t flags); +void MpmFreePattern(MpmCtx *mpm_ctx, MpmPattern *p); + +int MpmAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, + uint16_t offset, uint16_t depth, uint32_t pid, + SigIntId sid, uint8_t flags); + /* Resize Signature ID array. Only called from MpmAddSids(). */ int MpmAddSidsResize(PatternMatcherQueue *pmq, uint32_t new_size);