aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

mpm: unify & localize mpm pattern (id) handling

Browse Source 
So far, the patterns as passed to the mpm's would use global id's that
were shared among all buffers, directions. This would lead to a fairly
large pattern id space. As the mpm algo's use the pattern id's to
prevent duplicate matching through a pattern id based bitarray,
shrinking this space will optimize performance.

This patch implements this. It sets a flag before adding the pattern
to the mpm ctx, instructing the mpm to ignore the provided pid and
handle pids management itself. This leads to a shrinking of the
bitarray size.

This is made possible by the previous work that removes the pid logic
from the code.

Next to this, this patch moves the pattern setup stage to common util
functions. This avoids code duplication.

Update ac, ac-bs and ac-ks to use this.
pull/1980/head
Victor Julien 11 years ago
parent ba9d43cce5
commit 9b3d4f7e24
9 changed files with 408 additions and 1035 deletions
Show Stats Download Patch File Download Diff File

8
src/detect-engine-mpm.c
Unescape Escape View File

@ -933,24 +933,24 @@ static void PopulateMpmHelperAddPatternToPktCtx(MpmCtx *mpm_ctx,
MpmAddPatternCI(mpm_ctx,
cd->content + cd->fp_chop_offset, cd->fp_chop_len,
pat_offset, pat_depth,
cd->id, s->num, flags);
cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
} else {
MpmAddPatternCI(mpm_ctx,
cd->content, cd->content_len,
pat_offset, pat_depth,
cd->id, s->num, flags);
cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
}
} else {
if (chop) {
MpmAddPatternCS(mpm_ctx,
cd->content + cd->fp_chop_offset, cd->fp_chop_len,
pat_offset, pat_depth,
cd->id, s->num, flags);
cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
} else {
MpmAddPatternCS(mpm_ctx,
cd->content, cd->content_len,
pat_offset, pat_depth,
cd->id, s->num, flags);
cd->id, s->num, flags|MPM_PATTERN_CTX_OWNS_ID);
}
}

329
src/util-mpm-ac-bs.c
Unescape Escape View File

@ -77,8 +77,6 @@ void SCACBSRegisterTests(void);
/* a placeholder to denote a failure transition in the goto table */
#define SC_AC_BS_FAIL (-1)
/* size of the hash table used to speed up pattern insertions initially */
#define INIT_HASH_SIZE 65536
#define STATE_QUEUE_CONTAINER_SIZE 65536
@ -132,276 +130,6 @@ static void SCACBSGetConfig()
return;
}
/**
* \internal
* \brief Creates a hash of the pattern. We use it for the hashing process
* during the initial pattern insertion time, to cull duplicate sigs.
*
* \param pat Pointer to the pattern.
* \param patlen Pattern length.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline uint32_t SCACBSInitHashRaw(uint8_t *pat, uint16_t patlen)
{
uint32_t hash = patlen * pat[0];
if (patlen > 1)
hash += pat[1];
return (hash % INIT_HASH_SIZE);
}
/**
* \internal
* \brief Looks up a pattern. We use it for the hashing process during the
* the initial pattern insertion time, to cull duplicate sigs.
*
* \param ctx Pointer to the AC ctx.
* \param pat Pointer to the pattern.
* \param patlen Pattern length.
* \param flags Flags. We don't need this.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline SCACBSPattern *SCACBSInitHashLookup(SCACBSCtx *ctx, uint8_t *pat,
uint16_t patlen, char flags,
uint32_t pid)
{
uint32_t hash = SCACBSInitHashRaw(pat, patlen);
if (ctx->init_hash == NULL) {
return NULL;
}
SCACBSPattern *t = ctx->init_hash[hash];
for ( ; t != NULL; t = t->next) {
if (t->id == pid)
return t;
}
return NULL;
}
/**
* \internal
* \brief Allocs a new pattern instance.
*
* \param mpm_ctx Pointer to the mpm context.
*
* \retval p Pointer to the newly created pattern.
*/
static inline SCACBSPattern *SCACBSAllocPattern(MpmCtx *mpm_ctx)
{
SCACBSPattern *p = SCMalloc(sizeof(SCACBSPattern));
if (unlikely(p == NULL)) {
exit(EXIT_FAILURE);
}
memset(p, 0, sizeof(SCACBSPattern));
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += sizeof(SCACBSPattern);
return p;
}
/**
* \internal
* \brief Used to free SCACBSPattern instances.
*
* \param mpm_ctx Pointer to the mpm context.
* \param p Pointer to the SCACBSPattern instance to be freed.
*/
static inline void SCACBSFreePattern(MpmCtx *mpm_ctx, SCACBSPattern *p)
{
if (p != NULL && p->cs != NULL && p->cs != p->ci) {
SCFree(p->cs);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->ci != NULL) {
SCFree(p->ci);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->original_pat != NULL) {
SCFree(p->original_pat);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL) {
SCFree(p);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= sizeof(SCACBSPattern);
}
return;
}
static inline uint32_t SCACBSInitHash(SCACBSPattern *p)
{
uint32_t hash = p->len * p->original_pat[0];
if (p->len > 1)
hash += p->original_pat[1];
return (hash % INIT_HASH_SIZE);
}
static inline int SCACBSInitHashAdd(SCACBSCtx *ctx, SCACBSPattern *p)
{
uint32_t hash = SCACBSInitHash(p);
if (ctx->init_hash == NULL) {
return 0;
}
if (ctx->init_hash[hash] == NULL) {
ctx->init_hash[hash] = p;
return 0;
}
SCACBSPattern *tt = NULL;
SCACBSPattern *t = ctx->init_hash[hash];
/* get the list tail */
do {
tt = t;
t = t->next;
} while (t != NULL);
tt->next = p;
return 0;
}
/**
* \internal
* \brief Add a pattern to the mpm-ac context.
*
* \param mpm_ctx Mpm context.
* \param pat Pointer to the pattern.
* \param patlen Length of the pattern.
* \param pid Pattern id
* \param sid Signature id (internal id).
* \param flags Pattern's MPM_PATTERN_* flags.
*
* \retval 0 On success.
* \retval -1 On failure.
*/
static int SCACBSAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth, uint32_t pid,
SigIntId sid, uint8_t flags)
{
SCACBSCtx *ctx = (SCACBSCtx *)mpm_ctx->ctx;
SCLogDebug("Adding pattern for ctx %p, patlen %"PRIu16" and pid %" PRIu32,
ctx, patlen, pid);
if (patlen == 0) {
SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0");
return 0;
}
/* check if we have already inserted this pattern */
SCACBSPattern *p = SCACBSInitHashLookup(ctx, pat, patlen, flags, pid);
if (p == NULL) {
SCLogDebug("Allocing new pattern");
/* p will never be NULL */
p = SCACBSAllocPattern(mpm_ctx);
p->len = patlen;
p->flags = flags;
p->id = pid;
p->original_pat = SCMalloc(patlen);
if (p->original_pat == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy(p->original_pat, pat, patlen);
p->ci = SCMalloc(patlen);
if (p->ci == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy_tolower(p->ci, pat, patlen);
/* setup the case sensitive part of the pattern */
if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
/* nocase means no difference between cs and ci */
p->cs = p->ci;
} else {
if (memcmp(p->ci, pat, p->len) == 0) {
/* no diff between cs and ci: pat is lowercase */
p->cs = p->ci;
} else {
p->cs = SCMalloc(patlen);
if (p->cs == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy(p->cs, pat, patlen);
}
}
/* put in the pattern hash */
SCACBSInitHashAdd(ctx, p);
//if (mpm_ctx->pattern_cnt == 65535) {
// SCLogError(SC_ERR_AHO_CORASICK, "Max search words reached. Can't "
// "insert anymore. Exiting");
// exit(EXIT_FAILURE);
//}
mpm_ctx->pattern_cnt++;
if (mpm_ctx->maxlen < patlen)
mpm_ctx->maxlen = patlen;
if (mpm_ctx->minlen == 0) {
mpm_ctx->minlen = patlen;
} else {
if (mpm_ctx->minlen > patlen)
mpm_ctx->minlen = patlen;
}
/* we need the max pat id */
if (pid > ctx->max_pat_id)
ctx->max_pat_id = pid;
p->sids_size = 1;
p->sids = SCMalloc(p->sids_size * sizeof(SigIntId));
BUG_ON(p->sids == NULL);
p->sids[0] = sid;
} else {
/* TODO figure out how we can be called multiple times for the same CTX with the same sid */
int found = 0;
uint32_t x = 0;
for (x = 0; x < p->sids_size; x++) {
if (p->sids[x] == sid) {
found = 1;
break;
}
}
if (!found) {
SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1)));
BUG_ON(sids == NULL);
p->sids = sids;
p->sids[p->sids_size] = sid;
p->sids_size++;
}
}
return 0;
error:
SCACBSFreePattern(mpm_ctx, p);
return -1;
}
/**
* \internal
* \brief Initialize a new state in the goto and output tables.
@ -1167,24 +895,24 @@ int SCACBSPreparePatterns(MpmCtx *mpm_ctx)
{
SCACBSCtx *ctx = (SCACBSCtx *)mpm_ctx->ctx;
if (mpm_ctx->pattern_cnt == 0 || ctx->init_hash == NULL) {
if (mpm_ctx->pattern_cnt == 0 || mpm_ctx->init_hash == NULL) {
SCLogDebug("no patterns supplied to this mpm_ctx");
return 0;
}
/* alloc the pattern array */
ctx->parray = (SCACBSPattern **)SCMalloc(mpm_ctx->pattern_cnt *
sizeof(SCACBSPattern *));
ctx->parray = (MpmPattern **)SCMalloc(mpm_ctx->pattern_cnt *
sizeof(MpmPattern *));
if (ctx->parray == NULL)
goto error;
memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(SCACBSPattern *));
memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(SCACBSPattern *));
mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
/* populate it with the patterns in the hash */
uint32_t i = 0, p = 0;
for (i = 0; i < INIT_HASH_SIZE; i++) {
SCACBSPattern *node = ctx->init_hash[i], *nnode = NULL;
for (i = 0; i < MPM_INIT_HASH_SIZE; i++) {
MpmPattern *node = mpm_ctx->init_hash[i], *nnode = NULL;
while(node != NULL) {
nnode = node->next;
node->next = NULL;
@ -1194,19 +922,19 @@ int SCACBSPreparePatterns(MpmCtx *mpm_ctx)
}
/* we no longer need the hash, so free it's memory */
SCFree(ctx->init_hash);
ctx->init_hash = NULL;
SCFree(mpm_ctx->init_hash);
mpm_ctx->init_hash = NULL;
/* the memory consumed by a single state in our goto table */
ctx->single_state_size = sizeof(int32_t) * 256;
/* handle no case patterns */
ctx->pid_pat_list = SCMalloc((ctx->max_pat_id + 1)* sizeof(SCACBSPatternList));
ctx->pid_pat_list = SCMalloc((mpm_ctx->max_pat_id + 1)* sizeof(SCACBSPatternList));
if (ctx->pid_pat_list == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
exit(EXIT_FAILURE);
}
memset(ctx->pid_pat_list, 0, (ctx->max_pat_id + 1) * sizeof(SCACBSPatternList));
memset(ctx->pid_pat_list, 0, (mpm_ctx->max_pat_id + 1) * sizeof(SCACBSPatternList));
for (i = 0; i < mpm_ctx->pattern_cnt; i++) {
if (!(ctx->parray[i]->flags & MPM_PATTERN_FLAG_NOCASE)) {
@ -1231,15 +959,15 @@ int SCACBSPreparePatterns(MpmCtx *mpm_ctx)
/* free all the stored patterns. Should save us a good 100-200 mbs */
for (i = 0; i < mpm_ctx->pattern_cnt; i++) {
if (ctx->parray[i] != NULL) {
SCACBSFreePattern(mpm_ctx, ctx->parray[i]);
MpmFreePattern(mpm_ctx, ctx->parray[i]);
}
}
SCFree(ctx->parray);
ctx->parray = NULL;
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(SCACBSPattern *));
mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
ctx->pattern_id_bitarray_size = (ctx->max_pat_id / 8) + 1;
ctx->pattern_id_bitarray_size = (mpm_ctx->max_pat_id / 8) + 1;
return 0;
@ -1291,12 +1019,11 @@ void SCACBSInitCtx(MpmCtx *mpm_ctx)
mpm_ctx->memory_size += sizeof(SCACBSCtx);
/* initialize the hash we use to speed up pattern insertions */
SCACBSCtx *ctx = (SCACBSCtx *)mpm_ctx->ctx;
ctx->init_hash = SCMalloc(sizeof(SCACBSPattern *) * INIT_HASH_SIZE);
if (ctx->init_hash == NULL) {
mpm_ctx->init_hash = SCMalloc(sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE);
if (mpm_ctx->init_hash == NULL) {
exit(EXIT_FAILURE);
}
memset(ctx->init_hash, 0, sizeof(SCACBSPattern *) * INIT_HASH_SIZE);
memset(mpm_ctx->init_hash, 0, sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE);
/* get conf values for AC from our yaml file. We have no conf values for
* now. We will certainly need this, as we develop the algo */
@ -1336,25 +1063,25 @@ void SCACBSDestroyCtx(MpmCtx *mpm_ctx)
if (ctx == NULL)
return;
if (ctx->init_hash != NULL) {
SCFree(ctx->init_hash);
ctx->init_hash = NULL;
if (mpm_ctx->init_hash != NULL) {
SCFree(mpm_ctx->init_hash);
mpm_ctx->init_hash = NULL;
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= (INIT_HASH_SIZE * sizeof(SCACBSPattern *));
mpm_ctx->memory_size -= (MPM_INIT_HASH_SIZE * sizeof(MpmPattern *));
}
if (ctx->parray != NULL) {
uint32_t i;
for (i = 0; i < mpm_ctx->pattern_cnt; i++) {
if (ctx->parray[i] != NULL) {
SCACBSFreePattern(mpm_ctx, ctx->parray[i]);
MpmFreePattern(mpm_ctx, ctx->parray[i]);
}
}
SCFree(ctx->parray);
ctx->parray = NULL;
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(SCACBSPattern *));
mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
}
if (ctx->state_table_u16 != NULL) {
@ -1385,7 +1112,7 @@ void SCACBSDestroyCtx(MpmCtx *mpm_ctx)
if (ctx->pid_pat_list != NULL) {
uint32_t i;
for (i = 0; i < (ctx->max_pat_id + 1); i++) {
for (i = 0; i < (mpm_ctx->max_pat_id + 1); i++) {
if (ctx->pid_pat_list[i].cs != NULL)
SCFree(ctx->pid_pat_list[i].cs);
if (ctx->pid_pat_list[i].sids != NULL)
@ -1629,7 +1356,7 @@ int SCACBSAddPatternCI(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
SigIntId sid, uint8_t flags)
{
flags |= MPM_PATTERN_FLAG_NOCASE;
return SCACBSAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
}
/**
@ -1653,7 +1380,7 @@ int SCACBSAddPatternCS(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth, uint32_t pid,
SigIntId sid, uint8_t flags)
{
return SCACBSAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
}
void SCACBSPrintSearchStats(MpmThreadCtx *mpm_thread_ctx)
@ -1679,8 +1406,8 @@ void SCACBSPrintInfo(MpmCtx *mpm_ctx)
printf(" Sizeof:\n");
printf(" MpmCtx %" PRIuMAX "\n", (uintmax_t)sizeof(MpmCtx));
printf(" SCACBSCtx: %" PRIuMAX "\n", (uintmax_t)sizeof(SCACBSCtx));
printf(" SCACBSPattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACBSPattern));
printf(" SCACBSPattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACBSPattern));
printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern));
printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);

31
src/util-mpm-ac-bs.h
Unescape Escape View File

@ -22,30 +22,11 @@
*
*/
#include "util-mpm.h"
#define SC_AC_BS_STATE_TYPE_U16 uint16_t
#define SC_AC_BS_STATE_TYPE_U32 uint32_t
typedef struct SCACBSPattern_ {
/* length of the pattern */
uint16_t len;
/* flags decribing the pattern */
uint8_t flags;
/* holds the original pattern that was added */
uint8_t *original_pat;
/* case sensitive */
uint8_t *cs;
/* case INsensitive */
uint8_t *ci;
/* pattern id */
uint32_t id;
/* sid(s) for this pattern */
uint32_t sids_size;
SigIntId *sids;
struct SCACBSPattern_ *next;
} SCACBSPattern;
typedef struct SCACBSPatternList_ {
uint8_t *cs;
uint16_t patlen;
@ -63,16 +44,13 @@ typedef struct SCACBSOutputTable_ {
} SCACBSOutputTable;
typedef struct SCACBSCtx_ {
/* hash used during ctx initialization */
SCACBSPattern **init_hash;
/* pattern arrays. We need this only during the goto table creation phase */
SCACBSPattern **parray;
MpmPattern **parray;
/* no of states used by ac */
uint32_t state_count;
uint16_t pattern_id_bitarray_size;
uint32_t pattern_id_bitarray_size;
/* the all important memory hungry state_table */
SC_AC_BS_STATE_TYPE_U16 (*state_table_u16)[256];
@ -91,7 +69,6 @@ typedef struct SCACBSCtx_ {
/* the size of each state */
uint16_t single_state_size;
uint32_t max_pat_id;
} SCACBSCtx;
typedef struct SCACBSThreadCtx_ {

362
src/util-mpm-ac-tile.c
Unescape Escape View File

@ -143,8 +143,6 @@ static void SCACTileDestroyInitCtx(MpmCtx *mpm_ctx);
/* a placeholder to denote a failure transition in the goto table */
#define SC_AC_TILE_FAIL (-1)
/* size of the hash table used to speed up pattern insertions initially */
#define INIT_HASH_SIZE 65536
#define STATE_QUEUE_CONTAINER_SIZE 65536
@ -167,193 +165,6 @@ static void SCACTileGetConfig()
{
}
/**
* \internal
* \brief Compares 2 patterns. We use it for the hashing process during the
* the initial pattern insertion time, to cull duplicate sigs.
*
* \param p Pointer to the first pattern(SCACTilePattern).
* \param pat Pointer to the second pattern(raw pattern array).
* \param patlen Pattern length.
* \param flags Flags. We don't need this.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline int SCACTileCmpPattern(SCACTilePattern *p, uint8_t *pat,
uint16_t patlen, char flags)
{
if (p->len != patlen)
return 0;
if (p->flags != flags)
return 0;
if (flags & MPM_PATTERN_FLAG_NOCASE) {
// Case insensitive
if (SCMemcmpLowercase(p->cs, pat, patlen) != 0)
return 0;
} else {
// Case sensitive
if (SCMemcmp(p->cs, pat, patlen) != 0)
return 0;
}
return 1;
}
/**
* \internal
* \brief Creates a hash of the pattern. We use it for the hashing process
* during the initial pattern insertion time, to cull duplicate sigs.
*
* \param pat Pointer to the pattern.
* \param patlen Pattern length.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline uint32_t SCACTileInitHashRaw(uint8_t *pat, uint16_t patlen)
{
uint32_t hash = patlen * pat[0];
if (patlen > 1)
hash += pat[1];
return (hash % INIT_HASH_SIZE);
}
/**
* \internal
* \brief Looks up a pattern. We use it for the hashing process during the
* the initial pattern insertion time, to cull duplicate sigs.
*
* \param ctx Pointer to the AC ctx.
* \param pat Pointer to the pattern.
* \param patlen Pattern length.
* \param flags Flags. We don't need this.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline SCACTilePattern *SCACTileInitHashLookup(SCACTileCtx *ctx,
uint8_t *pat,
uint16_t patlen,
char flags,
uint32_t pid)
{
uint32_t hash = SCACTileInitHashRaw(pat, patlen);
if (ctx->init_hash == NULL) {
return NULL;
}
SCACTilePattern *t = ctx->init_hash[hash];
for ( ; t != NULL; t = t->next) {
if (t->id == pid) {
return t;
}
}
return NULL;
}
/**
* \internal
* \brief Allocs a new pattern instance.
*
* \param mpm_ctx Pointer to the mpm context.
*
* \retval p Pointer to the newly created pattern.
*/
static inline SCACTilePattern *SCACTileAllocPattern(MpmCtx *mpm_ctx)
{
SCACTilePattern *p = SCMalloc(sizeof(SCACTilePattern));
if (unlikely(p == NULL)) {
exit(EXIT_FAILURE);
}
memset(p, 0, sizeof(SCACTilePattern));
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += sizeof(SCACTilePattern);
return p;
}
/**
* \internal
* \brief Used to free SCACTilePattern instances.
*
* \param mpm_ctx Pointer to the mpm context.
* \param p Pointer to the SCACTilePattern instance to be freed.
* \param free Free the above pointer or not.
*/
static void SCACTileFreePattern(MpmCtx *mpm_ctx, SCACTilePattern *p)
{
if (p != NULL && p->cs != NULL && p->cs != p->ci) {
SCFree(p->cs);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->ci != NULL) {
SCFree(p->ci);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->original_pat != NULL) {
SCFree(p->original_pat);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->sids != NULL) {
SCFree(p->sids);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= (p->sids_size * sizeof(SigIntId));
}
if (p != NULL) {
SCFree(p);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= sizeof(SCACTilePattern);
}
}
static inline uint32_t SCACTileInitHash(SCACTilePattern *p)
{
uint32_t hash = p->len * p->original_pat[0];
if (p->len > 1)
hash += p->original_pat[1];
return (hash % INIT_HASH_SIZE);
}
static inline int SCACTileInitHashAdd(SCACTileCtx *ctx, SCACTilePattern *p)
{
uint32_t hash = SCACTileInitHash(p);
if (ctx->init_hash == NULL) {
return 0;
}
if (ctx->init_hash[hash] == NULL) {
ctx->init_hash[hash] = p;
return 0;
}
SCACTilePattern *tt = NULL;
SCACTilePattern *t = ctx->init_hash[hash];
/* get the list tail */
do {
tt = t;
t = t->next;
} while (t != NULL);
tt->next = p;
return 0;
}
/**
* \internal
@ -362,7 +173,7 @@ static inline int SCACTileInitHashAdd(SCACTileCtx *ctx, SCACTilePattern *p)
* characters, so could just set to 1 instead of counting.
*/
static inline void SCACTileHistogramAlphabet(SCACTileCtx *ctx,
SCACTilePattern *p)
MpmPattern *p)
{
for (int i = 0; i < p->len; i++) {
ctx->alpha_hist[p->ci[i]]++;
@ -413,137 +224,6 @@ static void SCACTileInitTranslateTable(SCACTileCtx *ctx)
ctx->alphabet_storage = 256;
}
/**
* \internal
* \brief Add a pattern to the mpm-ac context.
*
* \param mpm_ctx Mpm context.
* \param pat Pointer to the pattern.
* \param patlen Length of the pattern.
* \param pid Pattern id
* \param sid Signature id (internal id).
* \param flags Pattern's MPM_PATTERN_* flags.
*
* \retval 0 On success.
* \retval -1 On failure.
*/
static int SCACTileAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth, uint32_t pid,
uint32_t sid, uint8_t flags)
{
SCACTileSearchCtx *search_ctx = (SCACTileSearchCtx *)mpm_ctx->ctx;
SCACTileCtx *ctx = search_ctx->init_ctx;
SCLogDebug("Adding pattern for ctx %p, patlen %"PRIu16" and pid %" PRIu32,
ctx, patlen, pid);
if (patlen == 0) {
SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0");
return 0;
}
/* Check if we have already inserted this pattern. */
SCACTilePattern *p = SCACTileInitHashLookup(ctx, pat, patlen, flags, pid);
if (p == NULL) {
SCLogDebug("Allocing new pattern");
/* p will never be NULL */
p = SCACTileAllocPattern(mpm_ctx);
p->len = patlen;
p->flags = flags;
p->id = pid;
p->original_pat = SCMalloc(patlen);
if (p->original_pat == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy(p->original_pat, pat, patlen);
p->ci = SCMalloc(patlen);
if (p->ci == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy_tolower(p->ci, pat, patlen);
/* setup the case sensitive part of the pattern */
if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
/* nocase means no difference between cs and ci */
p->cs = p->ci;
} else {
if (memcmp(p->ci, pat, p->len) == 0) {
/* no diff between cs and ci: pat is lowercase */
p->cs = p->ci;
} else {
p->cs = SCMalloc(patlen);
if (p->cs == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy(p->cs, pat, patlen);
}
}
/* put in the pattern hash */
SCACTileInitHashAdd(ctx, p);
/* Count alphabet usages */
SCACTileHistogramAlphabet(ctx, p);
//if (mpm_ctx->pattern_cnt == 65535) {
// SCLogError(SC_ERR_AHO_CORASICK, "Max search words reached. Can't "
// "insert anymore. Exiting");
// exit(EXIT_FAILURE);
//}
mpm_ctx->pattern_cnt++;
if (mpm_ctx->maxlen < patlen)
mpm_ctx->maxlen = patlen;
if (mpm_ctx->minlen == 0) {
mpm_ctx->minlen = patlen;
} else {
if (mpm_ctx->minlen > patlen)
mpm_ctx->minlen = patlen;
}
p->sids_size = 1;
p->sids = SCMalloc(p->sids_size * sizeof(SigIntId));
BUG_ON(p->sids == NULL);
p->sids[0] = sid;
//SCLogInfo("MPM added %u:%u", pid, sid);
} else {
/* TODO figure out how we can be called multiple times for the same CTX with the same sid */
int found = 0;
uint32_t x = 0;
for (x = 0; x < p->sids_size; x++) {
if (p->sids[x] == sid) {
found = 1;
break;
}
}
if (!found) {
SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1)));
BUG_ON(sids == NULL);
p->sids = sids;
p->sids[p->sids_size] = sid;
p->sids_size++;
//SCLogInfo("p->sids_size %u", p->sids_size);
//SCLogInfo("MPM added %u:%u (append)", pid, sid);
} else {
//SCLogInfo("rule %u already part of pid %u", sid, pid);
}
}
return 0;
error:
SCACTileFreePattern(mpm_ctx, p);
return -1;
}
static void SCACTileReallocOutputTable(SCACTileCtx *ctx, int new_state_count)
{
@ -1198,33 +878,34 @@ int SCACTilePreparePatterns(MpmCtx *mpm_ctx)
return 0;
}
SCACTileCtx *ctx = search_ctx->init_ctx;
if (ctx->init_hash == NULL) {
if (mpm_ctx->init_hash == NULL) {
SCLogDebug("no patterns supplied to this mpm_ctx");
return 0;
}
/* alloc the pattern array */
ctx->parray = (SCACTilePattern **)SCMalloc(mpm_ctx->pattern_cnt *
sizeof(SCACTilePattern *));
ctx->parray = (MpmPattern **)SCMalloc(mpm_ctx->pattern_cnt *
sizeof(MpmPattern *));
if (ctx->parray == NULL)
goto error;
memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(SCACTilePattern *));
memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
/* populate it with the patterns in the hash */
uint32_t i = 0, p = 0;
for (i = 0; i < INIT_HASH_SIZE; i++) {
SCACTilePattern *node = ctx->init_hash[i], *nnode = NULL;
for (i = 0; i < MPM_INIT_HASH_SIZE; i++) {
MpmPattern *node = mpm_ctx->init_hash[i], *nnode = NULL;
while(node != NULL) {
nnode = node->next;
node->next = NULL;
ctx->parray[p++] = node;
SCACTileHistogramAlphabet(ctx, node);
node = nnode;
}
}
/* we no longer need the hash, so free it's memory */
SCFree(ctx->init_hash);
ctx->init_hash = NULL;
SCFree(mpm_ctx->init_hash);
mpm_ctx->init_hash = NULL;
/* Handle case patterns by storing a copy of the pattern to compare
* to each possible match (no-case).
@ -1336,12 +1017,11 @@ void SCACTileInitCtx(MpmCtx *mpm_ctx)
mpm_ctx->memory_size += sizeof(SCACTileCtx);
/* initialize the hash we use to speed up pattern insertions */
SCACTileCtx *ctx = search_ctx->init_ctx;
ctx->init_hash = SCMalloc(sizeof(SCACTilePattern *) * INIT_HASH_SIZE);
if (ctx->init_hash == NULL) {
mpm_ctx->init_hash = SCMalloc(sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE);
if (mpm_ctx->init_hash == NULL) {
exit(EXIT_FAILURE);
}
memset(ctx->init_hash, 0, sizeof(SCACTilePattern *) * INIT_HASH_SIZE);
memset(mpm_ctx->init_hash, 0, sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE);
/* get conf values for AC from our yaml file. We have no conf values for
* now. We will certainly need this, as we develop the algo */
@ -1374,16 +1054,16 @@ static void SCACTileDestroyInitCtx(MpmCtx *mpm_ctx)
if (ctx == NULL)
return;
if (ctx->init_hash != NULL) {
SCFree(ctx->init_hash);
ctx->init_hash = NULL;
if (mpm_ctx->init_hash != NULL) {
SCFree(mpm_ctx->init_hash);
mpm_ctx->init_hash = NULL;
}
if (ctx->parray != NULL) {
uint32_t i;
for (i = 0; i < mpm_ctx->pattern_cnt; i++) {
if (ctx->parray[i] != NULL) {
SCACTileFreePattern(mpm_ctx, ctx->parray[i]);
MpmFreePattern(mpm_ctx, ctx->parray[i]);
}
}
@ -1739,7 +1419,7 @@ int SCACTileAddPatternCI(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
SigIntId sid, uint8_t flags)
{
flags |= MPM_PATTERN_FLAG_NOCASE;
return SCACTileAddPattern(mpm_ctx, pat, patlen, offset, depth,
return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth,
pid, sid, flags);
}
@ -1764,7 +1444,7 @@ int SCACTileAddPatternCS(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth, uint32_t pid,
SigIntId sid, uint8_t flags)
{
return SCACTileAddPattern(mpm_ctx, pat, patlen, offset, depth,
return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth,
pid, sid, flags);
}
@ -1789,8 +1469,8 @@ void SCACTilePrintInfo(MpmCtx *mpm_ctx)
printf(" Sizeof:\n");
printf(" MpmCtx %" PRIuMAX "\n", (uintmax_t)sizeof(MpmCtx));
printf(" SCACTileCtx: %" PRIuMAX "\n", (uintmax_t)sizeof(SCACTileCtx));
printf(" SCACTilePattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACTilePattern));
printf(" SCACTilePattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACTilePattern));
printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern));
printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);

26
src/util-mpm-ac-tile.h
Unescape Escape View File

@ -26,27 +26,6 @@
#ifndef __UTIL_MPM_AC_TILE__H__
#define __UTIL_MPM_AC_TILE__H__
typedef struct SCACTilePattern_ {
/* length of the pattern */
uint16_t len;
/* flags decribing the pattern */
uint8_t flags;
/* holds the original pattern that was added */
uint8_t *original_pat;
/* case sensitive */
uint8_t *cs;
/* case INsensitive */
uint8_t *ci;
/* pattern id */
uint32_t id;
/* sid(s) for this pattern */
uint32_t sids_size;
SigIntId *sids;
struct SCACTilePattern_ *next;
} SCACTilePattern;
typedef struct SCACTilePatternList_ {
uint8_t *cs;
uint16_t patlen;
@ -98,12 +77,9 @@ typedef struct SCACTileCtx_ {
/* Indexed by MpmPatternIndex */
SCACTilePatternList *pattern_list;
/* hash used during ctx initialization */
SCACTilePattern **init_hash;
/* pattern arrays. We need this only during the goto table
creation phase */
SCACTilePattern **parray;
MpmPattern **parray;
/* goto_table, failure table and output table. Needed to create
* state_table. Will be freed, once we have created the

340
src/util-mpm-ac.c
Unescape Escape View File

@ -86,8 +86,6 @@ void SCACRegisterTests(void);
/* a placeholder to denote a failure transition in the goto table */
#define SC_AC_FAIL (-1)
/* size of the hash table used to speed up pattern insertions initially */
#define INIT_HASH_SIZE 65536
#define STATE_QUEUE_CONTAINER_SIZE 65536
@ -118,286 +116,6 @@ static void SCACGetConfig()
return;
}
/**
* \internal
* \brief Creates a hash of the pattern. We use it for the hashing process
* during the initial pattern insertion time, to cull duplicate sigs.
*
* \param pat Pointer to the pattern.
* \param patlen Pattern length.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline uint32_t SCACInitHashRaw(uint8_t *pat, uint16_t patlen)
{
uint32_t hash = patlen * pat[0];
if (patlen > 1)
hash += pat[1];
return (hash % INIT_HASH_SIZE);
}
/**
* \internal
* \brief Looks up a pattern. We use it for the hashing process during the
* the initial pattern insertion time, to cull duplicate sigs.
*
* \param ctx Pointer to the AC ctx.
* \param pat Pointer to the pattern.
* \param patlen Pattern length.
* \param flags Flags. We don't need this.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline SCACPattern *SCACInitHashLookup(SCACCtx *ctx, uint8_t *pat,
uint16_t patlen, char flags,
uint32_t pid)
{
uint32_t hash = SCACInitHashRaw(pat, patlen);
if (ctx->init_hash == NULL) {
return NULL;
}
SCACPattern *t = ctx->init_hash[hash];
for ( ; t != NULL; t = t->next) {
if (t->id == pid)
return t;
}
return NULL;
}
/**
* \internal
* \brief Allocs a new pattern instance.
*
* \param mpm_ctx Pointer to the mpm context.
*
* \retval p Pointer to the newly created pattern.
*/
static inline SCACPattern *SCACAllocPattern(MpmCtx *mpm_ctx)
{
SCACPattern *p = SCMalloc(sizeof(SCACPattern));
if (unlikely(p == NULL)) {
exit(EXIT_FAILURE);
}
memset(p, 0, sizeof(SCACPattern));
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += sizeof(SCACPattern);
return p;
}
/**
* \internal
* \brief Used to free SCACPattern instances.
*
* \param mpm_ctx Pointer to the mpm context.
* \param p Pointer to the SCACPattern instance to be freed.
* \param free Free the above pointer or not.
*/
static inline void SCACFreePattern(MpmCtx *mpm_ctx, SCACPattern *p)
{
if (p != NULL && p->cs != NULL && p->cs != p->ci) {
SCFree(p->cs);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->ci != NULL) {
SCFree(p->ci);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->original_pat != NULL) {
SCFree(p->original_pat);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->sids != NULL) {
SCFree(p->sids);
}
if (p != NULL) {
SCFree(p);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= sizeof(SCACPattern);
}
return;
}
static inline uint32_t SCACInitHash(SCACPattern *p)
{
uint32_t hash = p->len * p->original_pat[0];
if (p->len > 1)
hash += p->original_pat[1];
return (hash % INIT_HASH_SIZE);
}
static inline int SCACInitHashAdd(SCACCtx *ctx, SCACPattern *p)
{
uint32_t hash = SCACInitHash(p);
if (ctx->init_hash == NULL) {
return 0;
}
if (ctx->init_hash[hash] == NULL) {
ctx->init_hash[hash] = p;
return 0;
}
SCACPattern *tt = NULL;
SCACPattern *t = ctx->init_hash[hash];
/* get the list tail */
do {
tt = t;
t = t->next;
} while (t != NULL);
tt->next = p;
return 0;
}
/**
* \internal
* \brief Add a pattern to the mpm-ac context.
*
* \param mpm_ctx Mpm context.
* \param pat Pointer to the pattern.
* \param patlen Length of the pattern.
* \param pid Pattern id
* \param sid Signature id (internal id).
* \param flags Pattern's MPM_PATTERN_* flags.
*
* \retval 0 On success.
* \retval -1 On failure.
*/
static int SCACAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth, uint32_t pid,
SigIntId sid, uint8_t flags)
{
SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx;
SCLogDebug("Adding pattern for ctx %p, patlen %"PRIu16" and pid %" PRIu32,
ctx, patlen, pid);
if (patlen == 0) {
SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0");
return 0;
}
/* check if we have already inserted this pattern */
SCACPattern *p = SCACInitHashLookup(ctx, pat, patlen, flags, pid);
if (p == NULL) {
SCLogDebug("Allocing new pattern");
/* p will never be NULL */
p = SCACAllocPattern(mpm_ctx);
p->len = patlen;
p->flags = flags;
p->id = pid;
p->original_pat = SCMalloc(patlen);
if (p->original_pat == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy(p->original_pat, pat, patlen);
p->ci = SCMalloc(patlen);
if (p->ci == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy_tolower(p->ci, pat, patlen);
/* setup the case sensitive part of the pattern */
if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
/* nocase means no difference between cs and ci */
p->cs = p->ci;
} else {
if (memcmp(p->ci, pat, p->len) == 0) {
/* no diff between cs and ci: pat is lowercase */
p->cs = p->ci;
} else {
p->cs = SCMalloc(patlen);
if (p->cs == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy(p->cs, pat, patlen);
}
}
/* put in the pattern hash */
SCACInitHashAdd(ctx, p);
//if (mpm_ctx->pattern_cnt == 65535) {
// SCLogError(SC_ERR_AHO_CORASICK, "Max search words reached. Can't "
// "insert anymore. Exiting");
// exit(EXIT_FAILURE);
//}
mpm_ctx->pattern_cnt++;
if (mpm_ctx->maxlen < patlen)
mpm_ctx->maxlen = patlen;
if (mpm_ctx->minlen == 0) {
mpm_ctx->minlen = patlen;
} else {
if (mpm_ctx->minlen > patlen)
mpm_ctx->minlen = patlen;
}
/* we need the max pat id */
if (pid > ctx->max_pat_id)
ctx->max_pat_id = pid;
p->sids_size = 1;
p->sids = SCMalloc(p->sids_size * sizeof(SigIntId));
BUG_ON(p->sids == NULL);
p->sids[0] = sid;
//SCLogInfo("MPM added %u:%u", pid, sid);
} else {
/* TODO figure out how we can be called multiple times for the same CTX with the same sid */
int found = 0;
uint32_t x = 0;
for (x = 0; x < p->sids_size; x++) {
if (p->sids[x] == sid) {
found = 1;
break;
}
}
if (!found) {
SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1)));
BUG_ON(sids == NULL);
p->sids = sids;
p->sids[p->sids_size] = sid;
p->sids_size++;
//SCLogInfo("p->sids_size %u", p->sids_size);
//SCLogInfo("MPM added %u:%u (append)", pid, sid);
} else {
//SCLogInfo("rule %u already part of pid %u", sid, pid);
}
}
return 0;
error:
SCACFreePattern(mpm_ctx, p);
return -1;
}
/**
* \internal
* \brief Initialize a new state in the goto and output tables.
@ -1036,24 +754,24 @@ int SCACPreparePatterns(MpmCtx *mpm_ctx)
{
SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx;
if (mpm_ctx->pattern_cnt == 0 || ctx->init_hash == NULL) {
if (mpm_ctx->pattern_cnt == 0 || mpm_ctx->init_hash == NULL) {
SCLogDebug("no patterns supplied to this mpm_ctx");
return 0;
}
/* alloc the pattern array */
ctx->parray = (SCACPattern **)SCMalloc(mpm_ctx->pattern_cnt *
sizeof(SCACPattern *));
ctx->parray = (MpmPattern **)SCMalloc(mpm_ctx->pattern_cnt *
sizeof(MpmPattern *));
if (ctx->parray == NULL)
goto error;
memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(SCACPattern *));
memset(ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(SCACPattern *));
mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
/* populate it with the patterns in the hash */
uint32_t i = 0, p = 0;
for (i = 0; i < INIT_HASH_SIZE; i++) {
SCACPattern *node = ctx->init_hash[i], *nnode = NULL;
for (i = 0; i < MPM_INIT_HASH_SIZE; i++) {
MpmPattern *node = mpm_ctx->init_hash[i], *nnode = NULL;
while(node != NULL) {
nnode = node->next;
node->next = NULL;
@ -1063,19 +781,19 @@ int SCACPreparePatterns(MpmCtx *mpm_ctx)
}
/* we no longer need the hash, so free it's memory */
SCFree(ctx->init_hash);
ctx->init_hash = NULL;
SCFree(mpm_ctx->init_hash);
mpm_ctx->init_hash = NULL;
/* the memory consumed by a single state in our goto table */
ctx->single_state_size = sizeof(int32_t) * 256;
/* handle no case patterns */
ctx->pid_pat_list = SCMalloc((ctx->max_pat_id + 1)* sizeof(SCACPatternList));
ctx->pid_pat_list = SCMalloc((mpm_ctx->max_pat_id + 1)* sizeof(SCACPatternList));
if (ctx->pid_pat_list == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
exit(EXIT_FAILURE);
}
memset(ctx->pid_pat_list, 0, (ctx->max_pat_id + 1) * sizeof(SCACPatternList));
memset(ctx->pid_pat_list, 0, (mpm_ctx->max_pat_id + 1) * sizeof(SCACPatternList));
for (i = 0; i < mpm_ctx->pattern_cnt; i++) {
if (!(ctx->parray[i]->flags & MPM_PATTERN_FLAG_NOCASE)) {
@ -1123,15 +841,16 @@ int SCACPreparePatterns(MpmCtx *mpm_ctx)
/* free all the stored patterns. Should save us a good 100-200 mbs */
for (i = 0; i < mpm_ctx->pattern_cnt; i++) {
if (ctx->parray[i] != NULL) {
SCACFreePattern(mpm_ctx, ctx->parray[i]);
MpmFreePattern(mpm_ctx, ctx->parray[i]);
}
}
SCFree(ctx->parray);
ctx->parray = NULL;
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(SCACPattern *));
mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
ctx->pattern_id_bitarray_size = (ctx->max_pat_id / 8) + 1;
ctx->pattern_id_bitarray_size = (mpm_ctx->max_pat_id / 8) + 1;
SCLogDebug("ctx->pattern_id_bitarray_size %u", ctx->pattern_id_bitarray_size);
return 0;
@ -1181,12 +900,11 @@ void SCACInitCtx(MpmCtx *mpm_ctx)
mpm_ctx->memory_size += sizeof(SCACCtx);
/* initialize the hash we use to speed up pattern insertions */
SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx;
ctx->init_hash = SCMalloc(sizeof(SCACPattern *) * INIT_HASH_SIZE);
if (ctx->init_hash == NULL) {
mpm_ctx->init_hash = SCMalloc(sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE);
if (mpm_ctx->init_hash == NULL) {
exit(EXIT_FAILURE);
}
memset(ctx->init_hash, 0, sizeof(SCACPattern *) * INIT_HASH_SIZE);
memset(mpm_ctx->init_hash, 0, sizeof(MpmPattern *) * MPM_INIT_HASH_SIZE);
/* get conf values for AC from our yaml file. We have no conf values for
* now. We will certainly need this, as we develop the algo */
@ -1226,25 +944,25 @@ void SCACDestroyCtx(MpmCtx *mpm_ctx)
if (ctx == NULL)
return;
if (ctx->init_hash != NULL) {
SCFree(ctx->init_hash);
ctx->init_hash = NULL;
if (mpm_ctx->init_hash != NULL) {
SCFree(mpm_ctx->init_hash);
mpm_ctx->init_hash = NULL;
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= (INIT_HASH_SIZE * sizeof(SCACPattern *));
mpm_ctx->memory_size -= (MPM_INIT_HASH_SIZE * sizeof(MpmPattern *));
}
if (ctx->parray != NULL) {
uint32_t i;
for (i = 0; i < mpm_ctx->pattern_cnt; i++) {
if (ctx->parray[i] != NULL) {
SCACFreePattern(mpm_ctx, ctx->parray[i]);
MpmFreePattern(mpm_ctx, ctx->parray[i]);
}
}
SCFree(ctx->parray);
ctx->parray = NULL;
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(SCACPattern *));
mpm_ctx->memory_size -= (mpm_ctx->pattern_cnt * sizeof(MpmPattern *));
}
if (ctx->state_table_u16 != NULL) {
@ -1276,7 +994,7 @@ void SCACDestroyCtx(MpmCtx *mpm_ctx)
if (ctx->pid_pat_list != NULL) {
uint32_t i;
for (i = 0; i < (ctx->max_pat_id + 1); i++) {
for (i = 0; i < (mpm_ctx->max_pat_id + 1); i++) {
if (ctx->pid_pat_list[i].cs != NULL)
SCFree(ctx->pid_pat_list[i].cs);
if (ctx->pid_pat_list[i].sids != NULL)
@ -1425,7 +1143,7 @@ int SCACAddPatternCI(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
SigIntId sid, uint8_t flags)
{
flags |= MPM_PATTERN_FLAG_NOCASE;
return SCACAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
}
/**
@ -1449,7 +1167,7 @@ int SCACAddPatternCS(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth, uint32_t pid,
SigIntId sid, uint8_t flags)
{
return SCACAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
return MpmAddPattern(mpm_ctx, pat, patlen, offset, depth, pid, sid, flags);
}
void SCACPrintSearchStats(MpmThreadCtx *mpm_thread_ctx)
@ -1475,8 +1193,8 @@ void SCACPrintInfo(MpmCtx *mpm_ctx)
printf(" Sizeof:\n");
printf(" MpmCtx %" PRIuMAX "\n", (uintmax_t)sizeof(MpmCtx));
printf(" SCACCtx: %" PRIuMAX "\n", (uintmax_t)sizeof(SCACCtx));
printf(" SCACPattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACPattern));
printf(" SCACPattern %" PRIuMAX "\n", (uintmax_t)sizeof(SCACPattern));
printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern));
printf(" MpmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(MpmPattern));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);

27
src/util-mpm-ac.h
Unescape Escape View File

@ -38,27 +38,6 @@
#include "flow.h"
#endif /* __SC_CUDA_SUPPORT__ */
typedef struct SCACPattern_ {
/* length of the pattern */
uint16_t len;
/* flags decribing the pattern */
uint8_t flags;
/* holds the original pattern that was added */
uint8_t *original_pat;
/* case sensitive */
uint8_t *cs;
/* case INsensitive */
uint8_t *ci;
/* pattern id */
uint32_t id;
/* sid(s) for this pattern */
uint32_t sids_size;
SigIntId *sids;
struct SCACPattern_ *next;
} SCACPattern;
typedef struct SCACPatternList_ {
uint8_t *cs;
uint16_t patlen;
@ -76,11 +55,8 @@ typedef struct SCACOutputTable_ {
} SCACOutputTable;
typedef struct SCACCtx_ {
/* hash used during ctx initialization */
SCACPattern **init_hash;
/* pattern arrays. We need this only during the goto table creation phase */
SCACPattern **parray;
MpmPattern **parray;
/* no of states used by ac */
uint32_t state_count;
@ -101,7 +77,6 @@ typedef struct SCACCtx_ {
/* the size of each state */
uint32_t single_state_size;
uint32_t max_pat_id;
uint32_t allocated_state_count;

282
src/util-mpm.c
Unescape Escape View File

@ -45,6 +45,7 @@
#include "util-cuda-handlers.h"
#include "detect-engine-mpm.h"
#endif
#include "util-memcpy.h"
/**
* \brief Register a new Mpm Context.
@ -556,6 +557,287 @@ int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen,
}
/**
* \internal
* \brief Creates a hash of the pattern. We use it for the hashing process
* during the initial pattern insertion time, to cull duplicate sigs.
*
* \param pat Pointer to the pattern.
* \param patlen Pattern length.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline uint32_t MpmInitHashRaw(uint8_t *pat, uint16_t patlen)
{
uint32_t hash = patlen * pat[0];
if (patlen > 1)
hash += pat[1];
return (hash % MPM_INIT_HASH_SIZE);
}
/**
* \internal
* \brief Looks up a pattern. We use it for the hashing process during the
* the initial pattern insertion time, to cull duplicate sigs.
*
* \param ctx Pointer to the AC ctx.
* \param pat Pointer to the pattern.
* \param patlen Pattern length.
* \param flags Flags. We don't need this.
*
* \retval hash A 32 bit unsigned hash.
*/
static inline MpmPattern *MpmInitHashLookup(MpmCtx *ctx, uint8_t *pat,
uint16_t patlen, char flags,
uint32_t pid)
{
uint32_t hash = MpmInitHashRaw(pat, patlen);
if (ctx->init_hash == NULL) {
return NULL;
}
MpmPattern *t = ctx->init_hash[hash];
for ( ; t != NULL; t = t->next) {
if (!(flags & MPM_PATTERN_CTX_OWNS_ID)) {
if (t->id == pid)
return t;
} else {
if (t->len == patlen &&
memcmp(pat, t->original_pat, patlen) == 0 &&
t->flags == flags)
{
return t;
}
}
}
return NULL;
}
/**
* \internal
* \brief Allocs a new pattern instance.
*
* \param mpm_ctx Pointer to the mpm context.
*
* \retval p Pointer to the newly created pattern.
*/
static inline MpmPattern *MpmAllocPattern(MpmCtx *mpm_ctx)
{
MpmPattern *p = SCMalloc(sizeof(MpmPattern));
if (unlikely(p == NULL)) {
exit(EXIT_FAILURE);
}
memset(p, 0, sizeof(MpmPattern));
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += sizeof(MpmPattern);
return p;
}
/**
* \internal
* \brief Used to free MpmPattern instances.
*
* \param mpm_ctx Pointer to the mpm context.
* \param p Pointer to the MpmPattern instance to be freed.
*/
void MpmFreePattern(MpmCtx *mpm_ctx, MpmPattern *p)
{
if (p != NULL && p->cs != NULL && p->cs != p->ci) {
SCFree(p->cs);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->ci != NULL) {
SCFree(p->ci);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL && p->original_pat != NULL) {
SCFree(p->original_pat);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= p->len;
}
if (p != NULL) {
SCFree(p);
mpm_ctx->memory_cnt--;
mpm_ctx->memory_size -= sizeof(MpmPattern);
}
return;
}
static inline uint32_t MpmInitHash(MpmPattern *p)
{
uint32_t hash = p->len * p->original_pat[0];
if (p->len > 1)
hash += p->original_pat[1];
return (hash % MPM_INIT_HASH_SIZE);
}
static inline int MpmInitHashAdd(MpmCtx *ctx, MpmPattern *p)
{
uint32_t hash = MpmInitHash(p);
if (ctx->init_hash == NULL) {
return 0;
}
if (ctx->init_hash[hash] == NULL) {
ctx->init_hash[hash] = p;
return 0;
}
MpmPattern *tt = NULL;
MpmPattern *t = ctx->init_hash[hash];
/* get the list tail */
do {
tt = t;
t = t->next;
} while (t != NULL);
tt->next = p;
return 0;
}
/**
* \internal
* \brief Add a pattern to the mpm-ac context.
*
* \param mpm_ctx Mpm context.
* \param pat Pointer to the pattern.
* \param patlen Length of the pattern.
* \param pid Pattern id
* \param sid Signature id (internal id).
* \param flags Pattern's MPM_PATTERN_* flags.
*
* \retval 0 On success.
* \retval -1 On failure.
*/
int MpmAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth, uint32_t pid,
SigIntId sid, uint8_t flags)
{
SCLogDebug("Adding pattern for ctx %p, patlen %"PRIu16" and pid %" PRIu32,
mpm_ctx, patlen, pid);
if (patlen == 0) {
SCLogWarning(SC_ERR_INVALID_ARGUMENTS, "pattern length 0");
return 0;
}
if (flags & MPM_PATTERN_CTX_OWNS_ID)
pid = UINT_MAX;
/* check if we have already inserted this pattern */
MpmPattern *p = MpmInitHashLookup(mpm_ctx, pat, patlen, flags, pid);
if (p == NULL) {
SCLogDebug("Allocing new pattern");
/* p will never be NULL */
p = MpmAllocPattern(mpm_ctx);
p->len = patlen;
p->flags = flags;
if (flags & MPM_PATTERN_CTX_OWNS_ID)
p->id = mpm_ctx->max_pat_id++;
else
p->id = pid;
p->original_pat = SCMalloc(patlen);
if (p->original_pat == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy(p->original_pat, pat, patlen);
p->ci = SCMalloc(patlen);
if (p->ci == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy_tolower(p->ci, pat, patlen);
/* setup the case sensitive part of the pattern */
if (p->flags & MPM_PATTERN_FLAG_NOCASE) {
/* nocase means no difference between cs and ci */
p->cs = p->ci;
} else {
if (memcmp(p->ci, pat, p->len) == 0) {
/* no diff between cs and ci: pat is lowercase */
p->cs = p->ci;
} else {
p->cs = SCMalloc(patlen);
if (p->cs == NULL)
goto error;
mpm_ctx->memory_cnt++;
mpm_ctx->memory_size += patlen;
memcpy(p->cs, pat, patlen);
}
}
/* put in the pattern hash */
MpmInitHashAdd(mpm_ctx, p);
mpm_ctx->pattern_cnt++;
if (mpm_ctx->maxlen < patlen)
mpm_ctx->maxlen = patlen;
if (mpm_ctx->minlen == 0) {
mpm_ctx->minlen = patlen;
} else {
if (mpm_ctx->minlen > patlen)
mpm_ctx->minlen = patlen;
}
/* we need the max pat id */
if (p->id > mpm_ctx->max_pat_id)
mpm_ctx->max_pat_id = p->id;
p->sids_size = 1;
p->sids = SCMalloc(p->sids_size * sizeof(SigIntId));
BUG_ON(p->sids == NULL);
p->sids[0] = sid;
} else {
/* we can be called multiple times for the same sid in the case
* of the 'single' modus. Here multiple rule groups share the
* same mpm ctx and might be adding the same pattern to the
* mpm_ctx */
int found = 0;
uint32_t x = 0;
for (x = 0; x < p->sids_size; x++) {
if (p->sids[x] == sid) {
found = 1;
break;
}
}
if (!found) {
SigIntId *sids = SCRealloc(p->sids, (sizeof(SigIntId) * (p->sids_size + 1)));
BUG_ON(sids == NULL);
p->sids = sids;
p->sids[p->sids_size] = sid;
p->sids_size++;
}
}
return 0;
error:
MpmFreePattern(mpm_ctx, p);
return -1;
}
/************************************Unittests*********************************/

38
src/util-mpm.h
Unescape Escape View File

@ -25,6 +25,9 @@
#define __UTIL_MPM_H__
#include "suricata-common.h"
#define MPM_INIT_HASH_SIZE 65536
enum {
MPM_NOTSET = 0,
@ -76,6 +79,27 @@ typedef struct PatternMatcherQueue_ {
} PatternMatcherQueue;
typedef struct MpmPattern_ {
/* length of the pattern */
uint16_t len;
/* flags decribing the pattern */
uint8_t flags;
/* holds the original pattern that was added */
uint8_t *original_pat;
/* case sensitive */
uint8_t *cs;
/* case INsensitive */
uint8_t *ci;
/* pattern id */
uint32_t id;
/* sid(s) for this pattern */
uint32_t sids_size;
SigIntId *sids;
struct MpmPattern_ *next;
} MpmPattern;
typedef struct MpmCtx_ {
void *ctx;
uint16_t mpm_type;
@ -94,6 +118,11 @@ typedef struct MpmCtx_ {
uint32_t memory_cnt;
uint32_t memory_size;
uint32_t max_pat_id;
/* hash used during ctx initialization */
MpmPattern **init_hash;
} MpmCtx;
/* if we want to retrieve an unique mpm context from the mpm context factory
@ -125,6 +154,9 @@ typedef struct MpmCtxFactoryContainer_ {
#define MPM_PATTERN_FLAG_OFFSET 0x08
/** one byte pattern (used in b2g) */
#define MPM_PATTERN_ONE_BYTE 0x10
/** the ctx uses it's own internal id instead of
* what is passed through the API */
#define MPM_PATTERN_CTX_OWNS_ID 0x20
typedef struct MpmTableElmt_ {
char *name;
@ -220,6 +252,12 @@ int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth,
uint32_t pid, SigIntId sid, uint8_t flags);
void MpmFreePattern(MpmCtx *mpm_ctx, MpmPattern *p);
int MpmAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen,
uint16_t offset, uint16_t depth, uint32_t pid,
SigIntId sid, uint8_t flags);
/* Resize Signature ID array. Only called from MpmAddSids(). */
int MpmAddSidsResize(PatternMatcherQueue *pmq, uint32_t new_size);

Write Preview
Loading…
Cancel
Save