Further improve B2gc. Add B2gm. Improve memory layout.

remotes/origin/master-1.1.x
Victor Julien 15 years ago
parent 4329261faa
commit 87f88867f4

@ -141,6 +141,7 @@ util-mpm-b2g.c util-mpm-b2g.h \
util-mpm-b2g-cuda.c util-mpm-b2g-cuda.h \
util-mpm-b3g.c util-mpm-b3g.h \
util-mpm-b2gc.c util-mpm-b2gc.h \
util-mpm-b2gm.c util-mpm-b2gm.h \
util-cidr.c util-cidr.h \
util-unittest.c util-unittest.h \
util-unittest-helper.c util-unittest-helper.h \
@ -169,6 +170,7 @@ util-strlcatu.c \
util-strlcpyu.c \
util-cuda.c util-cuda.h \
util-cuda-handlers.c util-cuda-handlers.h \
util-optimize.h \
util-privs.c util-privs.h \
util-decode-asn1.c util-decode-asn1.h \
util-ringbuffer.c util-ringbuffer.h \

@ -74,6 +74,7 @@ SCEnumCharMap sc_mpm_algo_map[] = {
{ "b2g_cuda", MPM_B2G_CUDA },
#endif
{ "b2gc", MPM_B2GC },
{ "b2gm", MPM_B2GM },
};
/**

@ -335,6 +335,7 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data) {
* table is always big enough
*/
PatternMatchThreadPrepare(&det_ctx->mtc, de_ctx->mpm_matcher, DetectContentMaxId(de_ctx));
PatternMatchThreadPrepare(&det_ctx->mtcs, de_ctx->mpm_matcher, DetectContentMaxId(de_ctx));
PatternMatchThreadPrepare(&det_ctx->mtcu, de_ctx->mpm_matcher, DetectUricontentMaxId(de_ctx));
//PmqSetup(&det_ctx->pmq, DetectEngineGetMaxSigId(de_ctx), DetectContentMaxId(de_ctx));

@ -119,7 +119,6 @@ void B2gPrintInfo(MpmCtx *mpm_ctx) {
printf(" B2gPattern %" PRIuMAX "\n", (uintmax_t)sizeof(B2gPattern));
printf(" B2gPattern %" PRIuMAX "\n", (uintmax_t)sizeof(B2gPattern));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Total Patterns: %" PRIu32 "\n", mpm_ctx->total_pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);
printf("Hash size: %" PRIu32 "\n", ctx->hash_size);
@ -353,7 +352,6 @@ static int B2gAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_
else if (mpm_ctx->minlen > patlen) mpm_ctx->minlen = patlen;
}
mpm_ctx->total_pattern_cnt++;
return 0;
error:

@ -55,9 +55,9 @@ typedef struct B2gPattern_ {
uint16_t len; /**< \todo we're limited to 32/64 byte lengths, uint8_t would be fine here */
uint8_t flags;
uint8_t pad0;
uint32_t id;
uint8_t *ci; /* case INsensitive */
uint8_t *cs; /* case sensitive */
uint32_t id;
struct B2gPattern_ *next;
} B2gPattern;

File diff suppressed because it is too large Load Diff

@ -27,6 +27,13 @@
#include "util-mpm.h"
#include "util-bloomfilter.h"
#define B2GC_HASHSHIFT_MAX 8
#define B2GC_HASHSHIFT_HIGHEST 7
#define B2GC_HASHSHIFT_HIGH 6
#define B2GC_HASHSHIFT_MEDIUM 5
#define B2GC_HASHSHIFT_LOW 4
#define B2GC_HASHSHIFT_LOWEST 3
//#define B2GC_HASHSHIFT 8
//#define B2GC_HASHSHIFT 7
//#define B2GC_HASHSHIFT 6
@ -43,7 +50,6 @@
//#define B2GC_WORD_SIZE 16
//#define B2GC_WORD_SIZE 8
#define B2GC_HASH16(a,b) (((a)<<B2GC_HASHSHIFT) | (b))
#define B2GC_Q 2
#define B2GC_SEARCHFUNC B2gcSearchBNDMq
@ -63,9 +69,10 @@
*/
typedef struct B2gcPatternHdr_ {
uint32_t flags:4; /* 4 flags */
uint32_t len:10; /* max 1023 */
uint32_t id:18; /* max 256k */
uint32_t np_offset; /* offset of the next pattern */
uint8_t len;
uint8_t flags;
uint16_t id;
} B2gcPatternHdr;
#define B2GC_GET_FLAGS(hdr) ((hdr)->flags)
@ -78,10 +85,9 @@ typedef struct B2gcPatternHdr_ {
*/
typedef struct B2gcPattern1_ {
uint32_t flags:4; /**< 4 flags max */
uint32_t id:18; /**< pattern id, max 262143 */
uint32_t pad:2; /**< 2 bits of padding */
uint32_t pat:8; /**< character to match */
uint8_t flags;
uint8_t pat;
uint16_t id;
} B2gcPattern1;
#define B2GC1_GET_FLAGS(hdr) ((hdr)->flags)
@ -93,41 +99,34 @@ typedef struct B2gcPattern_ {
uint16_t len;
uint8_t flags;
uint8_t pad0;
uint8_t *pat;
uint32_t id;
uint8_t *pat;
} B2gcPattern;
typedef struct B2gcCtx_ {
B2GC_TYPE m;
B2GC_TYPE *B2G;
/* we store our own multi byte search func ptr here for B2gcSearch1 */
uint32_t (*Search)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
/* hash for looking up the idx in the pattern array */
uint16_t *ha1;
uint8_t *patterns1;
uint32_t pat_x_cnt;
uint32_t pat_1_cnt;
/* we store our own multi byte search func ptr here for B2gcSearch1 */
uint32_t (*MBSearch)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
uint32_t *ha;
/* patterns in the format |hdr|pattern|hdr|pattern|... */
uint8_t *patterns;
B2GC_TYPE m;
uint32_t hash_size;
B2GC_TYPE *B2GC;
/* we store our own multi byte search func ptr here for B2gcSearch1 */
uint32_t (*Search)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
uint8_t *pminlen; /* array containing the minimal length */
BloomFilter **bloom;
/* we store our own multi byte search func ptr here for B2gcSearch1 */
uint32_t (*MBSearch2)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
uint32_t (*MBSearch)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
uint32_t *ha;
/* patterns in the format |hdr|pattern|hdr|pattern|... */
uint8_t *patterns;
/* Hash table used at ctx initialization to keep and ordered list of
* patterns. The ordered list is used to build the ordered lookup
* array. */
HashListTable *b2gc_init_hash;
HashListTable *b2gc_sort_hash;
HashListTable *b2gc_sort_hash1;
uint32_t pat_1_cnt;
} B2gcCtx;
typedef struct B2gcThreadCtx_ {

File diff suppressed because it is too large Load Diff

@ -0,0 +1,143 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*/
#ifndef __UTIL_MPM_B2GM_H__
#define __UTIL_MPM_B2GM_H__
#include "util-mpm.h"
#include "util-bloomfilter.h"
#define B2GM_HASHSHIFT_MAX 8
#define B2GM_HASHSHIFT_HIGHEST 7
#define B2GM_HASHSHIFT_HIGH 6
#define B2GM_HASHSHIFT_MEDIUM 5
#define B2GM_HASHSHIFT_LOW 4
#define B2GM_HASHSHIFT_LOWEST 3
//#define B2GM_TYPE uint64_t
#define B2GM_TYPE uint32_t
//#define B2GM_TYPE uint16_t
//#define B2GM_TYPE uint8_t
//#define B2GM_WORD_SIZE 64
#define B2GM_WORD_SIZE 32
//#define B2GM_WORD_SIZE 16
//#define B2GM_WORD_SIZE 8
#define B2GM_Q 2
#define B2GM_SEARCHFUNC B2gmSearchBNDMq
//#define B2GM_SEARCHFUNC B2gmSearch
//#define B2GM_COUNTERS
#define B2GM_FLAG_NOCASE 0x01
#define B2GM_FLAG_FINAL 0x02
typedef struct B2gmPattern_ {
uint8_t len;
uint8_t flags;
uint16_t id;
#if __WORDSIZE == 64
uint32_t pad;
#endif
uint8_t *pat;
struct B2gmPattern_ *next;
} B2gmPattern;
typedef struct B2gmPattern1_ {
uint8_t flags;
uint8_t pat;
uint16_t id;
} B2gmPattern1;
typedef struct B2gmLookup_ {
uint16_t pminlen;
uint8_t pminlenb; /* bloom */
uint8_t pad0;
#if __WORDSIZE == 64
uint32_t pad1;
#endif
BloomFilter *bloom;
B2gmPattern *hash;
} B2gmLookup;
typedef struct B2gmCtx_ {
/* we store our own multi byte search func ptr here for B2gmSearch1 */
uint32_t (*Search)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
/* hash for looking up the idx in the pattern array */
uint16_t *ha1;
uint8_t *patterns1;
/* we store our own multi byte search func ptr here for B2gmSearch1 */
//uint32_t (*MBSearch2)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
uint32_t (*MBSearch)(struct MpmCtx_ *, struct MpmThreadCtx_ *, PatternMatcherQueue *, uint8_t *, uint16_t);
uint16_t pat_1_cnt;
uint16_t pat_x_cnt;
#if __WORDSIZE == 64
uint32_t pad1;
#endif
B2GM_TYPE *B2GM;
B2GM_TYPE m;
#if __WORDSIZE == 64
uint32_t pad0;
#endif
B2gmLookup *lookup;
/* pattern arrays */
B2gmPattern **parray;
/* hash used during ctx initialization */
B2gmPattern **init_hash;
//uint8_t s0;
uint32_t hash_size;
} B2gmCtx;
typedef struct B2gmThreadCtx_ {
#ifdef B2GM_COUNTERS
uint32_t stat_pminlen_calls;
uint32_t stat_pminlen_total;
uint32_t stat_bloom_calls;
uint32_t stat_bloom_hits;
uint32_t stat_calls;
uint32_t stat_m_total;
uint32_t stat_d0;
uint32_t stat_d0_hashloop;
uint32_t stat_loop_match;
uint32_t stat_loop_no_match;
uint32_t stat_num_shift;
uint32_t stat_total_shift;
uint32_t stat_test_buf;
uint32_t stat_test_buf_ok;
uint32_t stat_test_buf_fail;
#endif /* B2GM_COUNTERS */
} B2gmThreadCtx;
void MpmB2gmRegister(void);
#endif /* __UTIL_MPM_B2GM_H__ */

@ -110,7 +110,6 @@ void B3gPrintInfo(MpmCtx *mpm_ctx) {
printf(" B3gPattern %" PRIuMAX "\n", (uintmax_t)sizeof(B3gPattern));
printf(" B3gHashItem %" PRIuMAX "\n", (uintmax_t)sizeof(B3gHashItem));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Total Patterns: %" PRIu32 "\n", mpm_ctx->total_pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);
printf("Hash size: %" PRIu32 "\n", ctx->hash_size);
@ -332,7 +331,6 @@ static int B3gAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_
else if (mpm_ctx->minlen > patlen) mpm_ctx->minlen = patlen;
}
mpm_ctx->total_pattern_cnt++;
return 0;
error:

@ -137,7 +137,6 @@ void WmPrintInfo(MpmCtx *mpm_ctx) {
printf(" WmPattern %" PRIuMAX "\n", (uintmax_t)sizeof(WmPattern));
printf(" WmHashItem %" PRIuMAX "\n", (uintmax_t)sizeof(WmHashItem));
printf("Unique Patterns: %" PRIu32 "\n", mpm_ctx->pattern_cnt);
printf("Total Patterns: %" PRIu32 "\n", mpm_ctx->total_pattern_cnt);
printf("Smallest: %" PRIu32 "\n", mpm_ctx->minlen);
printf("Largest: %" PRIu32 "\n", mpm_ctx->maxlen);
printf("Max shiftlen: %" PRIu32 "\n", ctx->shiftlen);
@ -385,7 +384,6 @@ static int WmAddPattern(MpmCtx *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t
else if (mpm_ctx->minlen > patlen) mpm_ctx->minlen = patlen;
}
mpm_ctx->total_pattern_cnt++;
return 0;
error:

@ -33,6 +33,7 @@
#include "util-mpm-b2g-cuda.h"
#include "util-mpm-b3g.h"
#include "util-mpm-b2gc.h"
#include "util-mpm-b2gm.h"
#include "util-hashlist.h"
/**
@ -218,6 +219,7 @@ void MpmTableSetup(void) {
#endif
MpmB3gRegister();
MpmB2gcRegister();
MpmB2gmRegister();
}
/** \brief Function to return the default hash size for the mpm algorithm,

@ -60,6 +60,7 @@ enum {
#endif
MPM_B3G,
MPM_B2GC,
MPM_B2GM,
/* table size */
MPM_TABLE_SIZE,
@ -94,14 +95,13 @@ typedef struct MpmCtx_ {
void *ctx;
uint16_t mpm_type;
uint32_t memory_cnt;
uint32_t memory_size;
uint32_t pattern_cnt; /* unique patterns */
uint32_t total_pattern_cnt; /* total patterns added */
uint16_t pattern_cnt; /* unique patterns */
uint16_t minlen;
uint16_t maxlen;
uint32_t memory_cnt;
uint32_t memory_size;
} MpmCtx;
/** pattern is case insensitive */

@ -0,0 +1,31 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
#ifndef __UTIL_OPTIMIZE_H__
#define __UTIL_OPTIMIZE_H__
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*/
#define likely(expr) __builtin_expect(!!(expr), 1)
#define unlikely(expr) __builtin_expect(!!(expr), 0)
#endif /* __UTIL_OPTIMIZE_H__ */

@ -133,7 +133,7 @@ cuda:
# compiled with --enable-cuda: b2g_cuda. Make sure to update your
# max-pending-packets setting above as well if you use b2g_cuda.
mpm-algo: b2gc
mpm-algo: b2g
# The memory settings for hash size of these algorithms can vary from lowest
# (2048) - low (4096) - medium (8192) - high (16384) - highest (32768) - max
@ -152,13 +152,19 @@ mpm-algo: b2gc
# settings.
pattern-matcher:
- b2gc:
search_algo: B2gSearchBNDMq
hash_size: low
bf_size: medium
- b2gm:
search_algo: B2gSearchBNDMq
hash_size: low
bf_size: medium
- b2g:
scan_algo: B2gScanBNDMq
search_algo: B2gSearchBNDMq
hash_size: low
bf_size: medium
- b3g:
scan_algo: B3gScanBNDMq
search_algo: B3gSearchBNDMq
hash_size: low
bf_size: medium

Loading…
Cancel
Save