Add implementation of the Simple BNDM 2gram pattern matcher algorithm.

src/Makefile.am

@@ -56,6 +56,7 @@ util-mpm-trie.c util-mpm-trie.h \
 util-mpm.c util-mpm.h \
 util-binsearch.c util-binsearch.h \
 util-mpm-wumanber.c util-mpm-wumanber.h \
+util-mpm-b2g.c util-mpm-b2g.h \
 util-cidr.c util-cidr.h \
 util-unittest.c util-unittest.h \
 util-hash.c util-hash.h \

src/detect-engine-mpm.c

@@ -64,7 +64,8 @@ void PatternMatchDestroy(MpmCtx *mc) {
 /* TODO remove this when we move to the rule groups completely */
 void PatternMatchPrepare(MpmCtx *mc)
 {
-    MpmInitCtx(mc, MPM_WUMANBER);
+    //MpmInitCtx(mc, MPM_WUMANBER);
+    MpmInitCtx(mc, MPM_B2G);
 }
 
 
@@ -167,14 +168,16 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         if (sh->mpm_ctx == NULL)
             goto error;
 
-        MpmInitCtx(sh->mpm_ctx, MPM_WUMANBER);
+        //MpmInitCtx(sh->mpm_ctx, MPM_WUMANBER);
+        MpmInitCtx(sh->mpm_ctx, MPM_B2G);
     }
     if (sh->flags & SIG_GROUP_HAVEURICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) {
         sh->mpm_uri_ctx = malloc(sizeof(MpmCtx));
         if (sh->mpm_uri_ctx == NULL)
             goto error;
 
-        MpmInitCtx(sh->mpm_uri_ctx, MPM_WUMANBER);
+        //MpmInitCtx(sh->mpm_uri_ctx, MPM_WUMANBER);
+        MpmInitCtx(sh->mpm_uri_ctx, MPM_B2G);
     }
 
     u_int16_t mpm_content_scan_maxlen = 65535, mpm_uricontent_scan_maxlen = 65535;

src/detect-uricontent.c

@@ -231,13 +231,7 @@ DoDetectUricontent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, SigMatch
 int DetectUricontentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signature *s, SigMatch *m)
 {
     u_int32_t len = 0;
-/*
-    if (s->id == 2008238) {
-        printf("scanning uricontent have %u\n", pmt->de_have_httpuri);
-        PrintRawUriFp(stdout,p->http_uri.raw[0],p->http_uri.raw_size[0]);
-        printf("\n");
-    }
-*/
+
     /* if we don't have a uri, don't bother scanning */
     if (pmt->de_have_httpuri == 0)
         return 0;
@@ -246,15 +240,10 @@ int DetectUricontentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p,
 
     /* see if we had a match */
     len = pmt->mtcu.match[co->id].len;
-/*
-    if (s->id == 2008238)
-        printf("len %u\n", len);
-*/
     if (len == 0)
         return 0;
 
 #ifdef DEBUG
-    if (s->id == 2008238) {
     printf("uricontent \'");
     PrintRawUriFp(stdout, co->uricontent, co->uricontent_len);    
     printf("\' matched %u time(s) at offsets: ", len);
@@ -264,7 +253,6 @@ int DetectUricontentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p,
         printf("%u ", tmpm->offset);
     }
     printf("\n");
-    }
 #endif
 
     return DoDetectUricontent(t, pmt, p, m, co);

src/detect.c

@@ -111,7 +111,7 @@ void DetectExitPrintStats(ThreadVars *tv, void *data) {
         (float)(pmt->pkts_uri_searched1/(float)(pmt->uris)*100),
         (float)(pmt->pkts_uri_searched1/(float)(pmt->pkts_uri_scanned1)*100));
     printf(" - (%s) URI (2byte) Uri's %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
-        pmt->pkts, pmt->pkts_uri_scanned2,
+        pmt->uris, pmt->pkts_uri_scanned2,
         (float)(pmt->pkts_uri_scanned2/(float)(pmt->uris)*100),
         pmt->pkts_uri_searched2,
         (float)(pmt->pkts_uri_searched2/(float)(pmt->uris)*100),
@@ -852,7 +852,7 @@ static DetectAddressGroup *GetHeadPtr(DetectAddressGroupsHead *head, int family)
     return grhead;
 }
 
-#define MAX_UNIQ_GROUPS 8
+#define MAX_UNIQ_GROUPS 3
 
 /* set unique_groups to 0 for no grouping.
  *

src/util-bloomfilter.c

@@ -32,10 +32,10 @@ BloomFilter *BloomFilterInit(u_int32_t size, u_int8_t iter, u_int32_t (*Hash)(vo
     bf->Hash = Hash;
 
     /* setup the bitarray */
-    bf->bitarray = malloc(bf->bitarray_size/8);
+    bf->bitarray = malloc((bf->bitarray_size/8)+1);
     if (bf->bitarray == NULL)
         goto error;
-    memset(bf->bitarray,0,bf->bitarray_size/8);
+    memset(bf->bitarray,0,(bf->bitarray_size/8)+1);
 
     return bf;
 

src/util-mpm-b2g.h

@@ -0,0 +1,103 @@
+#ifndef __UTIL_MPM_B2G_H__
+#define __UTIL_MPM_B2G_H__
+
+#include "util-mpm.h"
+#include "util-bloomfilter.h"
+
+#define B2G_NOCASE 0x01
+#define B2G_SCAN   0x02
+
+//#define B2G_HASHSIZE 65536
+#define B2G_HASHSIZE 16384
+//#define B2G_HASHSHIFT 8
+#define B2G_HASHSHIFT 6
+#define B2G_TYPE u_int32_t
+//#define B2G_TYPE u_int16_t
+//#define B2G_TYPE u_int8_t
+//#define B2G_WORD_SIZE 16
+//#define B2G_WORD_SIZE 8
+#define B2G_WORD_SIZE 32
+static int B2G_S0 = 1;
+
+#define B2G_BLOOMSIZE 512
+
+#define B2G_HASH16(a,b) (((a)<<B2G_HASHSHIFT) | (b))
+#define B2G_Q           2
+
+//#define B2G_COUNTERS
+
+typedef struct _B2gPattern {
+    u_int8_t *cs; /* case sensitive */
+    u_int8_t *ci; /* case INsensitive */
+    u_int16_t len;
+    struct _B2gPattern *next;
+    u_int16_t prefix_ci;
+    u_int16_t prefix_cs;
+    u_int8_t flags;
+    MpmEndMatch *em;
+} B2gPattern;
+
+typedef struct _B2gHashItem_ {
+    u_int8_t flags;
+    u_int16_t idx;
+    struct _B2gHashItem_ *nxt;
+    u_int8_t p_min_len;
+} B2gHashItem;
+
+typedef struct _B2gCtx {
+    /* hash used during ctx initialization */
+    B2gPattern **init_hash;
+
+    B2G_TYPE scan_m;
+    B2G_TYPE search_m;
+    B2G_TYPE *scan_B2G;
+    B2G_TYPE *search_B2G;
+
+    u_int16_t scan_shiftlen;
+    u_int16_t search_shiftlen;
+
+    u_int32_t scan_hash_size;
+    B2gHashItem **scan_hash;
+    BloomFilter **scan_bloom;
+    B2gHashItem scan_hash1[256];
+    u_int32_t search_hash_size;
+    B2gHashItem **search_hash;
+    B2gHashItem search_hash1[256];
+
+    /* we store our own multi byte scan ptr here for B2gSearch1 */
+    u_int32_t (*MBScan)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
+    /* we store our own multi byte search ptr here for B2gSearch1 */
+    u_int32_t (*MBSearch)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
+
+    /* pattern arrays */
+    B2gPattern **parray;
+} B2gCtx;
+
+typedef struct _B2gThreadCtx {
+#ifdef B2G_COUNTERS
+    u_int32_t scan_stat_pminlen_calls;
+    u_int32_t scan_stat_pminlen_total;
+    u_int32_t scan_stat_bloom_calls;
+    u_int32_t scan_stat_bloom_hits;
+    u_int32_t scan_stat_calls;
+    u_int32_t scan_stat_m_total;
+    u_int32_t scan_stat_d0;
+    u_int32_t scan_stat_d0_hashloop;
+    u_int32_t scan_stat_loop_match;
+    u_int32_t scan_stat_loop_no_match;
+    u_int32_t scan_stat_num_shift;
+    u_int32_t scan_stat_total_shift;
+
+    u_int32_t search_stat_d0;
+    u_int32_t search_stat_loop_match;
+    u_int32_t search_stat_loop_no_match;
+    u_int32_t search_stat_num_shift;
+    u_int32_t search_stat_total_shift;
+#endif /* B2G_COUNTERS */
+} B2gThreadCtx;
+
+void MpmB2gRegister(void);
+
+
+#endif
+

src/util-mpm.c

@@ -10,6 +10,7 @@
 /* include pattern matchers */
 #include "util-mpm-trie.h"
 #include "util-mpm-wumanber.h"
+#include "util-mpm-b2g.h"
 
 /* cleanup list with all matches
  *
@@ -221,6 +222,7 @@ void MpmTableSetup(void) {
 
     //MpmTrieRegister();
     MpmWuManberRegister();
+    MpmB2gRegister();
 }
 
 void MpmRegisterTests(void) {

src/util-mpm.h

@@ -10,6 +10,7 @@
 enum {
     MPM_TRIE,
     MPM_WUMANBER,
+    MPM_B2G,
 
     /* table size */
     MPM_TABLE_SIZE,