Add b3g 3gram BNDM pattern matcher. Fix multi queue nfq initialization. Improve speed of b2g and wumanber.

src/Makefile.am

@@ -57,6 +57,7 @@ util-mpm.c util-mpm.h \
 util-binsearch.c util-binsearch.h \
 util-mpm-wumanber.c util-mpm-wumanber.h \
 util-mpm-b2g.c util-mpm-b2g.h \
+util-mpm-b3g.c util-mpm-b3g.h \
 util-cidr.c util-cidr.h \
 util-unittest.c util-unittest.h \
 util-hash.c util-hash.h \

src/detect-engine-mpm.c

@@ -18,6 +18,10 @@
 #include "detect-content.h"
 #include "detect-uricontent.h"
 
+//#define PM   MPM_WUMANBER
+#define PM   MPM_B2G
+//#define PM   MPM_B3G
+
 u_int32_t PacketPatternScan(ThreadVars *t, PatternMatcherThread *pmt, Packet *p) {
     u_int32_t ret;
 
@@ -64,8 +68,7 @@ void PatternMatchDestroy(MpmCtx *mc) {
 /* TODO remove this when we move to the rule groups completely */
 void PatternMatchPrepare(MpmCtx *mc)
 {
-    //MpmInitCtx(mc, MPM_WUMANBER);
+    MpmInitCtx(mc, PM);
-    MpmInitCtx(mc, MPM_B2G);
 }
 
 
@@ -168,16 +171,14 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         if (sh->mpm_ctx == NULL)
             goto error;
 
-        //MpmInitCtx(sh->mpm_ctx, MPM_WUMANBER);
+        MpmInitCtx(sh->mpm_ctx, PM);
-        MpmInitCtx(sh->mpm_ctx, MPM_B2G);
     }
     if (sh->flags & SIG_GROUP_HAVEURICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) {
         sh->mpm_uri_ctx = malloc(sizeof(MpmCtx));
         if (sh->mpm_uri_ctx == NULL)
             goto error;
 
-        //MpmInitCtx(sh->mpm_uri_ctx, MPM_WUMANBER);
+        MpmInitCtx(sh->mpm_uri_ctx, PM);
-        MpmInitCtx(sh->mpm_uri_ctx, MPM_B2G);
     }
 
     u_int16_t mpm_content_scan_maxlen = 65535, mpm_uricontent_scan_maxlen = 65535;

src/detect.c

@@ -271,8 +271,8 @@ void SigLoadSignatures (void)
 #define LOADSIGS
 #ifdef LOADSIGS
     int good = 0, bad = 0;
-    //FILE *fp = fopen("/etc/vips/rules/bleeding-all.rules", "r");
+    FILE *fp = fopen("/etc/vips/rules/bleeding-all.rules", "r");
-    FILE *fp = fopen("/home/victor/rules/all.rules", "r");
+    //FILE *fp = fopen("/home/victor/rules/all.rules", "r");
     //FILE *fp = fopen("/home/victor/rules/vips-http.sigs", "r");
     //FILE *fp = fopen("/home/victor/rules/emerging-dshield.rules", "r");
     //FILE *fp = fopen("/home/victor/rules/emerging-web.rules", "r");
@@ -485,7 +485,6 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p)
 
         //printf("idx %u, pmt->pmq.sig_id_array_cnt %u, s->id %u (MPM? %s)\n", idx, pmt->pmq.sig_id_array_cnt, s->id, s->flags & SIG_FLAG_MPM ? "TRUE":"FALSE");
         //printf("Sig %u\n", s->id);
-
         /* check the source & dst port in the sig */
         if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) {
             if (!(s->flags & SIG_FLAG_DP_ANY)) {
@@ -500,6 +499,7 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p)
                     continue;
             }
         }
+
         /* check the source address */
         if (!(s->flags & SIG_FLAG_SRC_ANY)) {
             DetectAddressGroup *saddr = DetectAddressLookupGroup(&s->src,&p->src);
@@ -3290,7 +3290,6 @@ int SigTest16 (void) {
 
     g_de_ctx->sig_list = SigInit("alert tcp any any -> any !$HTTP_PORTS (msg:\"ET POLICY Inbound HTTP CONNECT Attempt on Off-Port\"; content:\"CONNECT \"; nocase; depth:8; content:\" HTTP/1.\"; nocase; within:1000; classtype:misc-activity; sid:2008284; rev:2;)");
     if (g_de_ctx->sig_list == NULL) {
-        result = 0;
         goto end;
     }
 
@@ -3302,7 +3301,7 @@ int SigTest16 (void) {
     if (PacketAlertCheck(&p, 2008284))
         result = 1;
     else
-        result = 0;
+        printf("sid:2008284 %s: ", PacketAlertCheck(&p, 2008284) ? "OK" : "FAIL");
 
     SigGroupCleanup();
     SigCleanSignatures();

src/source-nfq.c

@@ -26,6 +26,8 @@ static NFQGlobalVars nfq_g;
 static NFQThreadVars nfq_t[NFQ_MAX_QUEUE];
 static u_int16_t receive_queue_num = 0;
 static u_int16_t verdict_queue_num = 0;
+static pthread_mutex_t nfq_init_lock;
+
 
 int ReceiveNFQ(ThreadVars *, Packet *, void *, PacketQueue *);
 int ReceiveNFQThreadInit(ThreadVars *, void **);
@@ -42,6 +44,7 @@ void TmModuleReceiveNFQRegister (void) {
     /* XXX create a general NFQ setup function */
     memset(&nfq_g, 0, sizeof(nfq_g));
     memset(&nfq_t, 0, sizeof(nfq_t));
+    pthread_mutex_init(&nfq_init_lock, NULL);
 
     tmm_modules[TMM_RECEIVENFQ].name = "ReceiveNFQ";
     tmm_modules[TMM_RECEIVENFQ].Init = ReceiveNFQThreadInit;
@@ -177,11 +180,11 @@ int NFQInitThread(NFQThreadVars *nfq_t, u_int16_t queue_num, u_int32_t queue_max
         }
     }
 
-    printf("NFQInitThread: binding this socket to queue '%u'\n", queue_num);
+    printf("NFQInitThread: binding this socket to queue '%u'\n", nfq_t->queue_num);
 
     /* pass the thread memory as a void ptr so the
      * callback function has access to it. */
-    nfq_t->qh = nfq_create_queue(nfq_t->h, queue_num, &cb, (void *)nfq_t);
+    nfq_t->qh = nfq_create_queue(nfq_t->h, nfq_t->queue_num, &cb, (void *)nfq_t);
     if (nfq_t->qh == NULL)
     {
         printf("error during nfq_create_queue()\n");
@@ -229,7 +232,8 @@ int NFQInitThread(NFQThreadVars *nfq_t, u_int16_t queue_num, u_int32_t queue_max
 }
 
 int ReceiveNFQThreadInit(ThreadVars *tv, void **data) {
-    //printf("ReceiveNFQThreadInit: starting... will bind to queuenum %u\n", receive_queue_num);
+    mutex_lock(&nfq_init_lock);
+    printf("ReceiveNFQThreadInit: starting... will bind to queuenum %u\n", receive_queue_num);
 
     NFQThreadVars *ntv = &nfq_t[receive_queue_num];
 
@@ -241,15 +245,18 @@ int ReceiveNFQThreadInit(ThreadVars *tv, void **data) {
     if (r < 0) {
         printf("NFQInitThread failed\n");
         //return -1;
+        mutex_unlock(&nfq_init_lock);
         exit(1);
     }
 
     *data = (void *)ntv;
     receive_queue_num++;
+    mutex_unlock(&nfq_init_lock);
     return 0;
 }
 
 int VerdictNFQThreadInit(ThreadVars *tv, void **data) {
+    mutex_lock(&nfq_init_lock);
     printf("VerdictNFQThreadInit: starting... will bind to queuenum %u\n", verdict_queue_num);
 
     /* no initialization, ReceiveNFQ takes care of that */
@@ -259,6 +266,7 @@ int VerdictNFQThreadInit(ThreadVars *tv, void **data) {
     verdict_queue_num++;
 
     printf("VerdictNFQThreadInit: ntv %p\n", ntv);
+    mutex_unlock(&nfq_init_lock);
     return 0;
 }
 
@@ -307,6 +315,8 @@ void NFQRecvPkt(NFQThreadVars *t) {
 int ReceiveNFQ(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq) {
     NFQThreadVars *ntv = (NFQThreadVars *)data;
 
+    //printf("%p receiving on queue %u\n", ntv, ntv->queue_num);
+
     /* XXX can we move this to initialization? */
     sigset_t sigs;
     sigfillset(&sigs);
@@ -343,6 +353,8 @@ void NFQSetVerdict(NFQThreadVars *t, Packet *p) {
     int ret;
     u_int32_t verdict;
 
+    //printf("%p verdicting on queue %u\n", t, t->queue_num);
+
     if (p->action == ACTION_ALERT) {
        verdict = NF_ACCEPT;
     } else if (p->action == ACTION_PASS) {

src/threadvars.h

@@ -29,6 +29,8 @@ typedef struct _ThreadVars {
     void *(*tm_func)(void *);
     void *tm_slots;
 
+    char set_cpu_affinity; /* bool: 0 no, 1 yes */
+    int cpu_affinity; /* cpu or core to set affinity to */
 //#ifdef NFQ
 //    NFQThreadVars *nfq_t;
 //#endif

src/tm-threads.c

@@ -1,11 +1,21 @@
 /* Copyright (c) 2008 Victor Julien <victor@inliniac.net> */
 
+#include <sys/types.h> /* for gettid(2) */
+#define _GNU_SOURCE
+#define __USE_GNU
+#include <sys/syscall.h>
+#include <sched.h>     /* for sched_setaffinity(2) */
+
 #include "vips.h"
 #include "threadvars.h"
 #include "tm-queues.h"
 #include "tm-queuehandlers.h"
 #include "tm-modules.h"
 
+/* prototypes */
+static int SetCPUAffinity(int cpu);
+
+
 /* root of the threadvars list */
 static ThreadVars *tv_root;
 
@@ -53,6 +63,9 @@ void *TmThreadsSlot1NoIn(void *td) {
     char run = 1;
     int r = 0;
 
+    if (tv->set_cpu_affinity == 1)
+        SetCPUAffinity(tv->cpu_affinity);
+
     if (s->s.SlotInit != NULL) {
         r = s->s.SlotInit(tv, &s->s.slot_data);
         if (r != 0) {
@@ -97,6 +110,9 @@ void *TmThreadsSlot1NoOut(void *td) {
     char run = 1;
     int r = 0;
 
+    if (tv->set_cpu_affinity == 1)
+        SetCPUAffinity(tv->cpu_affinity);
+
     if (s->s.SlotInit != NULL) {
         r = s->s.SlotInit(tv, &s->s.slot_data);
         if (r != 0) {
@@ -135,6 +151,9 @@ void *TmThreadsSlot1NoInOut(void *td) {
     char run = 1;
     int r = 0;
 
+    if (tv->set_cpu_affinity == 1)
+        SetCPUAffinity(tv->cpu_affinity);
+
     //printf("TmThreadsSlot1NoInOut: %s starting\n", tv->name);
 
     if (s->s.SlotInit != NULL) {
@@ -178,6 +197,9 @@ void *TmThreadsSlot1(void *td) {
     char run = 1;
     int r = 0;
 
+    if (tv->set_cpu_affinity == 1)
+        SetCPUAffinity(tv->cpu_affinity);
+
     //printf("TmThreadsSlot1: %s starting\n", tv->name);
 
     if (s->s.SlotInit != NULL) {
@@ -237,6 +259,9 @@ void *TmThreadsSlot2(void *td) {
     char run = 1;
     int r = 0;
 
+    if (tv->set_cpu_affinity == 1)
+        SetCPUAffinity(tv->cpu_affinity);
+
     //printf("TmThreadsSlot2: %s starting\n", tv->name);
 
     if (s->s1.SlotInit != NULL) {
@@ -325,6 +350,9 @@ void *TmThreadsSlot3(void *td) {
     char run = 1;
     int r = 0;
 
+    if (tv->set_cpu_affinity == 1)
+        SetCPUAffinity(tv->cpu_affinity);
+
     //printf("TmThreadsSlot3: %s starting\n", tv->name);
 
     if (s->s1.SlotInit != NULL) {
@@ -481,6 +509,9 @@ void *TmThreadsSlotVar(void *td) {
     int r = 0;
     TmSlot *slot = NULL;
 
+    if (tv->set_cpu_affinity == 1)
+        SetCPUAffinity(tv->cpu_affinity);
+
     //printf("TmThreadsSlot1: %s starting\n", tv->name);
 
     for (slot = s->s; slot != NULL; slot = slot->slot_next) {
@@ -672,6 +703,31 @@ void TmVarSlotSetFuncAppend(ThreadVars *tv, TmModule *tm) {
     }
 }
 
+/* called from the thread */
+static int SetCPUAffinity(int cpu) {
+    //pthread_t tid = pthread_self();
+    pid_t tid = syscall(SYS_gettid);
+    cpu_set_t cs;
+
+    printf("Setting CPU Affinity for thread %u to CPU %d\n", tid, cpu);
+
+    CPU_ZERO(&cs);
+    CPU_SET(cpu,&cs);
+
+    int r = sched_setaffinity(tid,sizeof(cpu_set_t),&cs); 
+    if (r != 0) {
+        printf("Warning: sched_setaffinity failed (%d): %s\n", r, strerror(errno));
+    }
+
+    return 0;
+}
+
+int TmThreadSetCPUAffinity(ThreadVars *tv, int cpu) {
+    tv->set_cpu_affinity = 1;
+    tv->cpu_affinity = cpu;
+    return 0;
+}
+
 ThreadVars *TmThreadCreate(char *name, char *inq_name, char *inqh_name, char *outq_name, char *outqh_name, char *slots) {
     ThreadVars *tv = NULL;
     Tmq *tmq = NULL;

src/tmqh-packetpool.c

@@ -42,22 +42,22 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
     char proot = 0;
 
     if (IS_TUNNEL_PKT(p)) {
-        printf("TmqhOutputPacketpool: tunnel packet: %p %s\n", p,p->root ? "upper layer":"root");
+        //printf("TmqhOutputPacketpool: tunnel packet: %p %s\n", p,p->root ? "upper layer":"root");
 
         /* get a lock */
         pthread_mutex_t *m = p->root ? &p->root->mutex_rtv_cnt : &p->mutex_rtv_cnt;
         mutex_lock(m);
 
         if (IS_TUNNEL_ROOT_PKT(p)) {
-            printf("TmqhOutputPacketpool: IS_TUNNEL_ROOT_PKT\n");
+            //printf("TmqhOutputPacketpool: IS_TUNNEL_ROOT_PKT\n");
             if (TUNNEL_PKT_TPR(p) == 0) {
-                printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) == 0\n");
+                //printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) == 0\n");
                 /* if this packet is the root and there are no
                  * more tunnel packets, enqueue it */
 
                 /* fall through */
             } else {
-                printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) > 0\n");
+                //printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) > 0\n");
                 /* if this is the root and there are more tunnel
                  * packets, don't add this. It's still referenced
                  * by the tunnel packets, and we will enqueue it
@@ -67,7 +67,7 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
                 return;
             }
         } else {
-            printf("TmqhOutputPacketpool: NOT IS_TUNNEL_ROOT_PKT\n");
+            //printf("TmqhOutputPacketpool: NOT IS_TUNNEL_ROOT_PKT\n");
             if (p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1) {
                 //printf("TmqhOutputPacketpool: p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1\n");
                 /* the root is ready and we are the last tunnel packet,
@@ -75,19 +75,19 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
                 TUNNEL_DECR_PKT_TPR_NOLOCK(p);
 
                 /* handle the root */
-                printf("TmqhOutputPacketpool: calling PacketEnqueue for root pkt\n");
+                //printf("TmqhOutputPacketpool: calling PacketEnqueue for root pkt\n");
                 proot = 1;
 
                 /* fall through */
             } else {
-                printf("TmqhOutputPacketpool: NOT p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1 (%u)\n", TUNNEL_PKT_TPR(p));
+                //printf("TmqhOutputPacketpool: NOT p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1 (%u)\n", TUNNEL_PKT_TPR(p));
                 TUNNEL_DECR_PKT_TPR_NOLOCK(p);
 
                  /* fall through */
             }
         }
         mutex_unlock(m);
-        printf("TmqhOutputPacketpool: tunnel stuff done, move on\n");
+        //printf("TmqhOutputPacketpool: tunnel stuff done, move on\n");
     }
 
     mutex_lock(&q->mutex_q);

src/util-bloomfilter.c

@@ -61,7 +61,7 @@ void BloomFilterFree(BloomFilter *bf) {
 void BloomFilterPrint(BloomFilter *bf) {
     printf("\n---------- Bloom Filter Stats -----------\n");
     printf("Buckets:               %u\n", bf->bitarray_size);
-    printf("Memory size:           %u bytes\n", bf->bitarray_size/8);
+    printf("Memory size:           %u bytes\n", bf->bitarray_size/8 + 1);
     printf("Hash function pointer: %p\n", bf->Hash);
     printf("Hash functions:        %u\n", bf->hash_iterations);
     printf("-----------------------------------------\n");
@@ -98,6 +98,13 @@ int BloomFilterTest(BloomFilter *bf, void *data, u_int16_t datalen) {
     return hit;
 }
 
+u_int32_t BloomFilterMemorySize(BloomFilter *bf) {
+     if (bf == NULL)
+         return 0;
+
+     return (sizeof(BloomFilter) + (bf->bitarray_size/8) + 1);
+}
+
 static u_int32_t BloomHash(void *data, u_int16_t datalen, u_int8_t iter, u_int32_t hash_size) {
      u_int8_t *d = (u_int8_t *)data;
      u_int32_t i;

src/util-bloomfilter.h

@@ -17,6 +17,7 @@ void BloomFilterFree(BloomFilter *);
 void BloomFilterPrint(BloomFilter *);
 int BloomFilterAdd(BloomFilter *, void *, u_int16_t);
 int BloomFilterTest(BloomFilter *, void *, u_int16_t);
+u_int32_t BloomFilterMemorySize(BloomFilter *);
 
 void BloomFilterRegisterTests(void);
 

src/util-mpm-b2g.h

@@ -8,22 +8,34 @@
 #define B2G_SCAN   0x02
 
 //#define B2G_HASHSIZE 65536
+//#define B2G_HASHSIZE 32768
 #define B2G_HASHSIZE 16384
+//#define B2G_HASHSIZE 8192
+//#define B2G_HASHSIZE 4096
+//#define B2G_HASHSIZE 2048
+
 //#define B2G_HASHSHIFT 8
+//#define B2G_HASHSHIFT 7
 #define B2G_HASHSHIFT 6
+//#define B2G_HASHSHIFT 5
+//#define B2G_HASHSHIFT 4
+//#define B2G_HASHSHIFT 3
+
 #define B2G_TYPE u_int32_t
 //#define B2G_TYPE u_int16_t
 //#define B2G_TYPE u_int8_t
 //#define B2G_WORD_SIZE 16
 //#define B2G_WORD_SIZE 8
 #define B2G_WORD_SIZE 32
-static int B2G_S0 = 1;
 
-#define B2G_BLOOMSIZE 512
+#define B2G_BLOOMSIZE 1024
 
 #define B2G_HASH16(a,b) (((a)<<B2G_HASHSHIFT) | (b))
 #define B2G_Q           2
 
+//#define B2G_SCANFUNC B2gScanBNDMq
+#define B2G_SCANFUNC B2gScan
+
 //#define B2G_COUNTERS
 
 typedef struct _B2gPattern {
@@ -31,8 +43,6 @@ typedef struct _B2gPattern {
     u_int8_t *ci; /* case INsensitive */
     u_int16_t len;
     struct _B2gPattern *next;
-    u_int16_t prefix_ci;
-    u_int16_t prefix_cs;
     u_int8_t flags;
     MpmEndMatch *em;
 } B2gPattern;
@@ -41,7 +51,6 @@ typedef struct _B2gHashItem_ {
     u_int8_t flags;
     u_int16_t idx;
     struct _B2gHashItem_ *nxt;
-    u_int8_t p_min_len;
 } B2gHashItem;
 
 typedef struct _B2gCtx {
@@ -53,18 +62,35 @@ typedef struct _B2gCtx {
     B2G_TYPE *scan_B2G;
     B2G_TYPE *search_B2G;
 
-    u_int16_t scan_shiftlen;
+    u_int8_t scan_s0;
-    u_int16_t search_shiftlen;
+    u_int8_t search_s0;
+
+    u_int16_t scan_1_pat_cnt;
+#ifdef B2G_SCAN2
+    u_int16_t scan_2_pat_cnt;
+#endif
+    u_int16_t scan_x_pat_cnt;
 
     u_int32_t scan_hash_size;
     B2gHashItem **scan_hash;
     BloomFilter **scan_bloom;
+    u_int8_t *scan_pminlen; /* array containing the minimal length
+                               of the patters in a hash bucket. Used
+                               for the BloomFilter. */
     B2gHashItem scan_hash1[256];
+#ifdef B2G_SCAN2
+    B2gHashItem scan_hash2[65536];
+#endif
     u_int32_t search_hash_size;
+    BloomFilter **search_bloom;
+    u_int8_t *search_pminlen; /* array containing the minimal length
+                                 of the patters in a hash bucket. Used
+                                 for the BloomFilter. */
     B2gHashItem **search_hash;
     B2gHashItem search_hash1[256];
 
     /* we store our own multi byte scan ptr here for B2gSearch1 */
+    u_int32_t (*MBScan2)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
     u_int32_t (*MBScan)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
     /* we store our own multi byte search ptr here for B2gSearch1 */
     u_int32_t (*MBSearch)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);

src/util-mpm-b3g.h

@@ -0,0 +1,125 @@
+#ifndef __UTIL_MPM_B3G_H__
+#define __UTIL_MPM_B3G_H__
+
+#include "util-mpm.h"
+#include "util-bloomfilter.h"
+
+#define B3G_NOCASE 0x01
+#define B3G_SCAN   0x02
+
+//#define B3G_HASHSIZE 65536
+#define B3G_HASHSIZE 32768
+//#define B3G_HASHSIZE 16384
+//#define B3G_HASHSIZE 8192
+//#define B3G_HASHSIZE 4096
+
+//#define B3G_HASHSHIFT 8
+#define B3G_HASHSHIFT 7
+//#define B3G_HASHSHIFT 6
+//#define B3G_HASHSHIFT 5
+//#define B3G_HASHSHIFT 4
+
+#define B3G_TYPE u_int32_t
+//#define B3G_TYPE u_int16_t
+//#define B3G_TYPE u_int8_t
+//#define B3G_WORD_SIZE 16
+//#define B3G_WORD_SIZE 8
+#define B3G_WORD_SIZE     32
+
+#define B3G_BLOOMSIZE     1024
+
+#define B3G_HASH(a,b,c)   (((a)<<B3G_HASHSHIFT) | (b)<<(B3G_HASHSHIFT-3) |(c))
+#define B3G_Q             3
+
+//#define B3G_SCANFUNC      B3gScan
+#define B3G_SCANFUNC      B3gScanBNDMq
+
+//#define B3G_COUNTERS
+
+typedef struct _B3gPattern {
+    u_int8_t *cs; /* case sensitive */
+    u_int8_t *ci; /* case INsensitive */
+    u_int16_t len;
+    struct _B3gPattern *next;
+    u_int8_t flags;
+    MpmEndMatch *em;
+} B3gPattern;
+
+typedef struct _B3gHashItem_ {
+    u_int8_t flags;
+    u_int16_t idx;
+    struct _B3gHashItem_ *nxt;
+} B3gHashItem;
+
+typedef struct _B3gCtx {
+    /* hash used during ctx initialization */
+    B3gPattern **init_hash;
+
+    B3G_TYPE scan_m;
+    B3G_TYPE search_m;
+    B3G_TYPE *scan_B3G;
+    B3G_TYPE *search_B3G;
+
+    u_int8_t scan_s0;
+    u_int8_t search_s0;
+
+    u_int16_t scan_1_pat_cnt;
+    u_int16_t scan_2_pat_cnt;
+    u_int16_t scan_x_pat_cnt;
+
+    u_int16_t search_1_pat_cnt;
+    u_int16_t search_2_pat_cnt;
+    u_int16_t search_x_pat_cnt;
+
+    u_int32_t scan_hash_size;
+    B3gHashItem **scan_hash;
+    BloomFilter **scan_bloom;
+    u_int8_t *scan_pminlen; /* array containing the minimal length
+                               of the patters in a hash bucket. Used
+                               for the BloomFilter. */
+    B3gHashItem scan_hash1[256];
+    B3gHashItem scan_hash2[65536];
+    u_int32_t search_hash_size;
+    B3gHashItem **search_hash;
+    B3gHashItem search_hash1[256];
+    B3gHashItem search_hash2[65536];
+
+    /* we store our own multi byte scan ptr here for B3gSearch1 */
+    u_int32_t (*MBScan2)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
+    u_int32_t (*MBScan)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
+    /* we store our own multi byte search ptr here for B3gSearch1 */
+    u_int32_t (*MBSearch2)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
+    u_int32_t (*MBSearch)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
+
+    /* pattern arrays */
+    B3gPattern **parray;
+} B3gCtx;
+
+typedef struct _B3gThreadCtx {
+#ifdef B3G_COUNTERS
+    u_int32_t scan_stat_pminlen_calls;
+    u_int32_t scan_stat_pminlen_total;
+    u_int32_t scan_stat_bloom_calls;
+    u_int32_t scan_stat_bloom_hits;
+    u_int32_t scan_stat_calls;
+    u_int32_t scan_stat_m_total;
+    u_int32_t scan_stat_d0;
+    u_int32_t scan_stat_d0_hashloop;
+    u_int32_t scan_stat_loop_match;
+    u_int32_t scan_stat_loop_no_match;
+    u_int32_t scan_stat_num_shift;
+    u_int32_t scan_stat_total_shift;
+
+    u_int32_t search_stat_d0;
+    u_int32_t search_stat_loop_match;
+    u_int32_t search_stat_loop_no_match;
+    u_int32_t search_stat_num_shift;
+    u_int32_t search_stat_total_shift;
+#endif /* B3G_COUNTERS */
+} B3gThreadCtx;
+
+void MpmB3gRegister(void);
+
+
+#endif
+

src/util-mpm-wumanber.h

@@ -4,10 +4,13 @@
 #define __UTIL_MPM_WUMANBER_H__
 
 #include "util-mpm.h"
+#include "util-bloomfilter.h"
 
 #define WUMANBER_NOCASE 0x01
 #define WUMANBER_SCAN   0x02
 
+#define WUMANBER_BLOOMSIZE 1024
+
 //#define WUMANBER_COUNTERS
 
 typedef struct _WmPattern {
@@ -36,6 +39,10 @@ typedef struct _WmCtx {
 
     u_int32_t scan_hash_size;
     WmHashItem **scan_hash;
+    BloomFilter **scan_bloom;
+    u_int8_t *scan_pminlen; /* array containing the minimal length
+                               of the patters in a hash bucket. Used
+                               for the BloomFilter. */
     WmHashItem scan_hash1[256];
     u_int32_t search_hash_size;
     WmHashItem **search_hash;
@@ -56,6 +63,10 @@ typedef struct _WmCtx {
 
 typedef struct _WmThreadCtx {
 #ifdef WUMANBER_COUNTERS
+    u_int32_t scan_stat_pminlen_calls;
+    u_int32_t scan_stat_pminlen_total;
+    u_int32_t scan_stat_bloom_calls;
+    u_int32_t scan_stat_bloom_hits;
     u_int32_t scan_stat_shift_null;
     u_int32_t scan_stat_loop_match;
     u_int32_t scan_stat_loop_no_match;

src/util-mpm.c

@@ -11,6 +11,7 @@
 #include "util-mpm-trie.h"
 #include "util-mpm-wumanber.h"
 #include "util-mpm-b2g.h"
+#include "util-mpm-b3g.h"
 
 /* cleanup list with all matches
  *
@@ -223,6 +224,7 @@ void MpmTableSetup(void) {
     //MpmTrieRegister();
     MpmWuManberRegister();
     MpmB2gRegister();
+    MpmB3gRegister();
 }
 
 void MpmRegisterTests(void) {

src/util-mpm.h

@@ -11,6 +11,7 @@ enum {
     MPM_TRIE,
     MPM_WUMANBER,
     MPM_B2G,
+    MPM_B3G,
 
     /* table size */
     MPM_TABLE_SIZE,

src/vips.c

@@ -208,11 +208,11 @@ int main(int argc, char **argv)
     UtRunSelftest(); /* inits and cleans up again */
     UtInitialize();
     TmModuleRegisterTests();
-    MpmRegisterTests();
     SigTableRegisterTests();
     HashTableRegisterTests();
     BloomFilterRegisterTests();
     BloomFilterCountingRegisterTests();
+    MpmRegisterTests();
     SigRegisterTests();
     UtRunTests();
     UtCleanup();
@@ -252,6 +252,9 @@ int main(int argc, char **argv)
 
     SigLoadSignatures();
 
+//#define PERF_THREADS
+#define MANY_THREADS
+#ifndef PERF_THREADS
     /* create the threads */
     ThreadVars *tv_receivenfq = TmThreadCreate("ReceiveNFQ","packetpool","packetpool","pickup-queue","simple","1slot_noinout");
     if (tv_receivenfq == NULL) {
@@ -269,7 +272,7 @@ int main(int argc, char **argv)
         printf("ERROR: TmThreadSpawn failed\n");
         exit(1);
     }
-#define MANY_THREADS
+
 #ifdef MANY_THREADS
     ThreadVars *tv_decode1 = TmThreadCreate("Decode1","pickup-queue","simple","decode-queue1","simple","1slot");
     if (tv_decode1 == NULL) {
@@ -440,6 +443,7 @@ int main(int argc, char **argv)
         printf("ERROR: TmThreadSpawn failed\n");
         exit(1);
     }
+
 #else /* MANY_THREADS */
     ThreadVars *tv_main = TmThreadCreate("MainThread","pickup-queue","simple","packetpool","packetpool","varslot");
     if (tv_main == NULL) {
@@ -466,7 +470,7 @@ int main(int argc, char **argv)
         exit(1);
     }
     TmVarSlotSetFuncAppend(tv_main,tm_module);
-
+/*
     tm_module = TmModuleGetByName("RespondReject");
     if (tm_module == NULL) {
         printf("ERROR: TmModuleGetByName for RespondReject failed\n");
@@ -508,13 +512,364 @@ int main(int argc, char **argv)
         exit(1);
     }
     TmVarSlotSetFuncAppend(tv_main,tm_module);
-
+*/
     if (TmThreadSpawn(tv_main) != 0) {
         printf("ERROR: TmThreadSpawn failed\n");
         exit(1);
     }
 
 #endif /* MANY_THREADS */
+#else /* PERF_THREADS */
+#if 0
+    /* create the threads */
+    ThreadVars *tv_receivenfq = TmThreadCreate("ReceiveNFQ1","packetpool","packetpool","pickup-queue","simple","1slot_noinout");
+    if (tv_receivenfq == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    TmModule *tm_module = TmModuleGetByName("ReceiveNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName failed for ReceiveNFQ\n");
+        exit(1);
+    }
+    Tm1SlotSetFunc(tv_receivenfq,tm_module);
+
+    if (TmThreadSpawn(tv_receivenfq) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+    ThreadVars *tv_decode1 = TmThreadCreate("Decode1","pickup-queue","simple","verdict-queue","simple","2slot");
+    if (tv_decode1 == NULL) {
+        printf("ERROR: TmThreadsCreate failed for Decode1\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("DecodeNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc1(tv_decode1,tm_module);
+
+    tm_module = TmModuleGetByName("Detect");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc2(tv_decode1,tm_module);
+
+    TmThreadSetCPUAffinity(tv_decode1,0);
+
+    if (TmThreadSpawn(tv_decode1) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+    ThreadVars *tv_decode2 = TmThreadCreate("Decode2","pickup-queue","simple","verdict-queue","simple","2slot");
+    if (tv_decode2 == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("DecodeNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc1(tv_decode2,tm_module);
+
+    tm_module = TmModuleGetByName("Detect");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc2(tv_decode2,tm_module);
+
+    TmThreadSetCPUAffinity(tv_decode2,1);
+
+    if (TmThreadSpawn(tv_decode2) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+/*
+    ThreadVars *tv_decode3 = TmThreadCreate("Decode3","pickup-queue","simple","verdict-queue","simple","2slot");
+    if (tv_decode3 == NULL) {
+        printf("ERROR: TmThreadsCreate failed for Decode1\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("DecodeNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc1(tv_decode3,tm_module);
+
+    tm_module = TmModuleGetByName("Detect");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc2(tv_decode3,tm_module);
+
+    TmThreadSetCPUAffinity(tv_decode3,0);
+
+    if (TmThreadSpawn(tv_decode3) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+    ThreadVars *tv_decode4 = TmThreadCreate("Decode4","pickup-queue","simple","verdict-queue","simple","2slot");
+    if (tv_decode4 == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("DecodeNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc1(tv_decode4,tm_module);
+
+    tm_module = TmModuleGetByName("Detect");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc2(tv_decode4,tm_module);
+
+    TmThreadSetCPUAffinity(tv_decode4,1);
+
+    if (TmThreadSpawn(tv_decode4) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+*/
+    //ThreadVars *tv_verdict = TmThreadCreate("Verdict","verdict-queue","simple","respond-queue","simple","1slot");
+    ThreadVars *tv_verdict = TmThreadCreate("Verdict","verdict-queue","simple","packetpool","packetpool","1slot");
+    if (tv_verdict == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("VerdictNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName VerdictNFQ failed\n");
+        exit(1);
+    }
+    Tm1SlotSetFunc(tv_verdict,tm_module);
+
+    if (TmThreadSpawn(tv_verdict) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+#endif
+
+    /* create the threads */
+    ThreadVars *tv_receivenfq = TmThreadCreate("ReceiveNFQ1","packetpool","packetpool","pickup-queue1","simple","1slot_noinout");
+    if (tv_receivenfq == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    TmModule *tm_module = TmModuleGetByName("ReceiveNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName failed for ReceiveNFQ\n");
+        exit(1);
+    }
+    Tm1SlotSetFunc(tv_receivenfq,tm_module);
+    TmThreadSetCPUAffinity(tv_receivenfq,0);
+
+    if (TmThreadSpawn(tv_receivenfq) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+    /* create the threads */
+    ThreadVars *tv_receivenfq2 = TmThreadCreate("ReceiveNFQ2","packetpool","packetpool","pickup-queue2","simple","1slot_noinout");
+    if (tv_receivenfq2 == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("ReceiveNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName failed for ReceiveNFQ\n");
+        exit(1);
+    }
+    Tm1SlotSetFunc(tv_receivenfq2,tm_module);
+    TmThreadSetCPUAffinity(tv_receivenfq2,1);
+
+    if (TmThreadSpawn(tv_receivenfq2) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+    ThreadVars *tv_decode1 = TmThreadCreate("Decode1","pickup-queue1","simple","packetpool","packetpool","3slot");
+    if (tv_decode1 == NULL) {
+        printf("ERROR: TmThreadsCreate failed for Decode1\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("DecodeNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm3SlotSetFunc1(tv_decode1,tm_module);
+
+    tm_module = TmModuleGetByName("Detect");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm3SlotSetFunc2(tv_decode1,tm_module);
+
+    tm_module = TmModuleGetByName("VerdictNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName VerdictNFQ failed\n");
+        exit(1);
+    }
+    Tm3SlotSetFunc3(tv_decode1,tm_module);
+    TmThreadSetCPUAffinity(tv_decode1,0);
+
+    if (TmThreadSpawn(tv_decode1) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+
+    ThreadVars *tv_decode2 = TmThreadCreate("Decode2","pickup-queue2","simple","packetpool","packetpool","3slot");
+    if (tv_decode2 == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("DecodeNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm3SlotSetFunc1(tv_decode2,tm_module);
+
+    tm_module = TmModuleGetByName("Detect");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    Tm3SlotSetFunc2(tv_decode2,tm_module);
+
+    tm_module = TmModuleGetByName("VerdictNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName VerdictNFQ failed\n");
+        exit(1);
+    }
+    Tm3SlotSetFunc3(tv_decode2,tm_module);
+    TmThreadSetCPUAffinity(tv_decode2,1);
+
+    if (TmThreadSpawn(tv_decode2) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+/*
+    //ThreadVars *tv_verdict = TmThreadCreate("Verdict","verdict-queue","simple","respond-queue","simple","1slot");
+    ThreadVars *tv_verdict = TmThreadCreate("Verdict","verdict-queue","simple","packetpool","packetpool","1slot");
+    if (tv_verdict == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("VerdictNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName VerdictNFQ failed\n");
+        exit(1);
+    }
+    Tm1SlotSetFunc(tv_verdict,tm_module);
+
+    if (TmThreadSpawn(tv_verdict) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+*/
+/*
+    ThreadVars *tv_rreject = TmThreadCreate("RespondReject","respond-queue","simple","alert-queue1","simple","1slot");
+    if (tv_rreject == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("RespondReject");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName for RespondReject failed\n");
+        exit(1);
+    }
+    Tm1SlotSetFunc(tv_rreject,tm_module);
+
+    if (TmThreadSpawn(tv_rreject) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+    ThreadVars *tv_alert = TmThreadCreate("AlertFastlog&Httplog","alert-queue1","simple","alert-queue2","simple","2slot");
+    if (tv_alert == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("AlertFastlog");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc1(tv_alert,tm_module);
+
+    tm_module = TmModuleGetByName("LogHttplog");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc2(tv_alert,tm_module);
+
+    if (TmThreadSpawn(tv_alert) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+    //ThreadVars *tv_unified = TmThreadCreate("AlertUnifiedLog","alert-queue2","simple","packetpool","packetpool","2slot");
+    ThreadVars *tv_unified = TmThreadCreate("AlertUnifiedLog","alert-queue2","simple","alert-queue3","simple","2slot");
+    if (tv_unified == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+
+    tm_module = TmModuleGetByName("AlertUnifiedLog");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc1(tv_unified,tm_module);
+
+    tm_module = TmModuleGetByName("AlertUnifiedAlert");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
+        exit(1);
+    }
+    Tm2SlotSetFunc2(tv_unified,tm_module);
+
+    if (TmThreadSpawn(tv_unified) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+
+    ThreadVars *tv_debugalert = TmThreadCreate("AlertDebuglog","alert-queue3","simple","packetpool","packetpool","1slot");
+    if (tv_debugalert == NULL) {
+        printf("ERROR: TmThreadsCreate failed\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("AlertDebuglog");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName failed\n");
+        exit(1);
+    }
+    Tm1SlotSetFunc(tv_debugalert,tm_module);
+
+    if (TmThreadSpawn(tv_debugalert) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+*/
+#endif /* PERF_THREADS */
 
     ThreadVars tv_flowmgr;
     memset(&tv_flowmgr, 0, sizeof(ThreadVars));

src/vips.h

@@ -14,7 +14,7 @@
 /* maximum number of simultanious threads. */
 #define NUM_THREADS 256
 /* max packets processed simultaniously */
-#define MAX_PENDING 500
+#define MAX_PENDING 512
 
 #define TRUE   1
 #define FALSE  0