big update

src/decode-udp.c

@@ -4,6 +4,8 @@
 #include "decode-udp.h"
 #include "decode-events.h"
 
+#include "flow.h"
+
 static int DecodeUDPPacket(ThreadVars *t, Packet *p, u_int8_t *pkt, u_int16_t len)
 {
     p->udph = (UDPHdr *)pkt;
@@ -44,5 +46,8 @@ void DecodeUDP(ThreadVars *t, Packet *p, u_int8_t *pkt, u_int16_t len)
         UDP_GET_SRC_PORT(p), UDP_GET_DST_PORT(p), UDP_HEADER_LEN, p->tcp_payload_len);
 #endif
 
+    /* Flow is an integral part of us */
+    FlowHandlePacket(t, p);
+
     return;
 }

src/detect-engine-mpm.c

@@ -21,16 +21,18 @@
 u_int32_t PacketPatternScan(ThreadVars *t, PatternMatcherThread *pmt, Packet *p) {
     u_int32_t ret;
 
-    ret = pmt->mc->Search(pmt->mc_scan, &pmt->mtc, p->tcp_payload, p->tcp_payload_len);
+    pmt->pmq.mode = PMQ_MODE_SCAN;
+    ret = pmt->mc->Scan(pmt->mc, &pmt->mtc, &pmt->pmq, p->tcp_payload, p->tcp_payload_len);
 
-    //printf("PacketPatternMatch: ret %u\n", ret);
+    //printf("PacketPatternScan: ret %u\n", ret);
     return ret;
 }
 
 u_int32_t PacketPatternMatch(ThreadVars *t, PatternMatcherThread *pmt, Packet *p) {
     u_int32_t ret;
 
-    ret = pmt->mc->Search(pmt->mc, &pmt->mtc, p->tcp_payload, p->tcp_payload_len);
+    pmt->pmq.mode = PMQ_MODE_SEARCH;
+    ret = pmt->mc->Search(pmt->mc, &pmt->mtc, &pmt->pmq, p->tcp_payload, p->tcp_payload_len);
 
     //printf("PacketPatternMatch: ret %u\n", ret);
     return ret;
@@ -38,6 +40,12 @@ u_int32_t PacketPatternMatch(ThreadVars *t, PatternMatcherThread *pmt, Packet *p
 
 /* cleans up the mpm instance after a match */
 void PacketPatternCleanup(ThreadVars *t, PatternMatcherThread *pmt) {
+    int i;
+    for (i = 0; i < pmt->pmq.sig_id_array_cnt; i++) {
+        pmt->pmq.sig_bitarray[(pmt->pmq.sig_id_array[i] / 8)] &= ~(1<<(pmt->pmq.sig_id_array[i] % 8));
+    }
+    pmt->pmq.sig_id_array_cnt = 0;
+
     /* content */
     if (pmt->mc != NULL && pmt->mc->Cleanup != NULL) {
         pmt->mc->Cleanup(&pmt->mtc);
@@ -137,8 +145,10 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         for (sm = s->match; sm != NULL; sm = sm->next) {
             if (sm->type == DETECT_CONTENT) {
                 co_cnt++;
+                s->flags |= SIG_FLAG_MPM;
             } else if (sm->type == DETECT_URICONTENT) {
                 ur_cnt++;
+                s->flags |= SIG_FLAG_MPM;
             }
         }
     }
@@ -158,13 +168,6 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             goto error;
 
         MpmInitCtx(sh->mpm_ctx, MPM_WUMANBER);
-
-        /* scan */
-        sh->mpm_scan_ctx = malloc(sizeof(MpmCtx));
-        if (sh->mpm_scan_ctx == NULL)
-            goto error;
-
-        MpmInitCtx(sh->mpm_scan_ctx, MPM_WUMANBER);
     }
     if (sh->flags & SIG_GROUP_HAVEURICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) {
         sh->mpm_uri_ctx = malloc(sizeof(MpmCtx));
@@ -172,13 +175,6 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             goto error;
 
         MpmInitCtx(sh->mpm_uri_ctx, MPM_WUMANBER);
-
-        /* scan */
-        sh->mpm_uri_scan_ctx = malloc(sizeof(MpmCtx));
-        if (sh->mpm_uri_scan_ctx == NULL)
-            goto error;
-
-        MpmInitCtx(sh->mpm_uri_scan_ctx, MPM_WUMANBER);
     }
 
     u_int16_t mpm_content_scan_maxlen = 65535, mpm_uricontent_scan_maxlen = 65535;
@@ -318,58 +314,51 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             if (sh->mpm_uricontent_maxlen > uricontent_maxlen)
                 sh->mpm_uricontent_maxlen = uricontent_maxlen;
         }
-//#if 0
-        /* scan ctx */
+
+        char content_scanadded = 0, uricontent_scanadded = 0;
         for (sm = s->match; sm != NULL; sm = sm->next) {
             if (sm->type == DETECT_CONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) {
                 DetectContentData *cd = (DetectContentData *)sm->ctx;
-                if (sh->mpm_content_maxlen >= cd->content_len) {
 
+                u_int16_t offset = s->flags & SIG_FLAG_RECURSIVE ? 0 : cd->offset;
+                u_int16_t depth = s->flags & SIG_FLAG_RECURSIVE ? 0 : cd->depth;
+
+                if (!content_scanadded && content_maxlen == cd->content_len) {
                     if (cd->flags & DETECT_CONTENT_NOCASE) {
-                        sh->mpm_scan_ctx->AddPatternNocase(sh->mpm_scan_ctx, cd->content, cd->content_len, cd->id);
+                        sh->mpm_ctx->AddScanPatternNocase(sh->mpm_ctx, cd->content, cd->content_len, offset, depth, cd->id, s->num);
                     } else {
-                        sh->mpm_scan_ctx->AddPattern(sh->mpm_scan_ctx, cd->content, cd->content_len, cd->id);
+                        sh->mpm_ctx->AddScanPattern(sh->mpm_ctx, cd->content, cd->content_len, offset, depth, cd->id, s->num);
                     }
-                    break; /* just add one per sig,
-                            * TODO see if we can select the best one */
-                }
-            }
-        }
-        for (sm = s->match; sm != NULL; sm = sm->next) {
-            if (sm->type == DETECT_URICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) {
-                DetectUricontentData *ud = (DetectUricontentData *)sm->ctx;
-                if (sh->mpm_uricontent_maxlen >= ud->uricontent_len) {
-
-                    if (ud->flags & DETECT_URICONTENT_NOCASE) {
-                        sh->mpm_uri_scan_ctx->AddPatternNocase(sh->mpm_uri_scan_ctx, ud->uricontent, ud->uricontent_len, ud->id);
+                    content_scanadded = 1;
+                } else {
+                    if (cd->flags & DETECT_CONTENT_NOCASE) {
+                        sh->mpm_ctx->AddPatternNocase(sh->mpm_ctx, cd->content, cd->content_len, offset, depth, cd->id, s->num);
                     } else {
-                        sh->mpm_uri_scan_ctx->AddPattern(sh->mpm_uri_scan_ctx, ud->uricontent, ud->uricontent_len, ud->id);
+                        sh->mpm_ctx->AddPattern(sh->mpm_ctx, cd->content, cd->content_len, offset, depth, cd->id, s->num);
                     }
-                    break;
-                }
-            }
-        }
-//#endif
-        /* search ctx */
-        for (sm = s->match; sm != NULL; sm = sm->next) {
-            if (sm->type == DETECT_CONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_COPY)) {
-                DetectContentData *cd = (DetectContentData *)sm->ctx;
-
-                if (cd->flags & DETECT_CONTENT_NOCASE) {
-                    sh->mpm_ctx->AddPatternNocase(sh->mpm_ctx, cd->content, cd->content_len, cd->id);
-                } else {
-                    sh->mpm_ctx->AddPattern(sh->mpm_ctx, cd->content, cd->content_len, cd->id);
                 }
             } else if (sm->type == DETECT_URICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) {
                 DetectUricontentData *ud = (DetectUricontentData *)sm->ctx;
 
-                if (ud->flags & DETECT_URICONTENT_NOCASE) {
-                    sh->mpm_uri_ctx->AddPatternNocase(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, ud->id);
+                if (!uricontent_scanadded && uricontent_maxlen == ud->uricontent_len) {
+                    if (ud->flags & DETECT_URICONTENT_NOCASE) {
+                        sh->mpm_uri_ctx->AddScanPatternNocase(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, 0, 0, ud->id, s->num);
+                    } else {
+                        sh->mpm_uri_ctx->AddScanPattern(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, 0, 0, ud->id, s->num);
+                    }
+                    uricontent_scanadded = 1;
                 } else {
-                    sh->mpm_uri_ctx->AddPattern(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, ud->id);
+                    if (ud->flags & DETECT_URICONTENT_NOCASE) {
+                        sh->mpm_uri_ctx->AddPatternNocase(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, 0, 0, ud->id, s->num);
+                    } else {
+                        sh->mpm_uri_ctx->AddPattern(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, 0, 0, ud->id, s->num);
+                    }
                 }
             }
         }
+
+        content_scanadded = 0;
+        uricontent_scanadded = 0;
     }
 
     /* content */
@@ -378,10 +367,6 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         if (sh->mpm_ctx->Prepare != NULL) {
             sh->mpm_ctx->Prepare(sh->mpm_ctx);
         }
-        /* scan ctx */
-        if (sh->mpm_scan_ctx->Prepare != NULL) {
-            sh->mpm_scan_ctx->Prepare(sh->mpm_scan_ctx);
-        }
 
         if (mpm_content_cnt && sh->mpm_content_maxlen > 1) {
             //printf("mpm_content_cnt %u, mpm_content_maxlen %d\n", mpm_content_cnt, mpm_content_maxlen);
@@ -389,7 +374,7 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
         } else {
             g_content_search++;
         }
-        //printf("(sh %p) mpm_content_cnt %u, mpm_content_maxlen %u, mpm_content_minlen %u, mpm_content_scan_maxlen %u\n", sh, mpm_content_cnt, mpm_content_maxlen, sh->mpm_content_minlen, mpm_content_scan_maxlen);
+        //printf("(sh %p) mpm_content_cnt %u, mpm_content_maxlen %u, mpm_content_scan_maxlen %u\n", sh, mpm_content_cnt, sh->mpm_content_maxlen, mpm_content_scan_maxlen);
 
         if (mpm_content_maxdepth) {
 //            printf("mpm_content_maxdepth %u\n", mpm_content_maxdepth);
@@ -406,14 +391,12 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
 
         //sh->mpm_ctx->PrintCtx(sh->mpm_ctx);
     }
+
     /* uricontent */
     if (sh->flags & SIG_GROUP_HAVEURICONTENT && !(sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY)) {
         if (sh->mpm_uri_ctx->Prepare != NULL) {
             sh->mpm_uri_ctx->Prepare(sh->mpm_uri_ctx);
         }
-        if (sh->mpm_uri_scan_ctx->Prepare != NULL) {
-            sh->mpm_uri_scan_ctx->Prepare(sh->mpm_uri_scan_ctx);
-        }
         if (mpm_uricontent_cnt && sh->mpm_uricontent_maxlen > 1) {
 //            printf("mpm_uricontent_cnt %u, mpm_uricontent_maxlen %d\n", mpm_uricontent_cnt, mpm_uricontent_maxlen);
             g_uricontent_scan++;
@@ -445,6 +428,23 @@ int PatternMatcherThreadInit(ThreadVars *t, void **data) {
      */
     mpm_ctx[0].InitThreadCtx(&mpm_ctx[0], &pmt->mtc, DetectContentMaxId());
     mpm_ctx[0].InitThreadCtx(&mpm_ctx[0], &pmt->mtcu, DetectUricontentMaxId());
+    u_int32_t max_sig_id = SigGetMaxId();
+
+    /* sig callback testing stuff below */
+    pmt->pmq.sig_id_array = malloc(max_sig_id * sizeof(u_int32_t));
+    if (pmt->pmq.sig_id_array == NULL) {
+        printf("ERROR: could not setup memory for pattern matcher: %s\n", strerror(errno));
+        exit(1);
+    }
+    memset(pmt->pmq.sig_id_array, 0, max_sig_id * sizeof(u_int32_t));
+    pmt->pmq.sig_id_array_cnt = 0;
+    /* lookup bitarray */
+    pmt->pmq.sig_bitarray = malloc(max_sig_id / 8 + 1);
+    if (pmt->pmq.sig_bitarray == NULL) {
+        printf("ERROR: could not setup memory for pattern matcher: %s\n", strerror(errno));
+        exit(1);
+    }
+    memset(pmt->pmq.sig_bitarray, 0, max_sig_id / 8 + 1);
 
     *data = (void *)pmt;
     //printf("PatternMatcherThreadInit: data %p pmt %p\n", *data, pmt);

src/detect-engine-port.c

@@ -946,6 +946,8 @@ DetectPort *PortParse(char *str) {
     if (dp == NULL)
         goto error;
 
+    /* XXX better input validation */
+
     /* we dup so we can put a nul-termination in it later */
     char *port = portdup;
 
@@ -957,10 +959,10 @@ DetectPort *PortParse(char *str) {
 
     /* see if the address is an ipv4 or ipv6 address */
     if ((port2 = strchr(port, ':')) != NULL)  {
-        /* 1.2.3.4-1.2.3.6 range format */
+        /* 80:81 range format */
         port[port2 - port] = '\0';
         port2++;
-        dp->port  = atoi(port);
+        dp->port = atoi(port);
         if (strcmp(port2,"") != 0)
             dp->port2 = atoi(port2);
         else
@@ -1472,6 +1474,29 @@ end:
     return result;
 }
 
+int PortTestParse07 (void) {
+    DetectPort *dd = NULL;
+    int result = 0;
+
+    int r = DetectPortParse(&dd,"!21:902");
+    if (r != 0)
+        goto end;
+
+    if (dd->next == NULL)
+        goto end;
+
+    if (dd->port != 0 || dd->port2 != 20)
+        goto end;
+
+    if (dd->next->port != 903 || dd->next->port2 != 65535)
+        goto end;
+
+    DetectPortCleanupList(dd);
+    result = 1;
+end:
+    return result;
+}
+
 
 void DetectPortTests(void) {
     UtRegisterTest("PortTestParse01", PortTestParse01, 1);
@@ -1480,5 +1505,6 @@ void DetectPortTests(void) {
     UtRegisterTest("PortTestParse04", PortTestParse04, 1);
     UtRegisterTest("PortTestParse05", PortTestParse05, 1);
     UtRegisterTest("PortTestParse06", PortTestParse06, 1);
+    UtRegisterTest("PortTestParse07", PortTestParse07, 1);
 }
 

src/detect-parse.c

@@ -304,7 +304,8 @@ int SigParsePort(Signature *s, const char *portstr, char flag) {
 
     /* XXX VJ exclude handling this for none UDP/TCP proto's */
 
-    if (portstr[0] == '!') {
+    /* XXX hack, fix this */
+    if (portstr[0] == '!' && portstr[1] == '$') {
         portstr++;
         negate = 1;
     }
@@ -526,7 +527,31 @@ end:
     return result;
 }
 
+int SigParseTest02 (void) {
+    int result = 0;
+    Signature *sig = NULL;
+
+    sig = SigInit("alert tcp any !21:902 -> any any (msg:\"ET MALWARE Suspicious 220 Banner on Local Port\"; content:\"220\"; offset:0; depth:4; pcre:\"/220[- ]/\"; classtype:non-standard-protocol; sid:2003055; rev:4;)");
+    if (sig == NULL) {
+        goto end;
+    }
+
+    DetectPort *port = NULL;
+    int r = DetectPortParse(&port, "0:20");
+
+    if (DetectPortCmp(sig->sp, port) == PORT_EQ) {
+        result = 1;
+    } else {
+        DetectPortPrint(port); printf(" != "); DetectPortPrint(sig->sp); printf(": ");
+    }
+
+    SigFree(sig);
+end:
+    return result;
+}
+
 void SigParseRegisterTests(void) {
     UtRegisterTest("SigParseTest01", SigParseTest01, 1);
+    UtRegisterTest("SigParseTest02", SigParseTest02, 1);
 }
 

src/detect-pcre.c

@@ -87,8 +87,6 @@ int DetectPcreMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signat
     if (p->tcp_payload_len == 0)
         return 0;
 
-    //printf("DetectPcre: pre match: t->pkt_ptr %p t->pkt_off %u\n", t->pkt_ptr, t->pkt_off);
-
     DetectPcreData *pe = (DetectPcreData *)m->ctx;
     if (s->flags & SIG_FLAG_RECURSIVE) {
         ptr = pmt->pkt_ptr ? pmt->pkt_ptr : p->tcp_payload;
@@ -112,15 +110,45 @@ int DetectPcreMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signat
             ret = pcre_get_substring((char *)ptr, ov, MAX_SUBSTRINGS, 1, &str_ptr);
             if (ret) {
                 if (strcmp(pe->capname,"http_uri") == 0) {
-                    if (pmt->de_scanned_httpuri == 1)
-                        PacketPatternCleanup(t, pmt);
-
-                    pmt->de_have_httpuri = 1;
-                    pmt->de_scanned_httpuri = 0;
-
                     p->http_uri.raw[pmt->pkt_cnt] = (u_int8_t *)str_ptr;
                     p->http_uri.raw_size[pmt->pkt_cnt] = ret;
                     p->http_uri.cnt = pmt->pkt_cnt + 1;
+
+                    /* count how many uri's we handle for stats */
+                    pmt->uris++;
+
+                    //printf("DetectPcre: URI pmt->sgh %p, pmt->mcu %p\n", pmt->sgh, pmt->mcu);
+                    //PrintRawUriFp(stdout,p->http_uri.raw[pmt->pkt_cnt],p->http_uri.raw_size[pmt->pkt_cnt]);
+                    //printf(" (pkt_cnt %u, mcu %p)\n", pmt->pkt_cnt, pmt->mcu);
+
+                    /* don't bother scanning if we don't have a pattern matcher ctx
+                     * which means we don't have uricontent sigs */
+                    if (pmt->mcu != NULL) {
+                        if (pmt->sgh->mpm_uricontent_maxlen <= p->http_uri.raw_size[pmt->pkt_cnt]) {
+                            if (pmt->sgh->mpm_uricontent_maxlen == 1)      pmt->pkts_uri_scanned1++;
+                            else if (pmt->sgh->mpm_uricontent_maxlen == 2) pmt->pkts_uri_scanned2++;
+                            else if (pmt->sgh->mpm_uricontent_maxlen == 3) pmt->pkts_uri_scanned3++;
+                            else if (pmt->sgh->mpm_uricontent_maxlen == 4) pmt->pkts_uri_scanned4++;
+                            else                                           pmt->pkts_uri_scanned++;
+
+                            pmt->pmq.mode = PMQ_MODE_SCAN;
+                            ret = pmt->mcu->Scan(pmt->mcu, &pmt->mtcu, &pmt->pmq, p->http_uri.raw[pmt->pkt_cnt], p->http_uri.raw_size[pmt->pkt_cnt]);
+                            if (ret > 0) {
+                                if (pmt->sgh->mpm_uricontent_maxlen == 1)      pmt->pkts_uri_searched1++;
+                                else if (pmt->sgh->mpm_uricontent_maxlen == 2) pmt->pkts_uri_searched2++;
+                                else if (pmt->sgh->mpm_uricontent_maxlen == 3) pmt->pkts_uri_searched3++;
+                                else if (pmt->sgh->mpm_uricontent_maxlen == 4) pmt->pkts_uri_searched4++;
+                                else                                           pmt->pkts_uri_searched++;
+
+                                pmt->pmq.mode = PMQ_MODE_SEARCH;
+                                ret += pmt->mcu->Search(pmt->mcu, &pmt->mtcu, &pmt->pmq, p->http_uri.raw[pmt->pkt_cnt], p->http_uri.raw_size[pmt->pkt_cnt]);
+
+                                /* indicate to uricontent that we have a uri,
+                                 * we scanned it _AND_ we found pattern matches. */
+                                pmt->de_have_httpuri = 1;
+                            }
+                        }
+                    }
                 } else {
                     if (pe->flags & DETECT_PCRE_CAPTURE_PKT) {
                         PktVarAdd(p, pe->capname, (u_int8_t *)str_ptr, ret);

src/detect-uricontent.c

@@ -231,72 +231,40 @@ DoDetectUricontent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, SigMatch
 int DetectUricontentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signature *s, SigMatch *m)
 {
     u_int32_t len = 0;
-    u_int32_t ret = 0, scan = 0;
-
-    //printf("scanning uricontent have %u scan %u\n", pmt->de_have_httpuri, pmt->de_scanned_httpuri);
-
+/*
+    if (s->id == 2008238) {
+        printf("scanning uricontent have %u\n", pmt->de_have_httpuri);
+        PrintRawUriFp(stdout,p->http_uri.raw[0],p->http_uri.raw_size[0]);
+        printf("\n");
+    }
+*/
     /* if we don't have a uri, don't bother scanning */
     if (pmt->de_have_httpuri == 0)
         return 0;
 
-    if (pmt->de_have_httpuri == 1 && pmt->de_scanned_httpuri == 0) {
-        pmt->de_scanned_httpuri = 1;
-
-        //printf("DetectUricontentMatch: pmt->sgh %p, pmt->mcu %p, pmt->mcu_scan %p\n", pmt->sgh, pmt->mcu, pmt->mcu_scan);
-
-        /* don't bother scanning if we don't have a pattern matcher ctx
-         * which means we don't have uricontent sigs */
-        if (pmt->mcu == NULL || pmt->mcu_scan == NULL)
-            return 0;
-
-        //printf("DetectUricontentMatch: going to scan uri buffer(s)\n");
-
-        /* scan all buffers we have */
-        u_int8_t i;
-        for (i = 0; i < p->http_uri.cnt; i++) {
-            //printf("p->http_uri.raw_size[%u] %u, %p, %s\n", i, p->http_uri.raw_size[i], p->http_uri.raw[i], p->http_uri.raw[i]);
-
-            if (pmt->sgh->mpm_uricontent_maxlen <= p->http_uri.raw_size[i]) {
-                if (pmt->sgh->mpm_uricontent_maxlen == 1)      pmt->pkts_uri_scanned1++;
-                else if (pmt->sgh->mpm_uricontent_maxlen == 2) pmt->pkts_uri_scanned2++;
-                else if (pmt->sgh->mpm_uricontent_maxlen == 3) pmt->pkts_uri_scanned3++;
-                else if (pmt->sgh->mpm_uricontent_maxlen == 4) pmt->pkts_uri_scanned4++;
-                else                                           pmt->pkts_uri_scanned++;
-
-                scan = pmt->mcu_scan->Search(pmt->mcu_scan, &pmt->mtcu, p->http_uri.raw[i], p->http_uri.raw_size[i]);
-                if (scan > 0) {
-                    if (pmt->sgh->mpm_uricontent_maxlen == 1)      pmt->pkts_uri_searched1++;
-                    else if (pmt->sgh->mpm_uricontent_maxlen == 2) pmt->pkts_uri_searched2++;
-                    else if (pmt->sgh->mpm_uricontent_maxlen == 3) pmt->pkts_uri_searched3++;
-                    else if (pmt->sgh->mpm_uricontent_maxlen == 4) pmt->pkts_uri_searched4++;
-                    else                                           pmt->pkts_uri_searched++;
-
-                    ret += pmt->mcu->Search(pmt->mcu, &pmt->mtcu, p->http_uri.raw[i], p->http_uri.raw_size[i]);
-                }
-                //printf("DetectUricontentMatch: ret %u\n", ret);
-            }
-        }
-
-        //printf("DetectUricontentMatch: final ret %u\n", ret);
-        if (ret == 0)
-            return 0;
-    }
-
     DetectUricontentData *co = (DetectUricontentData *)m->ctx;
 
     /* see if we had a match */
     len = pmt->mtcu.match[co->id].len;
+/*
+    if (s->id == 2008238)
+        printf("len %u\n", len);
+*/
     if (len == 0)
         return 0;
 
 #ifdef DEBUG
-    printf("uricontent \'%s\' matched %u time(s) at offsets: ", co->uricontent, len);
+    if (s->id == 2008238) {
+    printf("uricontent \'");
+    PrintRawUriFp(stdout, co->uricontent, co->uricontent_len);    
+    printf("\' matched %u time(s) at offsets: ", len);
 
     MpmMatch *tmpm = NULL;
     for (tmpm = pmt->mtcu.match[co->id].top; tmpm != NULL; tmpm = tmpm->next) {
         printf("%u ", tmpm->offset);
     }
     printf("\n");
+    }
 #endif
 
     return DoDetectUricontent(t, pmt, p, m, co);

src/detect.c

@@ -43,6 +43,8 @@
 #include "action-globals.h"
 #include "tm-modules.h"
 
+#include "pkt-var.h"
+
 #include "util-unittest.h"
 
 static DetectEngineCtx *g_de_ctx = NULL;
@@ -50,8 +52,6 @@ static u_int32_t mpm_memory_size = 0;
 
 SigMatch *SigMatchAlloc(void);
 void SigMatchFree(SigMatch *sm);
-int SignatureTupleCmp(SignatureTuple *a, SignatureTuple *b);
-int SignatureTupleCmpRaw(DetectAddressGroup *src, DetectAddressGroup *dst, DetectPort *sp, DetectPort *dp, u_int8_t proto, SignatureTuple *b);
 void DetectExitPrintStats(ThreadVars *tv, void *data);
 
 /* tm module api functions */
@@ -104,35 +104,35 @@ void DetectExitPrintStats(ThreadVars *tv, void *data) {
         (float)(pmt->pkts_searched/(float)(pmt->pkts)*100),
         (float)(pmt->pkts_searched/(float)(pmt->pkts_scanned)*100));
 
-    printf(" - (%s) URI (1byte) Pkts %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
-        pmt->pkts, pmt->pkts_uri_scanned1,
-        (float)(pmt->pkts_uri_scanned1/(float)(pmt->pkts)*100),
+    printf(" - (%s) URI (1byte) Uri's %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
+        pmt->uris, pmt->pkts_uri_scanned1,
+        (float)(pmt->pkts_uri_scanned1/(float)(pmt->uris)*100),
         pmt->pkts_uri_searched1,
-        (float)(pmt->pkts_uri_searched1/(float)(pmt->pkts)*100),
+        (float)(pmt->pkts_uri_searched1/(float)(pmt->uris)*100),
         (float)(pmt->pkts_uri_searched1/(float)(pmt->pkts_uri_scanned1)*100));
-    printf(" - (%s) URI (2byte) Pkts %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
+    printf(" - (%s) URI (2byte) Uri's %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
         pmt->pkts, pmt->pkts_uri_scanned2,
-        (float)(pmt->pkts_uri_scanned2/(float)(pmt->pkts)*100),
+        (float)(pmt->pkts_uri_scanned2/(float)(pmt->uris)*100),
         pmt->pkts_uri_searched2,
-        (float)(pmt->pkts_uri_searched2/(float)(pmt->pkts)*100),
+        (float)(pmt->pkts_uri_searched2/(float)(pmt->uris)*100),
         (float)(pmt->pkts_uri_searched2/(float)(pmt->pkts_uri_scanned2)*100));
-    printf(" - (%s) URI (3byte) Pkts %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
-        pmt->pkts, pmt->pkts_uri_scanned3,
-        (float)(pmt->pkts_uri_scanned3/(float)(pmt->pkts)*100),
+    printf(" - (%s) URI (3byte) Uri's %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
+        pmt->uris, pmt->pkts_uri_scanned3,
+        (float)(pmt->pkts_uri_scanned3/(float)(pmt->uris)*100),
         pmt->pkts_uri_searched3,
-        (float)(pmt->pkts_uri_searched3/(float)(pmt->pkts)*100),
+        (float)(pmt->pkts_uri_searched3/(float)(pmt->uris)*100),
         (float)(pmt->pkts_uri_searched3/(float)(pmt->pkts_uri_scanned3)*100));
-    printf(" - (%s) URI (4byte) Pkts %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
-        pmt->pkts, pmt->pkts_uri_scanned4,
-        (float)(pmt->pkts_uri_scanned4/(float)(pmt->pkts)*100),
+    printf(" - (%s) URI (4byte) Uri's %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
+        pmt->uris, pmt->pkts_uri_scanned4,
+        (float)(pmt->pkts_uri_scanned4/(float)(pmt->uris)*100),
         pmt->pkts_uri_searched4,
-        (float)(pmt->pkts_uri_searched4/(float)(pmt->pkts)*100),
+        (float)(pmt->pkts_uri_searched4/(float)(pmt->uris)*100),
         (float)(pmt->pkts_uri_searched4/(float)(pmt->pkts_uri_scanned4)*100));
-    printf(" - (%s) (+byte) Pkts %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
-        pmt->pkts, pmt->pkts_uri_scanned,
-        (float)(pmt->pkts_uri_scanned/(float)(pmt->pkts)*100),
+    printf(" - (%s) URI (+byte) Uri's %u, Scanned %u (%02.1f), Searched %u (%02.1f): %02.1f%%.\n", tv->name,
+        pmt->uris, pmt->pkts_uri_scanned,
+        (float)(pmt->pkts_uri_scanned/(float)(pmt->uris)*100),
         pmt->pkts_uri_searched,
-        (float)(pmt->pkts_uri_searched/(float)(pmt->pkts)*100),
+        (float)(pmt->pkts_uri_searched/(float)(pmt->uris)*100),
         (float)(pmt->pkts_uri_searched/(float)(pmt->pkts_uri_scanned)*100));
 }
 
@@ -143,7 +143,6 @@ void SigLoadSignatures (void)
     /* intialize the de_ctx */
     g_de_ctx = DetectEngineCtxInit();
 
-
     /* The next 3 rules handle HTTP header capture. */
 
     /* http_uri -- for uricontent */
@@ -159,14 +158,14 @@ void SigLoadSignatures (void)
     prevsig = sig;
 
     /* http_host -- for the log-httplog module */
-    sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP host cap\"; flow:to_server; content:\"Host:\"; pcre:\"/^Host: (?P<pkt_http_host>.*)\\r\\n/m\"; noalert; sid:3;)");
+    sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP host cap\"; flow:to_server; content:\"|0d 0a|Host:\"; pcre:\"/^Host: (?P<pkt_http_host>.*)\\r\\n/m\"; noalert; sid:3;)");
     if (sig == NULL)
         return;
     prevsig->next = sig;
     prevsig = sig;
 
     /* http_ua -- for the log-httplog module */
-    sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP UA cap\"; flow:to_server; content:\"User-Agent:\"; pcre:\"/^User-Agent: (?P<pkt_http_ua>.*)\\r\\n/m\"; noalert; sid:4;)");
+    sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP UA cap\"; flow:to_server; content:\"|0d 0a|User-Agent:\"; pcre:\"/^User-Agent: (?P<pkt_http_ua>.*)\\r\\n/m\"; noalert; sid:4;)");
     if (sig == NULL)
         return;
     prevsig->next = sig;
@@ -276,6 +275,7 @@ void SigLoadSignatures (void)
     //FILE *fp = fopen("/home/victor/rules/vips-http.sigs", "r");
     //FILE *fp = fopen("/home/victor/rules/emerging-dshield.rules", "r");
     //FILE *fp = fopen("/home/victor/rules/emerging-web.rules", "r");
+    //FILE *fp = fopen("/home/victor/rules/emerging-policy.rules", "r");
     //FILE *fp = fopen("/home/victor/rules/emerging-p2p.rules", "r");
     //FILE *fp = fopen("/home/victor/rules/emerging-web-small.rules", "r");
     //FILE *fp = fopen("/home/victor/rules/web-misc.rules", "r");
@@ -310,8 +310,6 @@ void SigLoadSignatures (void)
     }
     fclose(fp);
     printf("SigLoadSignatures: %d successfully loaded from file. %d sigs failed to load\n", good, bad);
-    printf("SigLoadSignatures: %u sigs with dstportany\n", DbgGetDstPortAnyCnt());
-
 #endif
 
     /* Setup the signature group lookup structure and
@@ -406,11 +404,8 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p)
 
     /* we assume we don't have an uri when we start inspection */
     pmt->de_have_httpuri = 0;
-    pmt->de_scanned_httpuri = 0;
     pmt->mc = NULL;
-    pmt->mc_scan = NULL;
     pmt->mcu = NULL;
-    pmt->mcu_scan = NULL;
     pmt->sgh = NULL;
 
     /* find the right mpm instance */
@@ -421,9 +416,7 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p)
         if (ag != NULL) {
             if (ag->port == NULL) {
                 pmt->mc = ag->sh->mpm_ctx;
-                pmt->mc_scan = ag->sh->mpm_scan_ctx;
                 pmt->mcu = ag->sh->mpm_uri_ctx;
-                pmt->mcu_scan = ag->sh->mpm_uri_scan_ctx;
                 pmt->sgh = ag->sh;
 
                 //printf("SigMatchSignatures: mc %p, mcu %p\n", pmt->mc, pmt->mcu);
@@ -436,9 +429,7 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p)
                     DetectPort *dport = DetectPortLookupGroup(sport->dst_ph,p->dp);
                     if (dport != NULL) {
                         pmt->mc = dport->sh->mpm_ctx;
-                        pmt->mc_scan = dport->sh->mpm_scan_ctx;
                         pmt->mcu = dport->sh->mpm_uri_ctx;
-                        pmt->mcu_scan = dport->sh->mpm_uri_scan_ctx;
                         pmt->sgh = dport->sh;
                     }
                 }
@@ -474,7 +465,8 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p)
                 else                                        pmt->pkts_searched++;
 
                 cnt += PacketPatternMatch(th_v, pmt, p);
-                //printf("search: cnt %u\n", cnt);
+
+                //printf("RAW: cnt %u, pmt->pmq.sig_id_array_cnt %u\n", cnt, pmt->pmq.sig_id_array_cnt);
             }
         }
     }
@@ -483,6 +475,14 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p)
     for (idx = 0; idx < pmt->sgh->sig_cnt; idx++) {
         sig = pmt->sgh->match_array[idx];
         s = g_de_ctx->sig_array[sig];
+
+        /* filter out sigs that want pattern matches, but
+         * have no matches */
+        if (!(pmt->pmq.sig_bitarray[(sig / 8)] & (1<<(sig % 8))) &&
+            (s->flags & SIG_FLAG_MPM))
+            continue;
+
+        //printf("idx %u, pmt->pmq.sig_id_array_cnt %u, s->id %u (MPM? %s)\n", idx, pmt->pmq.sig_id_array_cnt, s->id, s->flags & SIG_FLAG_MPM ? "TRUE":"FALSE");
         //printf("Sig %u\n", s->id);
 
         /* check the source & dst port in the sig */
@@ -564,7 +564,7 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p)
                     /* only if the last matched as well, we have a hit */
                     if (sm == NULL) {
                         fmatch = 1;
-
+//printf("DE : sig %u matched\n", s->id);
                         if (!(s->flags & SIG_FLAG_NOALERT)) {
                             PacketAlertAppend(p, 1, s->id, s->rev, s->prio, s->msg);
 
@@ -1390,7 +1390,6 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr
                         de_ctx->mpm_unique++;
                     } else {
                         sgr->sh->mpm_ctx = mpmsh->mpm_ctx;
-                        sgr->sh->mpm_scan_ctx = mpmsh->mpm_scan_ctx;
                         sgr->sh->flags |= SIG_GROUP_HEAD_MPM_COPY;
                         SigGroupHeadClearContent(sgr->sh);
 
@@ -1410,7 +1409,6 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr
                         de_ctx->mpm_uri_unique++;
                     } else {
                         sgr->sh->mpm_uri_ctx = mpmsh->mpm_uri_ctx;
-                        sgr->sh->mpm_uri_scan_ctx = mpmsh->mpm_uri_scan_ctx;
                         sgr->sh->flags |= SIG_GROUP_HEAD_MPM_URI_COPY;
                         SigGroupHeadClearUricontent(sgr->sh);
 
@@ -1790,7 +1788,6 @@ static int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, De
                                         de_ctx->mpm_unique++;
                                     } else {
                                         dp->sh->mpm_ctx = mpmsh->mpm_ctx;
-                                        dp->sh->mpm_scan_ctx = mpmsh->mpm_scan_ctx;
                                         dp->sh->flags |= SIG_GROUP_HEAD_MPM_COPY;
                                         SigGroupHeadClearContent(dp->sh);
 
@@ -1810,7 +1807,6 @@ static int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, De
                                         de_ctx->mpm_uri_unique++;
                                     } else {
                                         dp->sh->mpm_uri_ctx = mpmsh->mpm_uri_ctx;
-                                        dp->sh->mpm_uri_scan_ctx = mpmsh->mpm_uri_scan_ctx;
                                         dp->sh->flags |= SIG_GROUP_HEAD_MPM_URI_COPY;
                                         SigGroupHeadClearUricontent(dp->sh);
 
@@ -2005,9 +2001,10 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) {
 //    DetectPortPrintMemory();
 //#endif
     if (!(de_ctx->flags & DE_QUIET)) {
-        printf("* MPM memory %u (dynamic %u, ctxs %u)\n",
+        printf("* MPM memory %u (dynamic %u, ctxs %u, avg per ctx %u)\n",
             mpm_memory_size + ((de_ctx->mpm_unique + de_ctx->mpm_uri_unique) * sizeof(MpmCtx)),
-            mpm_memory_size, ((de_ctx->mpm_unique + de_ctx->mpm_uri_unique) * sizeof(MpmCtx)));
+            mpm_memory_size, ((de_ctx->mpm_unique + de_ctx->mpm_uri_unique) * sizeof(MpmCtx)),
+            mpm_memory_size / de_ctx->mpm_unique);
 
         printf(" * Max sig id %u, array size %u\n", SigGetMaxId(), SigGetMaxId() / 8 + 1);
         printf("* Signature group heads: unique %u, copies %u.\n", de_ctx->gh_unique, de_ctx->gh_reuse);
@@ -2761,6 +2758,10 @@ int SigTest06 (void) {
     SigMatchSignatures(&th_v, pmt, &p);
     if (PacketAlertCheck(&p, 1) && PacketAlertCheck(&p, 2))
         result = 1;
+    else
+        printf("sid:1 %s, sid:2 %s: ",
+            PacketAlertCheck(&p, 1) ? "OK" : "FAIL",
+            PacketAlertCheck(&p, 2) ? "OK" : "FAIL");
 
     SigGroupCleanup();
     SigCleanSignatures();
@@ -2879,6 +2880,10 @@ int SigTest08 (void) {
     SigMatchSignatures(&th_v, pmt, &p);
     if (PacketAlertCheck(&p, 1) && PacketAlertCheck(&p, 2))
         result = 1;
+    else
+        printf("sid:1 %s, sid:2 %s: ",
+            PacketAlertCheck(&p, 1) ? "OK" : "FAIL",
+            PacketAlertCheck(&p, 2) ? "OK" : "FAIL");
 
     SigGroupCleanup();
     SigCleanSignatures();
@@ -3304,6 +3309,121 @@ end:
     return result;
 }
 
+int SigTest17 (void) {
+    u_int8_t *buf = (u_int8_t *)
+                    "GET /one/ HTTP/1.1\r\n"    /* 20 */
+                    "Host: one.example.org\r\n" /* 23, 43 */
+                    "\r\n\r\n"                  /* 4,  47 */
+                    "GET /two/ HTTP/1.1\r\n"    /* 20, 67 */
+                    "Host: two.example.org\r\n" /* 23, 90 */
+                    "\r\n\r\n";                 /* 4,  94 */
+    u_int16_t buflen = strlen((char *)buf);
+    Packet p;
+    ThreadVars th_v;
+    PatternMatcherThread *pmt;
+    int result = 0;
+
+    memset(&th_v, 0, sizeof(th_v));
+    memset(&p, 0, sizeof(p));
+    p.src.family = AF_INET;
+    p.dst.family = AF_INET;
+    p.tcp_payload = buf;
+    p.tcp_payload_len = buflen;
+    p.proto = IPPROTO_TCP;
+    p.dp = 80;
+
+    g_de_ctx = DetectEngineCtxInit();
+    if (g_de_ctx == NULL) {
+        goto end;
+    }
+
+    g_de_ctx->flags |= DE_QUIET;
+
+    g_de_ctx->sig_list = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP host cap\"; content:\"Host:\"; pcre:\"/^Host: (?P<pkt_http_host>.*)\\r\\n/m\"; noalert; sid:1;)");
+    if (g_de_ctx->sig_list == NULL) {
+        result = 0;
+        goto end;
+    }
+
+    SigGroupBuild(g_de_ctx);
+    PatternMatchPrepare(mpm_ctx);
+    PatternMatcherThreadInit(&th_v, (void *)&pmt);
+
+    SigMatchSignatures(&th_v, pmt, &p);
+    PktVar *pv_hn = PktVarGet(&p, "http_host");
+    if (pv_hn != NULL) {
+        if (memcmp(pv_hn->value, "one.example.org", pv_hn->value_len < 15 ? pv_hn->value_len : 15) == 0)
+            result = 1;
+        else {
+            printf("\"");
+            PrintRawUriFp(stdout, pv_hn->value, pv_hn->value_len);
+            printf("\" != \"one.example.org\": ");
+        }
+    } else {
+        printf("Pkt var http_host not captured: ");
+    }
+
+    SigGroupCleanup();
+    SigCleanSignatures();
+
+    PatternMatcherThreadDeinit(&th_v, (void *)pmt);
+    PatternMatchDestroy(mpm_ctx);
+    DetectEngineCtxFree(g_de_ctx);
+end:
+    return result;
+}
+
+int SigTest18 (void) {
+    u_int8_t *buf = (u_int8_t *)
+                    "220 (vsFTPd 2.0.5)\r\n";
+    u_int16_t buflen = strlen((char *)buf);
+    Packet p;
+    ThreadVars th_v;
+    PatternMatcherThread *pmt;
+    int result = 0;
+
+    memset(&th_v, 0, sizeof(th_v));
+    memset(&p, 0, sizeof(p));
+    p.src.family = AF_INET;
+    p.dst.family = AF_INET;
+    p.tcp_payload = buf;
+    p.tcp_payload_len = buflen;
+    p.proto = IPPROTO_TCP;
+    p.dp = 34260;
+    p.sp = 21;
+
+    g_de_ctx = DetectEngineCtxInit();
+    if (g_de_ctx == NULL) {
+        goto end;
+    }
+
+    g_de_ctx->flags |= DE_QUIET;
+
+    g_de_ctx->sig_list = SigInit("alert tcp any !21:902 -> any any (msg:\"ET MALWARE Suspicious 220 Banner on Local Port\"; content:\"220\"; offset:0; depth:4; pcre:\"/220[- ]/\"; classtype:non-standard-protocol; sid:2003055; rev:4;)");
+    if (g_de_ctx->sig_list == NULL) {
+        result = 0;
+        goto end;
+    }
+
+    SigGroupBuild(g_de_ctx);
+    PatternMatchPrepare(mpm_ctx);
+    PatternMatcherThreadInit(&th_v, (void *)&pmt);
+
+    SigMatchSignatures(&th_v, pmt, &p);
+    if (!PacketAlertCheck(&p, 2003055))
+        result = 1;
+    else
+        printf("signature shouldn't match, but did: ");
+
+    SigGroupCleanup();
+    SigCleanSignatures();
+    PatternMatcherThreadDeinit(&th_v, (void *)pmt);
+    PatternMatchDestroy(mpm_ctx);
+    DetectEngineCtxFree(g_de_ctx);
+end:
+    return result;
+}
+
 void SigRegisterTests(void) {
     SigParseRegisterTests();
     UtRegisterTest("SigTest01 -- HTTP URI cap", SigTest01, 1);
@@ -3322,5 +3442,7 @@ void SigRegisterTests(void) {
     UtRegisterTest("SigTest14 -- content order matching, distance 0", SigTest14, 1);
     UtRegisterTest("SigTest15 -- port negation sig (no match)", SigTest15, 1);
     UtRegisterTest("SigTest16 -- port negation sig (match)", SigTest16, 1);
+    UtRegisterTest("SigTest17 -- HTTP Host Pkt var capture", SigTest17, 1);
+    UtRegisterTest("SigTest18 -- Ftp negation sig test", SigTest18, 1);
 }
 

src/detect.h

@@ -8,15 +8,18 @@
 #include "detect-content.h"
 #include "detect-uricontent.h"
 
-#define SIG_FLAG_RECURSIVE 0x01
-#define SIG_FLAG_SRC_ANY   0x02
-#define SIG_FLAG_DST_ANY   0x04
-#define SIG_FLAG_SP_ANY    0x08
-#define SIG_FLAG_DP_ANY    0x10
-#define SIG_FLAG_NOALERT   0x20
-#define SIG_FLAG_IPONLY    0x40 /* ip only signature */
-
-#define DE_QUIET           0x01
+/* Signature flags */
+#define SIG_FLAG_RECURSIVE 0x0001 /* recurive capturing enabled */
+#define SIG_FLAG_SRC_ANY   0x0002 /* source is any */
+#define SIG_FLAG_DST_ANY   0x0004 /* destination is any */
+#define SIG_FLAG_SP_ANY    0x0008 /* source port is any */
+#define SIG_FLAG_DP_ANY    0x0010 /* destination port is any */
+#define SIG_FLAG_NOALERT   0x0020 /* no alert flag is set */
+#define SIG_FLAG_IPONLY    0x0040 /* ip only signature */
+#define SIG_FLAG_MPM       0x0080 /* sig has mpm portion (content, uricontent, etc) */
+
+/* Detection Engine flags */
+#define DE_QUIET           0x01   /* DE is quiet (esp for unittests) */
 
 typedef struct _PatternMatcherThread {
     /* detection engine variables */
@@ -28,24 +31,18 @@ typedef struct _PatternMatcherThread {
 
     /* http_uri stuff for uricontent */
     char de_have_httpuri;
-    char de_scanned_httpuri;
 
     /* pointer to the current mpm ctx that is stored
      * in a rule group head -- can be either a content
-     * or uricontent ctx.
-     *
-     * XXX rename to mpm_ctx as soon as the threading
-     * thing above is renamed as well  */
-    MpmCtx *mc;       /* search ctx */
-    MpmCtx *mc_scan;  /* scan ctx */
-    MpmCtx *mcu;
-    MpmCtx *mcu_scan;
-
+     * or uricontent ctx. */
+    MpmCtx *mc;  /* content */
+    MpmCtx *mcu; /* uricontent */
     MpmThreadCtx mtc;
     MpmThreadCtx mtcu;
-
     struct _SigGroupHead *sgh;
+    PatternMatcherQueue pmq;
 
+    /* counters */
     u_int32_t pkts;
     u_int32_t pkts_scanned;
     u_int32_t pkts_searched;
@@ -58,6 +55,7 @@ typedef struct _PatternMatcherThread {
     u_int32_t pkts_scanned4;
     u_int32_t pkts_searched4;
 
+    u_int32_t uris;
     u_int32_t pkts_uri_scanned;
     u_int32_t pkts_uri_searched;
     u_int32_t pkts_uri_scanned1;
@@ -68,12 +66,13 @@ typedef struct _PatternMatcherThread {
     u_int32_t pkts_uri_searched3;
     u_int32_t pkts_uri_scanned4;
     u_int32_t pkts_uri_searched4;
+
 } PatternMatcherThread;
 
 typedef struct _Signature {
-    u_int8_t flags;
+    u_int16_t flags;
 
-    u_int32_t num; /* signature number */
+    u_int32_t num; /* signature number, internal id */
     u_int32_t id;
     u_int8_t rev;
     u_int8_t prio;
@@ -138,7 +137,7 @@ typedef struct DetectEngineCtx_ {
               mpm_uri_tot_patcnt;
 
 } DetectEngineCtx;
-
+/*
 typedef struct SignatureTuple_ {
     DetectAddressGroup *src;
     DetectAddressGroup *dst;
@@ -153,7 +152,7 @@ typedef struct SignatureTuple_ {
 
     u_int32_t cnt;
 } SignatureTuple;
-
+*/
 /* container for content matches... we use this to compare
  * group heads for contents
  * XXX name */
@@ -187,10 +186,8 @@ typedef struct _SigGroupHead {
 
     /* pattern matcher instance */
     MpmCtx *mpm_ctx;      /* search */
-    MpmCtx *mpm_scan_ctx; /* scan */
     u_int16_t mpm_content_maxlen;
     MpmCtx *mpm_uri_ctx;
-    MpmCtx *mpm_uri_scan_ctx;
     u_int16_t mpm_uricontent_maxlen;
 
     /* number of sigs in this head */

src/flow-hash.c

@@ -9,19 +9,32 @@
 #include "flow-util.h"
 #include "flow-private.h"
 
+/* calculate the hash key for this packet
+ *
+ * we're using:
+ *  hash_rand -- set at init time
+ *  source port
+ *  destination port
+ *  source address
+ *  destination address
+ *  recursion level -- for tunnels, make sure different tunnel layers can
+ *                     never get mixed up.
+ */
 u_int32_t FlowGetKey(Packet *p) {
     FlowKey *k = (FlowKey *)p;
     u_int32_t key;
 
     if (p->ip4h != NULL)
-        key = (flow_config.hash_rand + k->sp + k->dp + \
-               k->src.addr_data32[0] + k->dst.addr_data32[0]) % flow_config.hash_size;
+        key = (flow_config.hash_rand + k->proto + k->sp + k->dp + \
+               k->src.addr_data32[0] + k->dst.addr_data32[0] + \
+               k->recursion_level) % flow_config.hash_size;
     else if (p->ip6h != NULL)
-        key = (flow_config.hash_rand + k->sp + k->dp + \
+        key = (flow_config.hash_rand + k->proto + k->sp + k->dp + \
                k->src.addr_data32[0] + k->src.addr_data32[1] + \
                k->src.addr_data32[2] + k->src.addr_data32[3] + \
                k->dst.addr_data32[0] + k->dst.addr_data32[1] + \
-               k->dst.addr_data32[2] + k->dst.addr_data32[3]) % flow_config.hash_size;
+               k->dst.addr_data32[2] + k->dst.addr_data32[3] + \
+               k->recursion_level) % flow_config.hash_size;
     else
         key = 0;
 
@@ -31,12 +44,14 @@ u_int32_t FlowGetKey(Packet *p) {
 /* Since two or more flows can have the same hash key, we need to compare
  * the flow with the current flow key. */
 #define CMP_FLOW(f1,f2) \
-    ((CMP_ADDR(&(f1)->src, &(f2)->src) && \
-      CMP_ADDR(&(f1)->dst, &(f2)->dst) && \
-      CMP_PORT((f1)->sp, (f2)->sp) && CMP_PORT((f1)->dp, (f2)->dp)) || \
-     (CMP_ADDR(&(f1)->src, &(f2)->dst) && \
-      CMP_ADDR(&(f1)->dst, &(f2)->src) && \
-      CMP_PORT((f1)->sp, (f2)->dp) && CMP_PORT((f1)->dp, (f2)->sp)))
+    (((CMP_ADDR(&(f1)->src, &(f2)->src) && \
+       CMP_ADDR(&(f1)->dst, &(f2)->dst) && \
+       CMP_PORT((f1)->sp, (f2)->sp) && CMP_PORT((f1)->dp, (f2)->dp)) || \
+      (CMP_ADDR(&(f1)->src, &(f2)->dst) && \
+       CMP_ADDR(&(f1)->dst, &(f2)->src) && \
+       CMP_PORT((f1)->sp, (f2)->dp) && CMP_PORT((f1)->dp, (f2)->sp))) && \
+     (f1)->proto == (f2)->proto && \
+     (f1)->recursion_level == (f2)->recursion_level)
 
 /* FlowGetFlowFromHash
  *

src/flow-util.c

@@ -49,10 +49,15 @@ void FlowFree(Flow *f)
     free(f);
 }
 
+/* initialize the flow from the first packet
+ * we see from it. */
 void FlowInit(Flow *f, Packet *p)
 {
     CLEAR_FLOW(f);
 
+    f->proto = p->proto;
+    f->recursion_level = p->recursion_level;
+
     if (p->ip4h != NULL) { /* XXX MACRO */
         SET_IPV4_SRC_ADDR(p,&f->src);
         SET_IPV4_DST_ADDR(p,&f->dst);
@@ -67,6 +72,9 @@ void FlowInit(Flow *f, Packet *p)
     if (p->tcph != NULL) { /* XXX MACRO */
         SET_TCP_SRC_PORT(p,&f->sp);
         SET_TCP_DST_PORT(p,&f->dp);
+    } else if (p->udph != NULL) { /* XXX MACRO */
+        SET_UDP_SRC_PORT(p,&f->sp);
+        SET_UDP_DST_PORT(p,&f->dp);
     } /* XXX handle default */
     else {
         printf("FIXME: %s:%s:%d\n", __FILE__, __FUNCTION__, __LINE__);

src/flow.h

@@ -33,6 +33,8 @@ typedef struct _FlowKey
 {
     Address src, dst;
     Port sp, dp;
+    u_int8_t proto;
+    u_int8_t recursion_level;
 
 } FlowKey;
 
@@ -40,6 +42,8 @@ typedef struct _Flow
 {
     Address src, dst;
     Port sp, dp;
+    u_int8_t proto;
+    u_int8_t recursion_level;
 
     u_int8_t flags;
 

src/source-nfq.c

@@ -250,20 +250,23 @@ int ReceiveNFQThreadInit(ThreadVars *tv, void **data) {
 }
 
 int VerdictNFQThreadInit(ThreadVars *tv, void **data) {
-    //printf("VerdictNFQThreadInit: starting... will bind to queuenum %u\n", verdict_queue_num);
+    printf("VerdictNFQThreadInit: starting... will bind to queuenum %u\n", verdict_queue_num);
 
     /* no initialization, ReceiveNFQ takes care of that */
     NFQThreadVars *ntv = &nfq_t[verdict_queue_num];
 
     *data = (void *)ntv;
     verdict_queue_num++;
+
+    printf("VerdictNFQThreadInit: ntv %p\n", ntv);
     return 0;
 }
 
 int VerdictNFQThreadDeinit(ThreadVars *tv, void *data) {
     NFQThreadVars *ntv = (NFQThreadVars *)data;
 
-    //printf("VerdictNFQThreadDeinit: starting... will close queuenum %u\n", ntv->queue_num);
+    printf("VerdictNFQThreadDeinit: ntv %p\n", ntv);
+    printf("VerdictNFQThreadDeinit: starting... will close queuenum %u\n", ntv->queue_num);
 
     nfq_destroy_queue(ntv->qh);
 
@@ -309,7 +312,16 @@ int ReceiveNFQ(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq) {
     sigfillset(&sigs);
     pthread_sigmask(SIG_BLOCK, &sigs, NULL);
 
+    /* do our nfq magic */
     NFQRecvPkt(ntv);
+
+    /* check if we have too many packets in the system
+     * so we will wait for some to free up */
+    mutex_lock(&mutex_pending);
+    if (pending > MAX_PENDING) {
+        pthread_cond_wait(&cond_pending, &mutex_pending);
+    }
+    mutex_unlock(&mutex_pending);
     return 0;
 }
 

src/tm-threads.c

@@ -9,77 +9,62 @@
 /* root of the threadvars list */
 static ThreadVars *tv_root;
 
+typedef struct _TmSlot {
+    /* function pointers */
+    int (*SlotInit)(ThreadVars *, void **);
+    int (*SlotFunc)(ThreadVars *, Packet *, void *, PacketQueue *);
+    void (*SlotExitPrintStats)(ThreadVars *, void *);
+    int (*SlotDeinit)(ThreadVars *, void *);
+
+    /* data storage */
+    void *slot_data;
+    PacketQueue slot_pq;
+
+    /* linked list, only used by TmVarSlot */
+    struct _TmSlot *slot_next;
+} TmSlot;
+
 /* 1 function slot */
 typedef struct _Tm1Slot {
-    int (*Slot1Init)(ThreadVars *, void **);
-    int (*Slot1Func)(ThreadVars *, Packet *, void *, PacketQueue *);
-    void (*Slot1ExitPrintStats)(ThreadVars *, void *);
-    int (*Slot1Deinit)(ThreadVars *, void *);
-    void *slot1_data;
-    PacketQueue slot1_pq;
+    TmSlot s;
 } Tm1Slot;
 
 /* 2 function slot */
 typedef struct _Tm2Slot {
-    int (*Slot1Init)(ThreadVars *, void **);
-    int (*Slot1Func)(ThreadVars *, Packet *, void *, PacketQueue *);
-    void (*Slot1ExitPrintStats)(ThreadVars *, void *);
-    int (*Slot1Deinit)(ThreadVars *, void *);
-    void *slot1_data;
-    PacketQueue slot1_pq;
-
-    int (*Slot2Init)(ThreadVars *, void **);
-    int (*Slot2Func)(ThreadVars *, Packet *, void *, PacketQueue *);
-    void (*Slot2ExitPrintStats)(ThreadVars *, void *);
-    int (*Slot2Deinit)(ThreadVars *, void *);
-    void *slot2_data;
-    PacketQueue slot2_pq;
+    TmSlot s1, s2;
 } Tm2Slot;
 
 /* 3 function slot */
 typedef struct _Tm3Slot {
-    int (*Slot1Init)(ThreadVars *, void **);
-    int (*Slot1Func)(ThreadVars *, Packet *, void *, PacketQueue *);
-    void (*Slot1ExitPrintStats)(ThreadVars *, void *);
-    int (*Slot1Deinit)(ThreadVars *, void *);
-    void *slot1_data;
-    PacketQueue slot1_pq;
-
-    int (*Slot2Init)(ThreadVars *, void **);
-    int (*Slot2Func)(ThreadVars *, Packet *, void *, PacketQueue *);
-    void (*Slot2ExitPrintStats)(ThreadVars *, void *);
-    int (*Slot2Deinit)(ThreadVars *, void *);
-    void *slot2_data;
-    PacketQueue slot2_pq;
-
-    int (*Slot3Init)(ThreadVars *, void **);
-    int (*Slot3Func)(ThreadVars *, Packet *, void *, PacketQueue *);
-    void (*Slot3ExitPrintStats)(ThreadVars *, void *);
-    int (*Slot3Deinit)(ThreadVars *, void *);
-    void *slot3_data;
-    PacketQueue slot3_pq;
+    TmSlot s1, s2, s3;
 } Tm3Slot;
 
+/* Variable number of function slots */
+typedef struct _TmVarSlot {
+    TmSlot *s;
+} TmVarSlot;
+
+/* 1 slot functions */
 
 void *TmThreadsSlot1NoIn(void *td) {
     ThreadVars *tv = (ThreadVars *)td;
-    Tm1Slot *s1 = (Tm1Slot *)tv->tm_slots;
+    Tm1Slot *s = (Tm1Slot *)tv->tm_slots;
     Packet *p = NULL;
     char run = 1;
     int r = 0;
 
-    if (s1->Slot1Init != NULL) {
-        r = s1->Slot1Init(tv, &s1->slot1_data);
+    if (s->s.SlotInit != NULL) {
+        r = s->s.SlotInit(tv, &s->s.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
-    memset(&s1->slot1_pq, 0, sizeof(PacketQueue));
+    memset(&s->s.slot_pq, 0, sizeof(PacketQueue));
 
     while(run) {
-        r = s1->Slot1Func(tv, p, s1->slot1_data, &s1->slot1_pq);
-        while (s1->slot1_pq.len > 0) {
-            Packet *extra = PacketDequeue(&s1->slot1_pq);
+        r = s->s.SlotFunc(tv, p, s->s.slot_data, &s->s.slot_pq);
+        while (s->s.slot_pq.len > 0) {
+            Packet *extra = PacketDequeue(&s->s.slot_pq);
             tv->tmqh_out(tv, extra);
         }
 
@@ -91,12 +76,12 @@ void *TmThreadsSlot1NoIn(void *td) {
             run = 0;
     }
 
-    if (s1->Slot1ExitPrintStats != NULL) {
-        s1->Slot1ExitPrintStats(tv, s1->slot1_data);
+    if (s->s.SlotExitPrintStats != NULL) {
+        s->s.SlotExitPrintStats(tv, s->s.slot_data);
     }
 
-    if (s1->Slot1Deinit != NULL) {
-        r = s1->Slot1Deinit(tv, s1->slot1_data);
+    if (s->s.SlotDeinit != NULL) {
+        r = s->s.SlotDeinit(tv, s->s.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
@@ -107,35 +92,35 @@ void *TmThreadsSlot1NoIn(void *td) {
 
 void *TmThreadsSlot1NoOut(void *td) {
     ThreadVars *tv = (ThreadVars *)td;
-    Tm1Slot *s1 = (Tm1Slot *)tv->tm_slots;
+    Tm1Slot *s = (Tm1Slot *)tv->tm_slots;
     Packet *p = NULL;
     char run = 1;
     int r = 0;
 
-    if (s1->Slot1Init != NULL) {
-        r = s1->Slot1Init(tv, &s1->slot1_data);
+    if (s->s.SlotInit != NULL) {
+        r = s->s.SlotInit(tv, &s->s.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
-    memset(&s1->slot1_pq, 0, sizeof(PacketQueue));
+    memset(&s->s.slot_pq, 0, sizeof(PacketQueue));
 
     while(run) {
         p = tv->tmqh_in(tv);
 
-        r = s1->Slot1Func(tv, p, s1->slot1_data, /* no outqh no pq */NULL);
+        r = s->s.SlotFunc(tv, p, s->s.slot_data, /* no outqh no pq */NULL);
         /* XXX handle error */
 
         if (tv->flags & THV_KILL)
             run = 0;
     }
 
-    if (s1->Slot1ExitPrintStats != NULL) {
-        s1->Slot1ExitPrintStats(tv, s1->slot1_data);
+    if (s->s.SlotExitPrintStats != NULL) {
+        s->s.SlotExitPrintStats(tv, s->s.slot_data);
     }
 
-    if (s1->Slot1Deinit != NULL) {
-        r = s1->Slot1Deinit(tv, s1->slot1_data);
+    if (s->s.SlotDeinit != NULL) {
+        r = s->s.SlotDeinit(tv, s->s.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
@@ -146,22 +131,22 @@ void *TmThreadsSlot1NoOut(void *td) {
 
 void *TmThreadsSlot1NoInOut(void *td) {
     ThreadVars *tv = (ThreadVars *)td;
-    Tm1Slot *s1 = (Tm1Slot *)tv->tm_slots;
+    Tm1Slot *s = (Tm1Slot *)tv->tm_slots;
     char run = 1;
     int r = 0;
 
     //printf("TmThreadsSlot1NoInOut: %s starting\n", tv->name);
 
-    if (s1->Slot1Init != NULL) {
-        r = s1->Slot1Init(tv, &s1->slot1_data);
+    if (s->s.SlotInit != NULL) {
+        r = s->s.SlotInit(tv, &s->s.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
-    memset(&s1->slot1_pq, 0, sizeof(PacketQueue));
+    memset(&s->s.slot_pq, 0, sizeof(PacketQueue));
 
     while(run) {
-        r = s1->Slot1Func(tv, NULL, s1->slot1_data, /* no outqh, no pq */NULL);
+        r = s->s.SlotFunc(tv, NULL, s->s.slot_data, /* no outqh, no pq */NULL);
         //printf("%s: TmThreadsSlot1NoInNoOut: r %d\n", tv->name, r);
         /* XXX handle error */
 
@@ -171,12 +156,12 @@ void *TmThreadsSlot1NoInOut(void *td) {
         }
     }
 
-    if (s1->Slot1ExitPrintStats != NULL) {
-        s1->Slot1ExitPrintStats(tv, s1->slot1_data);
+    if (s->s.SlotExitPrintStats != NULL) {
+        s->s.SlotExitPrintStats(tv, s->s.slot_data);
     }
 
-    if (s1->Slot1Deinit != NULL) {
-        r = s1->Slot1Deinit(tv, s1->slot1_data);
+    if (s->s.SlotDeinit != NULL) {
+        r = s->s.SlotDeinit(tv, s->s.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
@@ -188,20 +173,20 @@ void *TmThreadsSlot1NoInOut(void *td) {
 
 void *TmThreadsSlot1(void *td) {
     ThreadVars *tv = (ThreadVars *)td;
-    Tm1Slot *s1 = (Tm1Slot *)tv->tm_slots;
+    Tm1Slot *s = (Tm1Slot *)tv->tm_slots;
     Packet *p = NULL;
     char run = 1;
     int r = 0;
 
     //printf("TmThreadsSlot1: %s starting\n", tv->name);
 
-    if (s1->Slot1Init != NULL) {
-        r = s1->Slot1Init(tv, &s1->slot1_data);
+    if (s->s.SlotInit != NULL) {
+        r = s->s.SlotInit(tv, &s->s.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
-    memset(&s1->slot1_pq, 0, sizeof(PacketQueue));
+    memset(&s->s.slot_pq, 0, sizeof(PacketQueue));
 
     while(run) {
         /* input a packet */
@@ -210,10 +195,10 @@ void *TmThreadsSlot1(void *td) {
         if (p == NULL) {
             //printf("%s: TmThreadsSlot1: p == NULL\n", tv->name);
         } else {
-            r = s1->Slot1Func(tv, p, s1->slot1_data, &s1->slot1_pq);
-            while (s1->slot1_pq.len > 0) {
+            r = s->s.SlotFunc(tv, p, s->s.slot_data, &s->s.slot_pq);
+            while (s->s.slot_pq.len > 0) {
                 /* handle new packets from this func */
-                Packet *extra_p = PacketDequeue(&s1->slot1_pq);
+                Packet *extra_p = PacketDequeue(&s->s.slot_pq);
                 tv->tmqh_out(tv, extra_p);
             }
 
@@ -230,12 +215,12 @@ void *TmThreadsSlot1(void *td) {
         }
     }
 
-    if (s1->Slot1ExitPrintStats != NULL) {
-        s1->Slot1ExitPrintStats(tv, s1->slot1_data);
+    if (s->s.SlotExitPrintStats != NULL) {
+        s->s.SlotExitPrintStats(tv, s->s.slot_data);
     }
 
-    if (s1->Slot1Deinit != NULL) {
-        r = s1->Slot1Deinit(tv, s1->slot1_data);
+    if (s->s.SlotDeinit != NULL) {
+        r = s->s.SlotDeinit(tv, s->s.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
@@ -247,21 +232,21 @@ void *TmThreadsSlot1(void *td) {
 
 void *TmThreadsSlot2(void *td) {
     ThreadVars *tv = (ThreadVars *)td;
-    Tm2Slot *s2 = (Tm2Slot *)tv->tm_slots;
+    Tm2Slot *s = (Tm2Slot *)tv->tm_slots;
     Packet *p = NULL;
     char run = 1;
     int r = 0;
 
     //printf("TmThreadsSlot2: %s starting\n", tv->name);
 
-    if (s2->Slot1Init != NULL) {
-        r = s2->Slot1Init(tv, &s2->slot1_data);
+    if (s->s1.SlotInit != NULL) {
+        r = s->s1.SlotInit(tv, &s->s1.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
-    if (s2->Slot2Init != NULL) {
-        r = s2->Slot2Init(tv, &s2->slot2_data);
+    if (s->s2.SlotInit != NULL) {
+        r = s->s2.SlotInit(tv, &s->s2.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
@@ -274,23 +259,23 @@ void *TmThreadsSlot2(void *td) {
         if (p == NULL) {
             //printf("%s: TmThreadsSlot1: p == NULL\n", tv->name);
         } else {
-            r = s2->Slot1Func(tv, p, s2->slot1_data, &s2->slot1_pq);
-            while (s2->slot1_pq.len > 0) {
+            r = s->s1.SlotFunc(tv, p, s->s1.slot_data, &s->s1.slot_pq);
+            while (s->s1.slot_pq.len > 0) {
                 /* handle new packets from this func */
-                Packet *extra_p = PacketDequeue(&s2->slot1_pq);
+                Packet *extra_p = PacketDequeue(&s->s1.slot_pq);
 
-                r = s2->Slot2Func(tv, extra_p, s2->slot2_data, &s2->slot2_pq);
-                while (s2->slot2_pq.len > 0) {
+                r = s->s2.SlotFunc(tv, extra_p, s->s2.slot_data, &s->s2.slot_pq);
+                while (s->s2.slot_pq.len > 0) {
                     /* handle new packets from this func */
-                    Packet *extra_p2 = PacketDequeue(&s2->slot2_pq);
+                    Packet *extra_p2 = PacketDequeue(&s->s2.slot_pq);
                     tv->tmqh_out(tv, extra_p2);
                 }
                 tv->tmqh_out(tv, extra_p);
             }
-            r = s2->Slot2Func(tv, p, s2->slot2_data, &s2->slot2_pq);
-            while (s2->slot2_pq.len > 0) {
+            r = s->s2.SlotFunc(tv, p, s->s2.slot_data, &s->s2.slot_pq);
+            while (s->s2.slot_pq.len > 0) {
                 /* handle new packets from this func */
-                Packet *extra_p = PacketDequeue(&s2->slot2_pq);
+                Packet *extra_p = PacketDequeue(&s->s2.slot_pq);
                 tv->tmqh_out(tv, extra_p);
             }
 
@@ -307,23 +292,23 @@ void *TmThreadsSlot2(void *td) {
         }
     }
 
-    if (s2->Slot1ExitPrintStats != NULL) {
-        s2->Slot1ExitPrintStats(tv, s2->slot1_data);
+    if (s->s1.SlotExitPrintStats != NULL) {
+        s->s1.SlotExitPrintStats(tv, s->s1.slot_data);
     }
 
-    if (s2->Slot1Deinit != NULL) {
-        r = s2->Slot1Deinit(tv, s2->slot1_data);
+    if (s->s1.SlotDeinit != NULL) {
+        r = s->s1.SlotDeinit(tv, s->s1.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
 
-    if (s2->Slot2ExitPrintStats != NULL) {
-        s2->Slot2ExitPrintStats(tv, s2->slot2_data);
+    if (s->s2.SlotExitPrintStats != NULL) {
+        s->s2.SlotExitPrintStats(tv, s->s2.slot_data);
     }
 
-    if (s2->Slot2Deinit != NULL) {
-        r = s2->Slot2Deinit(tv, s2->slot2_data);
+    if (s->s2.SlotDeinit != NULL) {
+        r = s->s2.SlotDeinit(tv, s->s2.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
@@ -335,27 +320,27 @@ void *TmThreadsSlot2(void *td) {
 
 void *TmThreadsSlot3(void *td) {
     ThreadVars *tv = (ThreadVars *)td;
-    Tm3Slot *s3 = (Tm3Slot *)tv->tm_slots;
+    Tm3Slot *s = (Tm3Slot *)tv->tm_slots;
     Packet *p = NULL;
     char run = 1;
     int r = 0;
 
     //printf("TmThreadsSlot3: %s starting\n", tv->name);
 
-    if (s3->Slot1Init != NULL) {
-        r = s3->Slot1Init(tv, &s3->slot1_data);
+    if (s->s1.SlotInit != NULL) {
+        r = s->s1.SlotInit(tv, &s->s1.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
-    if (s3->Slot2Init != NULL) {
-        r = s3->Slot2Init(tv, &s3->slot2_data);
+    if (s->s2.SlotInit != NULL) {
+        r = s->s2.SlotInit(tv, &s->s2.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
-    if (s3->Slot3Init != NULL) {
-        r = s3->Slot3Init(tv, &s3->slot3_data);
+    if (s->s3.SlotInit != NULL) {
+        r = s->s3.SlotInit(tv, &s->s3.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
@@ -369,20 +354,20 @@ void *TmThreadsSlot3(void *td) {
             //printf("%s: TmThreadsSlot1: p == NULL\n", tv->name);
         } else {
             /* slot 1 */
-            r = s3->Slot1Func(tv, p, s3->slot1_data, &s3->slot1_pq);
-            while (s3->slot1_pq.len > 0) {
+            r = s->s1.SlotFunc(tv, p, s->s1.slot_data, &s->s1.slot_pq);
+            while (s->s1.slot_pq.len > 0) {
                 /* handle new packets from this func */
-                Packet *extra_p = PacketDequeue(&s3->slot1_pq);
+                Packet *extra_p = PacketDequeue(&s->s1.slot_pq);
 
-                r = s3->Slot2Func(tv, extra_p, s3->slot2_data, &s3->slot2_pq);
-                while (s3->slot2_pq.len > 0) {
+                r = s->s2.SlotFunc(tv, extra_p, s->s2.slot_data, &s->s2.slot_pq);
+                while (s->s2.slot_pq.len > 0) {
                     /* handle new packets from this func */
-                    Packet *extra_p2 = PacketDequeue(&s3->slot2_pq);
+                    Packet *extra_p2 = PacketDequeue(&s->s2.slot_pq);
 
-                    r = s3->Slot3Func(tv, extra_p2, s3->slot3_data, &s3->slot3_pq);
-                    while (s3->slot3_pq.len > 0) {
+                    r = s->s3.SlotFunc(tv, extra_p2, s->s3.slot_data, &s->s3.slot_pq);
+                    while (s->s3.slot_pq.len > 0) {
                         /* handle new packets from this func */
-                        Packet *extra_p3 = PacketDequeue(&s3->slot3_pq);
+                        Packet *extra_p3 = PacketDequeue(&s->s3.slot_pq);
                         tv->tmqh_out(tv, extra_p3);
                     }
                     tv->tmqh_out(tv, extra_p2);
@@ -391,25 +376,25 @@ void *TmThreadsSlot3(void *td) {
             }
 
             /* slot 2 */
-            r = s3->Slot2Func(tv, p, s3->slot2_data, &s3->slot2_pq);
-            while (s3->slot2_pq.len > 0) {
+            r = s->s2.SlotFunc(tv, p, s->s2.slot_data, &s->s2.slot_pq);
+            while (s->s2.slot_pq.len > 0) {
                 /* handle new packets from this func */
-                Packet *extra_p = PacketDequeue(&s3->slot2_pq);
+                Packet *extra_p = PacketDequeue(&s->s2.slot_pq);
 
-                r = s3->Slot3Func(tv, extra_p, s3->slot3_data, &s3->slot3_pq);
-                while (s3->slot3_pq.len > 0) {
+                r = s->s3.SlotFunc(tv, extra_p, s->s3.slot_data, &s->s3.slot_pq);
+                while (s->s3.slot_pq.len > 0) {
                     /* handle new packets from this func */
-                    Packet *extra_p2 = PacketDequeue(&s3->slot3_pq);
+                    Packet *extra_p2 = PacketDequeue(&s->s3.slot_pq);
                     tv->tmqh_out(tv, extra_p2);
                 }
                 tv->tmqh_out(tv, extra_p);
             }
 
             /* slot 3 */
-            r = s3->Slot3Func(tv, p, s3->slot3_data, &s3->slot3_pq);
-            while (s3->slot3_pq.len > 0) {
+            r = s->s3.SlotFunc(tv, p, s->s3.slot_data, &s->s3.slot_pq);
+            while (s->s3.slot_pq.len > 0) {
                 /* handle new packets from this func */
-                Packet *extra_p = PacketDequeue(&s3->slot3_pq);
+                Packet *extra_p = PacketDequeue(&s->s3.slot_pq);
                 tv->tmqh_out(tv, extra_p);
             }
 
@@ -426,34 +411,34 @@ void *TmThreadsSlot3(void *td) {
         }
     }
 
-    if (s3->Slot1ExitPrintStats != NULL) {
-        s3->Slot1ExitPrintStats(tv, s3->slot1_data);
+    if (s->s1.SlotExitPrintStats != NULL) {
+        s->s1.SlotExitPrintStats(tv, s->s1.slot_data);
     }
 
-    if (s3->Slot1Deinit != NULL) {
-        r = s3->Slot1Deinit(tv, s3->slot1_data);
+    if (s->s1.SlotDeinit != NULL) {
+        r = s->s1.SlotDeinit(tv, s->s1.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
 
-    if (s3->Slot2ExitPrintStats != NULL) {
-        s3->Slot2ExitPrintStats(tv, s3->slot2_data);
+    if (s->s2.SlotExitPrintStats != NULL) {
+        s->s2.SlotExitPrintStats(tv, s->s2.slot_data);
     }
 
-    if (s3->Slot2Deinit != NULL) {
-        r = s3->Slot2Deinit(tv, s3->slot2_data);
+    if (s->s2.SlotDeinit != NULL) {
+        r = s->s2.SlotDeinit(tv, s->s2.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
     }
 
-    if (s3->Slot3ExitPrintStats != NULL) {
-        s3->Slot3ExitPrintStats(tv, s3->slot3_data);
+    if (s->s3.SlotExitPrintStats != NULL) {
+        s->s3.SlotExitPrintStats(tv, s->s3.slot_data);
     }
 
-    if (s3->Slot3Deinit != NULL) {
-        r = s3->Slot3Deinit(tv, s3->slot3_data);
+    if (s->s3.SlotDeinit != NULL) {
+        r = s->s3.SlotDeinit(tv, s->s3.slot_data);
         if (r != 0) {
             pthread_exit((void *) -1);
         }
@@ -463,6 +448,88 @@ void *TmThreadsSlot3(void *td) {
     pthread_exit((void *) 0);
 }
 
+/* separate run function so we can call it recursively */
+static inline int TmThreadsSlotVarRun (ThreadVars *tv, Packet *p, TmSlot *slot) {
+    int r = 0;
+    TmSlot *s = NULL;
+
+    for (s = slot; s != NULL; s = s->slot_next) {
+        r = s->SlotFunc(tv, p, s->slot_data, &s->slot_pq);
+        /* XXX handle error */
+
+        /* handle new packets */
+        while (s->slot_pq.len > 0) {
+            Packet *extra_p = PacketDequeue(&s->slot_pq);
+
+            /* see if we need to process the packet */
+            if (s->slot_next != NULL) {
+                TmThreadsSlotVarRun(tv, extra_p, s->slot_next);
+                /* XXX handle error */
+            }
+            tv->tmqh_out(tv, extra_p);
+        }
+    }
+
+    return 0;
+}
+
+void *TmThreadsSlotVar(void *td) {
+    ThreadVars *tv = (ThreadVars *)td;
+    TmVarSlot *s = (TmVarSlot *)tv->tm_slots;
+    Packet *p = NULL;
+    char run = 1;
+    int r = 0;
+    TmSlot *slot = NULL;
+
+    //printf("TmThreadsSlot1: %s starting\n", tv->name);
+
+    for (slot = s->s; slot != NULL; slot = slot->slot_next) {
+        if (slot->SlotInit != NULL) {
+            r = slot->SlotInit(tv, &slot->slot_data);
+            if (r != 0) {
+                pthread_exit((void *) -1);
+            }
+        }
+        memset(&slot->slot_pq, 0, sizeof(PacketQueue));
+    }
+
+    while(run) {
+        /* input a packet */
+        p = tv->tmqh_in(tv);
+
+        if (p == NULL) {
+            //printf("%s: TmThreadsSlot1: p == NULL\n", tv->name);
+        } else {
+            r = TmThreadsSlotVarRun(tv, p, s->s);
+            /* XXX handle error */
+
+            /* output the packet */
+            tv->tmqh_out(tv, p);
+        }
+
+        if (tv->flags & THV_KILL) {
+            //printf("%s: TmThreadsSlot1: KILL is set\n", tv->name);
+            run = 0;
+        }
+    }
+
+    for (slot = s->s; slot != NULL; slot = slot->slot_next) {
+        if (slot->SlotExitPrintStats != NULL) {
+            slot->SlotExitPrintStats(tv, slot->slot_data);
+        }
+
+        if (slot->SlotDeinit != NULL) {
+            r = slot->SlotDeinit(tv, slot->slot_data);
+            if (r != 0) {
+                pthread_exit((void *) -1);
+            }
+        }
+    }
+
+    //printf("TmThreadsSlot1: %s ending\n", tv->name);
+    pthread_exit((void *) 0);
+}
+
 int TmThreadSetSlots(ThreadVars *tv, char *name) {
     u_int16_t size = 0;
 
@@ -484,6 +551,9 @@ int TmThreadSetSlots(ThreadVars *tv, char *name) {
     } else if (strcmp(name, "3slot") == 0) {
         size = sizeof(Tm3Slot);
         tv->tm_func = TmThreadsSlot3;
+    } else if (strcmp(name, "varslot") == 0) {
+        size = sizeof(TmVarSlot);
+        tv->tm_func = TmThreadsSlotVar;
     }
 
     tv->tm_slots = malloc(size);
@@ -498,79 +568,108 @@ error:
 void Tm1SlotSetFunc(ThreadVars *tv, TmModule *tm) {
     Tm1Slot *s1 = (Tm1Slot *)tv->tm_slots;
 
-    if (s1->Slot1Func != NULL)
+    if (s1->s.SlotFunc != NULL)
         printf("Warning: slot 1 is already set tp %p, "
-               "overwriting with %p\n", s1->Slot1Func, tm->Func);
+               "overwriting with %p\n", s1->s.SlotFunc, tm->Func);
 
-    s1->Slot1Init = tm->Init;
-    s1->Slot1Func = tm->Func;
-    s1->Slot1ExitPrintStats = tm->ExitPrintStats;
-    s1->Slot1Deinit = tm->Deinit;
+    s1->s.SlotInit = tm->Init;
+    s1->s.SlotFunc = tm->Func;
+    s1->s.SlotExitPrintStats = tm->ExitPrintStats;
+    s1->s.SlotDeinit = tm->Deinit;
 }
 
 void Tm2SlotSetFunc1(ThreadVars *tv, TmModule *tm) {
-    Tm2Slot *s2 = (Tm2Slot *)tv->tm_slots;
+    Tm2Slot *s = (Tm2Slot *)tv->tm_slots;
 
-    if (s2->Slot1Func != NULL)
+    if (s->s1.SlotFunc != NULL)
         printf("Warning: slot 1 is already set tp %p, "
-               "overwriting with %p\n", s2->Slot1Func, tm->Func);
+               "overwriting with %p\n", s->s1.SlotFunc, tm->Func);
 
-    s2->Slot1Init = tm->Init;
-    s2->Slot1Func = tm->Func;
-    s2->Slot1ExitPrintStats = tm->ExitPrintStats;
-    s2->Slot1Deinit = tm->Deinit;
+    s->s1.SlotInit = tm->Init;
+    s->s1.SlotFunc = tm->Func;
+    s->s1.SlotExitPrintStats = tm->ExitPrintStats;
+    s->s1.SlotDeinit = tm->Deinit;
 }
 
 void Tm2SlotSetFunc2(ThreadVars *tv, TmModule *tm) {
-    Tm2Slot *s2 = (Tm2Slot *)tv->tm_slots;
+    Tm2Slot *s = (Tm2Slot *)tv->tm_slots;
 
-    if (s2->Slot2Func != NULL)
+    if (s->s2.SlotFunc != NULL)
         printf("Warning: slot 2 is already set tp %p, "
-               "overwriting with %p\n", s2->Slot2Func, tm->Func);
+               "overwriting with %p\n", s->s2.SlotFunc, tm->Func);
 
-    s2->Slot2Init = tm->Init;
-    s2->Slot2Func = tm->Func;
-    s2->Slot2ExitPrintStats = tm->ExitPrintStats;
-    s2->Slot2Deinit = tm->Deinit;
+    s->s2.SlotInit = tm->Init;
+    s->s2.SlotFunc = tm->Func;
+    s->s2.SlotExitPrintStats = tm->ExitPrintStats;
+    s->s2.SlotDeinit = tm->Deinit;
 }
 
 void Tm3SlotSetFunc1(ThreadVars *tv, TmModule *tm) {
-    Tm3Slot *s3 = (Tm3Slot *)tv->tm_slots;
+    Tm3Slot *s = (Tm3Slot *)tv->tm_slots;
 
-    if (s3->Slot1Func != NULL)
+    if (s->s1.SlotFunc != NULL)
         printf("Warning: slot 1 is already set tp %p, "
-               "overwriting with %p\n", s3->Slot1Func, tm->Func);
+               "overwriting with %p\n", s->s1.SlotFunc, tm->Func);
 
-    s3->Slot1Init = tm->Init;
-    s3->Slot1Func = tm->Func;
-    s3->Slot1ExitPrintStats = tm->ExitPrintStats;
-    s3->Slot1Deinit = tm->Deinit;
+    s->s1.SlotInit = tm->Init;
+    s->s1.SlotFunc = tm->Func;
+    s->s1.SlotExitPrintStats = tm->ExitPrintStats;
+    s->s1.SlotDeinit = tm->Deinit;
 }
 
 void Tm3SlotSetFunc2(ThreadVars *tv, TmModule *tm) {
-    Tm3Slot *s3 = (Tm3Slot *)tv->tm_slots;
+    Tm3Slot *s = (Tm3Slot *)tv->tm_slots;
 
-    if (s3->Slot2Func != NULL)
+    if (s->s2.SlotFunc != NULL)
         printf("Warning: slot 2 is already set tp %p, "
-               "overwriting with %p\n", s3->Slot2Func, tm->Func);
+               "overwriting with %p\n", s->s2.SlotFunc, tm->Func);
 
-    s3->Slot2Init = tm->Init;
-    s3->Slot2Func = tm->Func;
-    s3->Slot2ExitPrintStats = tm->ExitPrintStats;
-    s3->Slot2Deinit = tm->Deinit;
+    s->s2.SlotInit = tm->Init;
+    s->s2.SlotFunc = tm->Func;
+    s->s2.SlotExitPrintStats = tm->ExitPrintStats;
+    s->s2.SlotDeinit = tm->Deinit;
 }
 
 void Tm3SlotSetFunc3(ThreadVars *tv, TmModule *tm) {
-    Tm3Slot *s3 = (Tm3Slot *)tv->tm_slots;
+    Tm3Slot *s = (Tm3Slot *)tv->tm_slots;
 
-    if (s3->Slot2Func != NULL)
+    if (s->s3.SlotFunc != NULL)
         printf("Warning: slot 3 is already set tp %p, "
-               "overwriting with %p\n", s3->Slot2Func, tm->Func);
+               "overwriting with %p\n", s->s3.SlotFunc, tm->Func);
+
+    s->s3.SlotInit = tm->Init;
+    s->s3.SlotFunc = tm->Func;
+    s->s3.SlotExitPrintStats = tm->ExitPrintStats;
+    s->s3.SlotDeinit = tm->Deinit;
+}
+
+void TmVarSlotSetFuncAppend(ThreadVars *tv, TmModule *tm) {
+    TmVarSlot *s = (TmVarSlot *)tv->tm_slots;
+    TmSlot *slot = malloc(sizeof(TmSlot));
+    if (slot == NULL) 
+        return;
+
+    memset(slot, 0, sizeof(TmSlot));
+
+    slot->SlotInit = tm->Init;
+    slot->SlotFunc = tm->Func;
+    slot->SlotExitPrintStats = tm->ExitPrintStats;
+    slot->SlotDeinit = tm->Deinit;
 
-    s3->Slot3Init = tm->Init;
-    s3->Slot3Func = tm->Func;
-    s3->Slot3ExitPrintStats = tm->ExitPrintStats;
-    s3->Slot3Deinit = tm->Deinit;
+    if (s->s == NULL) {
+        s->s = slot;
+    } else {
+        TmSlot *a = s->s, *b = NULL;
+
+        /* get the last slot */
+        for ( ; a != NULL; a = a->slot_next) {
+             b = a;
+        }
+        /* append the new slot */
+        if (b != NULL) {
+            b->slot_next = slot;
+        }
+    }
 }
 
 ThreadVars *TmThreadCreate(char *name, char *inq_name, char *inqh_name, char *outq_name, char *outqh_name, char *slots) {
@@ -668,22 +767,26 @@ void TmThreadKillThreads(void) {
 
     while (t) {
         t->flags |= THV_KILL;
-        //printf("TmThreadKillThreads: told thread %s to stop\n", t->name);
+        printf("TmThreadKillThreads: told thread %s to stop\n", t->name);
 
         if (t->inq != NULL) {
             int i;
 
-            //printf("TmThreadKillThreads: t->inq->usecnt %u\n", t->inq->usecnt);
+            printf("TmThreadKillThreads: t->inq->usecnt %u\n", t->inq->usecnt);
 
             /* signal the queue for the number of users */
             for (i = 0; i < t->inq->usecnt; i++)
                 pthread_cond_signal(&trans_q[t->inq->id].cond_q);
+            /* to be sure, signal more */
+            for (i = 0; i < 1000; i++)
+                pthread_cond_signal(&trans_q[t->inq->id].cond_q);
+
+            printf("TmThreadKillThreads: signalled t->inq->id %u\n", t->inq->id);
         }
-        //printf("TmThreadKillThreads: signalled t->inq->id %u\n", t->inq->id);
 
         /* join it */
         pthread_join(t->t, NULL);
-        //printf("TmThreadKillThreads: thread %s stopped\n", t->name);
+        printf("TmThreadKillThreads: thread %s stopped\n", t->name);
 
         t = t->next;
     }

src/tm-threads.h

@@ -4,6 +4,10 @@
 void Tm1SlotSetFunc(ThreadVars *, TmModule *);
 void Tm2SlotSetFunc1(ThreadVars *, TmModule *);
 void Tm2SlotSetFunc2(ThreadVars *, TmModule *);
+void Tm3SlotSetFunc1(ThreadVars *, TmModule *);
+void Tm3SlotSetFunc2(ThreadVars *, TmModule *);
+void Tm3SlotSetFunc3(ThreadVars *, TmModule *);
+void TmVarSlotSetFuncAppend(ThreadVars *, TmModule *);
 ThreadVars *TmThreadCreate(char *name, char *inq_name, char *inqh_name, char *outq_name, char *outqh_name, char *slots);
 int TmThreadSpawn(ThreadVars *);
 void TmThreadKillThreads(void);

src/tmqh-packetpool.c

@@ -21,11 +21,18 @@ Packet *TmqhInputPacketpool(ThreadVars *t)
     /* XXX */
     Packet *p = SetupPkt();
 
+/*
+ * Disabled because it can enter a 'wait' state, while
+ * keeping the nfq queue locked thus making it impossble
+ * to free packets, the exact condition we are waiting
+ * for. VJ 09-01-16
+ *
     mutex_lock(&mutex_pending);
     if (pending > MAX_PENDING) {
         pthread_cond_wait(&cond_pending, &mutex_pending);
     }
     mutex_unlock(&mutex_pending);
+*/
     return p;
 }
 
@@ -35,22 +42,22 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
     char proot = 0;
 
     if (IS_TUNNEL_PKT(p)) {
-        //printf("TmqhOutputPacketpool: tunnel packet: %p %s\n", p,p->root ? "upper layer":"root");
+        printf("TmqhOutputPacketpool: tunnel packet: %p %s\n", p,p->root ? "upper layer":"root");
 
         /* get a lock */
         pthread_mutex_t *m = p->root ? &p->root->mutex_rtv_cnt : &p->mutex_rtv_cnt;
         mutex_lock(m);
 
         if (IS_TUNNEL_ROOT_PKT(p)) {
-            //printf("TmqhOutputPacketpool: IS_TUNNEL_ROOT_PKT\n");
+            printf("TmqhOutputPacketpool: IS_TUNNEL_ROOT_PKT\n");
             if (TUNNEL_PKT_TPR(p) == 0) {
-                //printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) == 0\n");
+                printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) == 0\n");
                 /* if this packet is the root and there are no
                  * more tunnel packets, enqueue it */
 
                 /* fall through */
             } else {
-                //printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) > 0\n");
+                printf("TmqhOutputPacketpool: TUNNEL_PKT_TPR(p) > 0\n");
                 /* if this is the root and there are more tunnel
                  * packets, don't add this. It's still referenced
                  * by the tunnel packets, and we will enqueue it
@@ -60,7 +67,7 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
                 return;
             }
         } else {
-            //printf("TmqhOutputPacketpool: NOT IS_TUNNEL_ROOT_PKT\n");
+            printf("TmqhOutputPacketpool: NOT IS_TUNNEL_ROOT_PKT\n");
             if (p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1) {
                 //printf("TmqhOutputPacketpool: p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1\n");
                 /* the root is ready and we are the last tunnel packet,
@@ -68,19 +75,19 @@ void TmqhOutputPacketpool(ThreadVars *t, Packet *p)
                 TUNNEL_DECR_PKT_TPR_NOLOCK(p);
 
                 /* handle the root */
-                //printf("TmqhOutputPacketpool: calling PacketEnqueue for root pkt\n");
+                printf("TmqhOutputPacketpool: calling PacketEnqueue for root pkt\n");
                 proot = 1;
 
                 /* fall through */
             } else {
-                //printf("TmqhOutputPacketpool: NOT p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1 (%u)\n", TUNNEL_PKT_TPR(p));
+                printf("TmqhOutputPacketpool: NOT p->root->tunnel_verdicted == 1 && TUNNEL_PKT_TPR(p) == 1 (%u)\n", TUNNEL_PKT_TPR(p));
                 TUNNEL_DECR_PKT_TPR_NOLOCK(p);
 
                  /* fall through */
             }
         }
         mutex_unlock(m);
-        //printf("TmqhOutputPacketpool: tunnel stuff done, move on\n");
+        printf("TmqhOutputPacketpool: tunnel stuff done, move on\n");
     }
 
     mutex_lock(&q->mutex_q);

src/util-mpm-trie.c

@@ -13,7 +13,7 @@
 #include "util-mpm-trie.h"
 
 #include "util-unittest.h"
-
+#if 0
 /*
  * TODO/IDEAS/XXX
  * - we know if we are interested in just the first match (simple content of
@@ -25,9 +25,9 @@
 /* prototypes to be exported */
 void TrieInitCtx(MpmCtx *mpm_ctx);
 void TrieThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int32_t);
-int TrieAddPattern(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t id);
-int TrieAddPatternNocase(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t id);
-u_int32_t TrieSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf, u_int16_t buflen);
+int TrieAddPattern(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t pid, u_int32_t sid);
+int TrieAddPatternNocase(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t pid, u_int32_t sid);
+u_int32_t TrieSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *, u_int8_t *buf, u_int16_t buflen);
 void TriePrintInfo(MpmCtx *mpm_ctx);
 void TriePrintThreadInfo(MpmThreadCtx *mpm_ctx);
 void TrieRegisterTests(void);
@@ -44,6 +44,7 @@ void MpmTrieRegister (void) {
     mpm_table[MPM_TRIE].AddPattern = TrieAddPattern;
     mpm_table[MPM_TRIE].AddPatternNocase = TrieAddPatternNocase;
     mpm_table[MPM_TRIE].Prepare = NULL;
+    mpm_table[MPM_TRIE].Scan = TrieSearch;
     mpm_table[MPM_TRIE].Search = TrieSearch;
     mpm_table[MPM_TRIE].Cleanup = MpmMatchCleanup;
     mpm_table[MPM_TRIE].PrintCtx = TriePrintInfo;
@@ -68,7 +69,7 @@ void MpmTrieRegister (void) {
 /* append an endmatch to a character node
  *
  * Only used in the initialization phase */
-static void TrieEndMatchAppend(MpmCtx *mpm_ctx, TrieCharacter *c, u_int32_t id)
+static void TrieEndMatchAppend(MpmCtx *mpm_ctx, TrieCharacter *c, u_int32_t pid, u_int32_t sid)
 {
     MpmEndMatch *em = MpmAllocEndMatch(mpm_ctx);
     if (em == NULL) {
@@ -76,7 +77,8 @@ static void TrieEndMatchAppend(MpmCtx *mpm_ctx, TrieCharacter *c, u_int32_t id)
         return;
     }
 
-    em->id = id;
+    em->id = pid;
+    em->sig_id = sid;
 
     if (c->em == NULL) {
         c->em = em;
@@ -128,8 +130,9 @@ static void TrieFreeCharacter (MpmCtx *mpm_ctx, TrieCharacter *c) {
 /* add a keyword to the search tree
  *
  * Only used in the initialization phase */
-static int DoTrieAddPattern(MpmCtx *mpm_ctx, TrieCharacter *c,
-            u_int8_t *key, u_int16_t keylen, u_int32_t id, char nocase)
+static int DoTrieAddPattern(MpmCtx *mpm_ctx, TrieCharacter *c, u_int8_t *key,
+                            u_int16_t keylen, u_int32_t pid, u_int32_t sid,
+                            char nocase)
 {
 #ifdef DEBUG
     /* DEBUG */
@@ -145,12 +148,12 @@ static int DoTrieAddPattern(MpmCtx *mpm_ctx, TrieCharacter *c,
     }
 #endif
 
-    if (keylen > mpm_ctx->maxlen)
-        mpm_ctx->maxlen = keylen;
-    if (mpm_ctx->minlen == 0)
-        mpm_ctx->minlen = keylen;
-    if (keylen < mpm_ctx->minlen)
-        mpm_ctx->minlen = keylen;
+    if (keylen > mpm_ctx->search_maxlen)
+        mpm_ctx->search_maxlen = keylen;
+    if (mpm_ctx->search_minlen == 0)
+        mpm_ctx->search_minlen = keylen;
+    if (keylen < mpm_ctx->search_minlen)
+        mpm_ctx->search_minlen = keylen;
 
     u_int16_t i;
     u_int8_t ch;
@@ -181,32 +184,32 @@ static int DoTrieAddPattern(MpmCtx *mpm_ctx, TrieCharacter *c,
         /* set the endmatch */
         if (i == keylen - 1) {
             // printf("TrieAddPattern: last char of keyword, now append an EndMatch\n");
-            TrieEndMatchAppend(mpm_ctx, c->nc[ch], id);
+            TrieEndMatchAppend(mpm_ctx, c->nc[ch], pid, sid);
         }
 
         c = c->nc[ch];
     }
 
-    if (id > mpm_ctx->max_pattern_id)
-        mpm_ctx->max_pattern_id = id;
+    if (pid > mpm_ctx->max_pattern_id)
+        mpm_ctx->max_pattern_id = pid;
 
     return 0;
 }
 
-int TrieAddPattern(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t id) {
+int TrieAddPattern(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t pid, u_int32_t sid) {
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx->ctx;
 
     trie_ctx->keywords++;
 
-    return(DoTrieAddPattern(mpm_ctx, &trie_ctx->root, key, keylen, id, 0 /* no nocase */));
+    return(DoTrieAddPattern(mpm_ctx, &trie_ctx->root, key, keylen, pid, sid, 0 /* no nocase */));
 }
 
-int TrieAddPatternNocase(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t id) {
+int TrieAddPatternNocase(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t pid, u_int32_t sid) {
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx->ctx;
 
     trie_ctx->nocase_keywords++;
 
-    return(DoTrieAddPattern(mpm_ctx, &trie_ctx->nocase_root, key, keylen, id, 1 /* nocase */));
+    return(DoTrieAddPattern(mpm_ctx, &trie_ctx->nocase_root, key, keylen, pid, sid, 1 /* nocase */));
 }
 
 static void TrieDoPrint(TrieCharacter *c, int depth)
@@ -313,8 +316,8 @@ TrieSpareDequeue (MpmThreadCtx *mpm_thread_ctx, TriePartialMatchList *q)
 
 static inline u_int32_t
 TrieSearchCharNocase(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
-                     TrieThreadCtx *trie_thread_ctx, TrieCharacter *c,
-                     TriePartialMatch **qpm, u_int8_t ch)
+                     TrieThreadCtx *trie_thread_ctx, PatternMatcherQueue *pmq,
+                     TrieCharacter *c, TriePartialMatch **qpm, u_int8_t ch)
 {
     TriePartialMatch *tmppm, *tpm;
 
@@ -339,7 +342,7 @@ TrieSearchCharNocase(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
             MpmEndMatch *em = tmppm->c->nc[ch]->em;
             if (em != NULL) {
                 for (; em != NULL; em = em->next) {
-                    MpmMatchAppend(mpm_thread_ctx, em, &mpm_thread_ctx->match[em->id],
+                    MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],
                                    trie_thread_ctx->buf - trie_thread_ctx->bufmin);
 
                     //printf("NOCASE MATCH! id %u, matched at offset %u, char %c\n", em->id,
@@ -380,7 +383,7 @@ TrieSearchCharNocase(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
             #endif /* MPM_DBG_PERF */
 
             for (; em != NULL; em = em->next) {
-                MpmMatchAppend(mpm_thread_ctx, em, &mpm_thread_ctx->match[em->id],
+                MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],
                                trie_thread_ctx->buf - trie_thread_ctx->bufmin);
                 //printf("NOCASE MATCH @search root! id %u, matched at offset %u, char %c\n", em->id,
                 //    trie_thread_ctx->buf - mpm_thread_ctx->bufmin, *mpm_thread_ctx->buf);
@@ -411,7 +414,7 @@ TrieSearchCharNocase(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                     #endif /* MPM_DBG_PERF */
 
                     for (; nem != NULL; nem = nem->next) {
-                         MpmMatchAppend(mpm_thread_ctx, nem, &mpm_thread_ctx->match[nem->id],
+                         MpmMatchAppend(mpm_thread_ctx, pmq, nem, &mpm_thread_ctx->match[nem->id],
                                         trie_thread_ctx->buf - trie_thread_ctx->bufmin);
                          //printf("MATCH! id %u, matched at offset %u\n", nem->id,
                          //    trie_thread_ctx->buf - mpm_thread_ctx->bufmin);
@@ -454,8 +457,8 @@ TrieSearchCharNocase(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
 
 static inline u_int32_t
 TrieSearchChar(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
-               TrieThreadCtx *trie_thread_ctx, TrieCharacter *c,
-               TriePartialMatch **qpm, u_int8_t ch)
+               TrieThreadCtx *trie_thread_ctx, PatternMatcherQueue *pmq,
+               TrieCharacter *c, TriePartialMatch **qpm, u_int8_t ch)
 {
     TriePartialMatch *tmppm, *tpm;
 
@@ -480,7 +483,7 @@ TrieSearchChar(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
             MpmEndMatch *em = tmppm->c->nc[ch]->em;
             if (em != NULL) {
                 for (; em != NULL; em = em->next) {
-                    MpmMatchAppend(mpm_thread_ctx, em, &mpm_thread_ctx->match[em->id],
+                    MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],
                                    trie_thread_ctx->buf - trie_thread_ctx->bufmin);
 
                     //printf("MATCH! id %u, matched at offset %u, char %c\n", em->id,
@@ -521,7 +524,7 @@ TrieSearchChar(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
             #endif /* DBG_MPM_PERF */
 
             for (; em != NULL; em = em->next) {
-                MpmMatchAppend(mpm_thread_ctx, em, &mpm_thread_ctx->match[em->id],
+                MpmMatchAppend(mpm_thread_ctx, pmq, em, &mpm_thread_ctx->match[em->id],
                                trie_thread_ctx->buf - trie_thread_ctx->bufmin);
 //                printf("MATCH! @search root id %u, matched at offset %u, char %c\n", em->id,
 //                    trie_thread_ctx->buf - mpm_thread_ctx->bufmin, *mpm_thread_ctx->buf);
@@ -557,7 +560,7 @@ TrieSearchChar(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                     #endif /* MPM_DBG_PERF */
 
                     for (; nem != NULL; nem = nem->next) {
-                         MpmMatchAppend(mpm_thread_ctx, nem, &mpm_thread_ctx->match[nem->id],
+                         MpmMatchAppend(mpm_thread_ctx, pmq, nem, &mpm_thread_ctx->match[nem->id],
                                         trie_thread_ctx->buf - trie_thread_ctx->bufmin);
 //                         printf("MATCH! id %u, matched at offset %u\n", nem->id,
 //                             trie_thread_ctx->buf - mpm_thread_ctx->bufmin);
@@ -607,7 +610,7 @@ TrieSearchChar(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
  *
  */
 u_int32_t
-TrieSearchOffsetDepth(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
+TrieSearchOffsetDepth(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq,
            u_int8_t *buf, u_int16_t buflen, u_int16_t offset, u_int16_t depth)
 {
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx->ctx;
@@ -632,10 +635,10 @@ TrieSearchOffsetDepth(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
      * to prevent having to go through the buf twice */
     for ( ; trie_thread_ctx->buf != trie_thread_ctx->bufmax;
             trie_thread_ctx->buf++) {
-        TrieSearchChar(mpm_ctx, mpm_thread_ctx, trie_thread_ctx,
+        TrieSearchChar(mpm_ctx, mpm_thread_ctx, trie_thread_ctx, pmq,
                    &trie_ctx->root, &trie_thread_ctx->pmqueue,
                    *trie_thread_ctx->buf);
-        TrieSearchCharNocase(mpm_ctx, mpm_thread_ctx, trie_thread_ctx,
+        TrieSearchCharNocase(mpm_ctx, mpm_thread_ctx, trie_thread_ctx, pmq,
                    &trie_ctx->nocase_root, &trie_thread_ctx->nocase_pmqueue,
                    trie_tolower(*trie_thread_ctx->buf));
     }
@@ -667,7 +670,7 @@ TrieSearchOffsetDepth(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
  *
  */
 u_int32_t
-TrieSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
+TrieSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, PatternMatcherQueue *pmq,
            u_int8_t *buf, u_int16_t buflen)
 {
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx->ctx;
@@ -689,10 +692,10 @@ TrieSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
      * to prevent having to go through the buf twice */
     for ( ; trie_thread_ctx->buf != trie_thread_ctx->bufmax;
             trie_thread_ctx->buf++) {
-        TrieSearchChar(mpm_ctx, mpm_thread_ctx, trie_thread_ctx,
+        TrieSearchChar(mpm_ctx, mpm_thread_ctx, trie_thread_ctx, pmq,
                    &trie_ctx->root, &trie_thread_ctx->pmqueue,
                    *trie_thread_ctx->buf);
-        TrieSearchCharNocase(mpm_ctx, mpm_thread_ctx, trie_thread_ctx,
+        TrieSearchCharNocase(mpm_ctx, mpm_thread_ctx, trie_thread_ctx, pmq,
                    &trie_ctx->nocase_root, &trie_thread_ctx->nocase_pmqueue,
                    trie_tolower(*trie_thread_ctx->buf));
     }
@@ -749,8 +752,8 @@ void TriePrintInfo(MpmCtx *mpm_ctx) {
     printf("\nMPM Trie stats:\n");
     printf("Patterns:        %u\n", trie_ctx->keywords);
     printf("Patterns Nocase: %u\n", trie_ctx->nocase_keywords);
-    printf(" -shortest len:  %u\n", mpm_ctx->minlen);
-    printf(" -longest len:   %u\n", mpm_ctx->maxlen);
+    printf(" -shortest len:  %u\n", mpm_ctx->search_minlen);
+    printf(" -longest len:   %u\n", mpm_ctx->search_maxlen);
     printf("Characters:      %u\n", trie_ctx->characters);
     printf("EndMatches:      %u\n", mpm_ctx->endmatches);
     printf("Memory blocks:   %u\n", mpm_ctx->memory_cnt);
@@ -918,7 +921,7 @@ int TrieTestInitAddPattern01 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    int ret = TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
+    int ret = TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234, 0);
     if (ret == 0)
         result = 1;
 
@@ -936,7 +939,7 @@ int TrieTestInitAddPattern02 (void) {
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234, 0);
 
     if (trie_ctx->root.nc['a'] != NULL)
         result = 1;
@@ -955,7 +958,7 @@ int TrieTestInitAddPattern03 (void) {
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234, 0);
 
     if (trie_ctx->root.nc['a']->min_matchlen_left == 4)
         result = 1;
@@ -974,7 +977,7 @@ int TrieTestInitAddPattern04 (void) {
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
-    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
+    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234, 0);
 
     if (trie_ctx->nocase_root.nc['a'] != NULL)
         result = 1;
@@ -993,7 +996,7 @@ int TrieTestInitAddPattern05 (void) {
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Abcd", 4, 1234);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Abcd", 4, 1234, 0);
 
     if (trie_ctx->root.nc['A'] != NULL)
         result = 1;
@@ -1012,7 +1015,7 @@ int TrieTestInitAddPattern06 (void) {
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234, 0);
 
     if (trie_ctx->root.nc['a'] != NULL &&
         trie_ctx->root.nc['a']->nc['b'] != NULL &&
@@ -1033,7 +1036,7 @@ int TrieTestInitAddPattern07 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234, 0);
 
     if (mpm_ctx.max_pattern_id == 1234)
         result = 1;
@@ -1050,10 +1053,10 @@ int TrieTestSearch01 (void) {
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcd", 4);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcd", 4);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 1)
@@ -1070,10 +1073,10 @@ int TrieTestSearch02 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abce", 4);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abce", 4);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 0)
@@ -1090,10 +1093,10 @@ int TrieTestSearch03 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 1)
@@ -1110,10 +1113,10 @@ int TrieTestSearch04 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"bcde", 4, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"bcde", 4, 0, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 1)
@@ -1130,10 +1133,10 @@ int TrieTestSearch05 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"efgh", 4, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"efgh", 4, 0, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 1)
@@ -1150,10 +1153,10 @@ int TrieTestSearch06 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"eFgH", 4, 0);
+    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"eFgH", 4, 0, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdEfGh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdEfGh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 1)
@@ -1170,11 +1173,11 @@ int TrieTestSearch07 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"abcd", 4, 0);
-    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"eFgH", 4, 1);
+    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0);
+    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"eFgH", 4, 1, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdEfGh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdEfGh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 2)
@@ -1191,11 +1194,11 @@ int TrieTestSearch08 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcde", 5, 0);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"bcde",  4, 1);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcde", 5, 0, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"bcde",  4, 1, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 2)
@@ -1212,10 +1215,10 @@ int TrieTestSearch09 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"ab", 2, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"ab", 2, 0, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"ab", 2);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"ab", 2);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 1)
@@ -1232,11 +1235,11 @@ int TrieTestSearch10 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"bc", 2, 0);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"gh", 2, 1);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"bc", 2, 0, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"gh", 2, 1, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 2)
@@ -1253,12 +1256,12 @@ int TrieTestSearch11 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"a", 1, 0);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"d", 1, 1);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"h", 1, 2);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"a", 1, 0, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"d", 1, 1, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"h", 1, 2, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 3)
@@ -1275,12 +1278,12 @@ int TrieTestSearch12 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"d", 1, 1);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2);
+    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"d", 1, 1, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 2)
@@ -1297,12 +1300,12 @@ int TrieTestSearch13 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"a", 1, 0);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"h", 1, 2);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"a", 1, 0, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"h", 1, 2, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 3)
@@ -1319,12 +1322,12 @@ int TrieTestSearch14 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2);
+    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
-    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
 
     if (cnt == 2)
@@ -1341,12 +1344,12 @@ int TrieTestSearch15 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"A", 1, 0);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
-    TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
 
     u_int32_t len = mpm_thread_ctx.match[1].len;
 
@@ -1366,12 +1369,12 @@ int TrieTestSearch16 (void) {
     MpmThreadCtx mpm_thread_ctx;
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
-    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1);
-    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2);
+    TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1, 0);
+    TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2, 0);
     TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
-    TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
+    TrieSearch(&mpm_ctx, &mpm_thread_ctx, NULL, (u_int8_t *)"abcdefgh", 8);
 
     u_int32_t len = mpm_thread_ctx.match[0].len;
 
@@ -1418,3 +1421,4 @@ void TrieRegisterTests(void) {
     UtRegisterTest("TrieTestSearch15", TrieTestSearch15, 1);
     UtRegisterTest("TrieTestSearch16", TrieTestSearch16, 1);
 }
+#endif

src/util-mpm-wumanber.h

@@ -5,7 +5,10 @@
 
 #include "util-mpm.h"
 
-#define NOCASE 0x01
+#define WUMANBER_NOCASE 0x01
+#define WUMANBER_SCAN   0x02
+
+//#define WUMANBER_COUNTERS
 
 typedef struct _WmPattern {
     u_int8_t *cs; /* case sensitive */
@@ -28,27 +31,43 @@ typedef struct _WmCtx {
     /* hash used during ctx initialization */
     WmPattern **init_hash;
 
-    u_int16_t shiftlen;
+    u_int16_t scan_shiftlen;
+    u_int16_t search_shiftlen;
+
+    u_int32_t scan_hash_size;
+    WmHashItem **scan_hash;
+    WmHashItem scan_hash1[256];
+    u_int32_t search_hash_size;
+    WmHashItem **search_hash;
+    WmHashItem search_hash1[256];
 
-    u_int32_t hash_size;
-    WmHashItem **hash;
-    WmHashItem hash1[256];
+    /* we store our own multi byte scan ptr here for WmSearch1 */
+    u_int32_t (*MBScan)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
     /* we store our own multi byte search ptr here for WmSearch1 */
-    u_int32_t (*MBSearch)(struct _MpmCtx *, struct _MpmThreadCtx *, u_int8_t *, u_int16_t);
+    u_int32_t (*MBSearch)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
 
     /* pattern arrays */
     WmPattern **parray;
 
     /* only used for multibyte pattern search */
-    u_int16_t *shifttable;
+    u_int16_t *scan_shifttable;
+    u_int16_t *search_shifttable;
 } WmCtx;
 
 typedef struct _WmThreadCtx {
-    u_int32_t stat_shift_null;
-    u_int32_t stat_loop_match;
-    u_int32_t stat_loop_no_match;
-    u_int32_t stat_num_shift;
-    u_int32_t stat_total_shift;
+#ifdef WUMANBER_COUNTERS
+    u_int32_t scan_stat_shift_null;
+    u_int32_t scan_stat_loop_match;
+    u_int32_t scan_stat_loop_no_match;
+    u_int32_t scan_stat_num_shift;
+    u_int32_t scan_stat_total_shift;
+
+    u_int32_t search_stat_shift_null;
+    u_int32_t search_stat_loop_match;
+    u_int32_t search_stat_loop_no_match;
+    u_int32_t search_stat_num_shift;
+    u_int32_t search_stat_total_shift;
+#endif /* WUMANBER_COUNTERS */
 } WmThreadCtx;
 
 void MpmWuManberRegister(void);

src/util-mpm.c

@@ -44,6 +44,7 @@ MpmMatchCleanup(MpmThreadCtx *thread_ctx) {
 
         m = nxt;
     }
+
 }
 
 /* allocate a match
@@ -68,12 +69,29 @@ MpmMatchAlloc(MpmThreadCtx *thread_ctx) {
 /* append a match to a bucket
  *
  * used at search runtime */
-inline void
-MpmMatchAppend(MpmThreadCtx *thread_ctx, MpmEndMatch *em, MpmMatchBucket *mb, u_int16_t offset)
+inline int
+MpmMatchAppend(MpmThreadCtx *thread_ctx, PatternMatcherQueue *pmq, MpmEndMatch *em, MpmMatchBucket *mb, u_int16_t offset, u_int16_t patlen)
 {
+    /* don't bother looking at sigs that didn't match
+     * when we scanned. There's not matching anyway. */
+    if (pmq != NULL && pmq->mode == PMQ_MODE_SEARCH) {
+        if (!(pmq->sig_bitarray[(em->sig_id / 8)] & (1<<(em->sig_id % 8))))
+            return 0;
+    }
+
+    /* XXX is this correct? */
     if (em->flags & MPM_ENDMATCH_SINGLE && mb->len)
-        return;
+        return 0;
+
+    /* check offset */
+    if (offset < em->offset)
+        return 0;
+
+    /* check depth */
+    if (em->depth && (offset+patlen) > em->depth)
+        return 0;
 
+    /* ok all checks passed, now append the match */
     MpmMatch *m;
     /* pull a match from the spare list */
     if (thread_ctx->sparelist != NULL) {
@@ -82,7 +100,7 @@ MpmMatchAppend(MpmThreadCtx *thread_ctx, MpmEndMatch *em, MpmMatchBucket *mb, u_
     } else {
         m = MpmMatchAlloc(thread_ctx);
         if (m == NULL)
-            return;
+            return 0;
     }
 
     m->offset = offset;
@@ -108,6 +126,18 @@ MpmMatchAppend(MpmThreadCtx *thread_ctx, MpmEndMatch *em, MpmMatchBucket *mb, u_
         m->qnext = thread_ctx->qlist;
         thread_ctx->qlist = m;
     }
+
+    if (pmq != NULL) {
+        /* make sure we only append a sig with a matching pattern once,
+         * so we won't inspect it more than once. For this we keep a
+         * bitarray of sig internal id's and flag each sig that matched */
+        if (!(pmq->sig_bitarray[(em->sig_id / 8)] & (1<<(em->sig_id % 8)))) {
+            pmq->sig_bitarray[(em->sig_id / 8)] |= (1<<(em->sig_id % 8));
+            pmq->sig_id_array[pmq->sig_id_array_cnt] = em->sig_id;
+            pmq->sig_id_array_cnt++;
+        }
+    }
+
 #ifdef DEBUG
     printf("MpmMatchAppend: len %u (offset %u)\n", mb->len, m->offset);
 
@@ -117,6 +147,8 @@ MpmMatchAppend(MpmThreadCtx *thread_ctx, MpmEndMatch *em, MpmMatchBucket *mb, u_
         tmp = tmp->qnext;
     }
 #endif
+
+    return 1;
 }
 
 void MpmMatchFree(MpmThreadCtx *ctx, MpmMatch *m) {
@@ -167,24 +199,27 @@ void MpmEndMatchFreeAll(MpmCtx *mpm_ctx, MpmEndMatch *em) {
 void MpmInitCtx (MpmCtx *mpm_ctx, u_int16_t matcher) {
     mpm_table[matcher].InitCtx(mpm_ctx);
 
-    mpm_ctx->InitCtx          = mpm_table[matcher].InitCtx;
-    mpm_ctx->InitThreadCtx    = mpm_table[matcher].InitThreadCtx;
-    mpm_ctx->DestroyCtx       = mpm_table[matcher].DestroyCtx;
-    mpm_ctx->DestroyThreadCtx = mpm_table[matcher].DestroyThreadCtx;
-    mpm_ctx->AddPattern       = mpm_table[matcher].AddPattern;
-    mpm_ctx->AddPatternNocase = mpm_table[matcher].AddPatternNocase;
-    mpm_ctx->Prepare          = mpm_table[matcher].Prepare;
-    mpm_ctx->Search           = mpm_table[matcher].Search;
-    mpm_ctx->PrintCtx         = mpm_table[matcher].PrintCtx;
-    mpm_ctx->PrintThreadCtx   = mpm_table[matcher].PrintThreadCtx;
-    mpm_ctx->Cleanup          = mpm_table[matcher].Cleanup;
+    mpm_ctx->InitCtx              = mpm_table[matcher].InitCtx;
+    mpm_ctx->InitThreadCtx        = mpm_table[matcher].InitThreadCtx;
+    mpm_ctx->DestroyCtx           = mpm_table[matcher].DestroyCtx;
+    mpm_ctx->DestroyThreadCtx     = mpm_table[matcher].DestroyThreadCtx;
+    mpm_ctx->AddScanPattern       = mpm_table[matcher].AddScanPattern;
+    mpm_ctx->AddScanPatternNocase = mpm_table[matcher].AddScanPatternNocase;
+    mpm_ctx->AddPattern           = mpm_table[matcher].AddPattern;
+    mpm_ctx->AddPatternNocase     = mpm_table[matcher].AddPatternNocase;
+    mpm_ctx->Prepare              = mpm_table[matcher].Prepare;
+    mpm_ctx->Scan                 = mpm_table[matcher].Scan;
+    mpm_ctx->Search               = mpm_table[matcher].Search;
+    mpm_ctx->PrintCtx             = mpm_table[matcher].PrintCtx;
+    mpm_ctx->PrintThreadCtx       = mpm_table[matcher].PrintThreadCtx;
+    mpm_ctx->Cleanup              = mpm_table[matcher].Cleanup;
 }
 
 
 void MpmTableSetup(void) {
     memset(mpm_table, 0, sizeof(mpm_table));
 
-    MpmTrieRegister();
+    //MpmTrieRegister();
     MpmWuManberRegister();
 }
 

src/util-mpm.h

@@ -22,6 +22,7 @@ typedef struct _MpmEndMatch {
     u_int16_t offset;
     u_int8_t flags;
     struct _MpmEndMatch *next;
+    u_int32_t sig_id; /* sig callback stuff -- internal id */
 } MpmEndMatch;
 
 typedef struct _MpmMatch {
@@ -55,6 +56,22 @@ typedef struct _MpmThreadCtx {
 
 } MpmThreadCtx;
 
+#define PMQ_MODE_SCAN   0
+#define PMQ_MODE_SEARCH 1
+
+/* helper structure for the detection engine. The Pattern Matcher thread
+ * has this and passes a pointer to it to the pattern matcher. The actual
+ * pattern matcher will fill the structure. */
+typedef struct _PatternMatcherQueue {
+    /* sig callback stuff XXX consider a separate struct for this*/
+    u_int32_t *sig_id_array; /* array with internal sig id's that had a
+                                pattern match. These will be inspected
+                                futher by the detection engine. */
+    u_int32_t sig_id_array_cnt;
+    u_int8_t *sig_bitarray;
+    char mode; /* 0: scan, 1: search */
+} PatternMatcherQueue;
+
 typedef struct _MpmCtx {
     void *ctx;
 
@@ -62,10 +79,13 @@ typedef struct _MpmCtx {
     void (*InitThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *, u_int32_t);
     void (*DestroyCtx)(struct _MpmCtx *);
     void (*DestroyThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *);
-    int  (*AddPattern)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int32_t);
-    int  (*AddPatternNocase)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int32_t);
+    int  (*AddScanPattern)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t);
+    int  (*AddScanPatternNocase)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t);
+    int  (*AddPattern)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t);
+    int  (*AddPatternNocase)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t);
     int  (*Prepare)(struct _MpmCtx *);
-    u_int32_t (*Search)(struct _MpmCtx *, struct _MpmThreadCtx *, u_int8_t *, u_int16_t);
+    u_int32_t (*Scan)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
+    u_int32_t (*Search)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
     void (*Cleanup)(struct _MpmThreadCtx *);
     void (*PrintCtx)(struct _MpmCtx *);
     void (*PrintThreadCtx)(struct _MpmThreadCtx *);
@@ -75,11 +95,14 @@ typedef struct _MpmCtx {
 
     u_int32_t endmatches;
 
+    u_int32_t scan_pattern_cnt;  /* scan patterns */
     u_int32_t pattern_cnt;       /* unique patterns */
     u_int32_t total_pattern_cnt; /* total patterns added */
 
-    u_int16_t minlen;
-    u_int16_t maxlen;
+    u_int16_t scan_minlen;
+    u_int16_t scan_maxlen;
+    u_int16_t search_minlen;
+    u_int16_t search_maxlen;
 
     /* this is used to determine the size of the match
      * loopup table */
@@ -93,10 +116,13 @@ typedef struct MpmTableElmt {
     void (*InitThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *, u_int32_t);
     void (*DestroyCtx)(struct _MpmCtx *);
     void (*DestroyThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *);
-    int  (*AddPattern)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int32_t);
-    int  (*AddPatternNocase)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int32_t);
+    int  (*AddScanPattern)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t);
+    int  (*AddScanPatternNocase)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t);
+    int  (*AddPattern)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t);
+    int  (*AddPatternNocase)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int16_t, u_int16_t, u_int32_t, u_int32_t);
     int  (*Prepare)(struct _MpmCtx *);
-    u_int32_t (*Search)(struct _MpmCtx *, struct _MpmThreadCtx *, u_int8_t *, u_int16_t);
+    u_int32_t (*Scan)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
+    u_int32_t (*Search)(struct _MpmCtx *, struct _MpmThreadCtx *, PatternMatcherQueue *, u_int8_t *, u_int16_t);
     void (*Cleanup)(struct _MpmThreadCtx *);
     void (*PrintCtx)(struct _MpmCtx *);
     void (*PrintThreadCtx)(struct _MpmThreadCtx *);
@@ -106,7 +132,7 @@ typedef struct MpmTableElmt {
 
 void MpmMatchCleanup(MpmThreadCtx *);
 MpmMatch *MpmMatchAlloc(MpmThreadCtx *);
-void MpmMatchAppend(MpmThreadCtx *, MpmEndMatch *, MpmMatchBucket *, u_int16_t);
+int MpmMatchAppend(MpmThreadCtx *, PatternMatcherQueue *, MpmEndMatch *, MpmMatchBucket *, u_int16_t, u_int16_t);
 MpmEndMatch *MpmAllocEndMatch (MpmCtx *);
 void MpmEndMatchFreeAll(MpmCtx *mpm_ctx, MpmEndMatch *em);
 void MpmMatchFreeSpares(MpmThreadCtx *mpm_ctx, MpmMatch *m);

src/vips.c

@@ -89,19 +89,30 @@ setup_signal_handler(int sig, void (*handler)())
 Packet *SetupPkt (void)
 {
     Packet *p = NULL;
-    do {
+//    do {
         mutex_lock(&packet_q.mutex_q);
         p = PacketDequeue(&packet_q);
         mutex_unlock(&packet_q.mutex_q);
 
         if (p == NULL) {
+            p = malloc(sizeof(Packet));
+            if (p == NULL) {
+                printf("ERROR: malloc failed: %s\n", strerror(errno));
+                exit(1);
+            }
+
+            p->pktvar = NULL;
+            CLEAR_TCP_PACKET(p);
+
             //TmqDebugList();
-            usleep(1000); /* sleep 1ms */
+            //usleep(1000); /* sleep 1ms */
 
             /* XXX check for recv'd signals, so
              * we can exit on signals received */
+
+            printf("SetupPkt: allocated a new packet...\n");
         }
-    } while (p == NULL);
+//    } while (p == NULL);
 
     CLEAR_PACKET(p);
     return p;
@@ -147,6 +158,7 @@ Packet *TunnelPktSetup(ThreadVars *t, Packet *parent, u_int8_t *pkt, u_int16_t l
     p->tunnel_proto = proto;
     p->pktlen = len;
     memcpy(&p->pkt, pkt, len);
+    p->recursion_level = parent->recursion_level + 1;
 
     /* set tunnel flags */
     SET_TUNNEL_PKT(p);
@@ -257,7 +269,8 @@ int main(int argc, char **argv)
         printf("ERROR: TmThreadSpawn failed\n");
         exit(1);
     }
-
+#define MANY_THREADS
+#ifdef MANY_THREADS
     ThreadVars *tv_decode1 = TmThreadCreate("Decode1","pickup-queue","simple","decode-queue1","simple","1slot");
     if (tv_decode1 == NULL) {
         printf("ERROR: TmThreadsCreate failed for Decode1\n");
@@ -427,6 +440,81 @@ int main(int argc, char **argv)
         printf("ERROR: TmThreadSpawn failed\n");
         exit(1);
     }
+#else /* MANY_THREADS */
+    ThreadVars *tv_main = TmThreadCreate("MainThread","pickup-queue","simple","packetpool","packetpool","varslot");
+    if (tv_main == NULL) {
+        printf("ERROR: TmThreadsCreate failed for MainThread\n");
+        exit(1);
+    }
+    tm_module = TmModuleGetByName("DecodeNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName DecodeNFQ failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    tm_module = TmModuleGetByName("Detect");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName Detect failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    tm_module = TmModuleGetByName("VerdictNFQ");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName VerdictNFQ failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    tm_module = TmModuleGetByName("RespondReject");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName for RespondReject failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    tm_module = TmModuleGetByName("AlertFastlog");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    tm_module = TmModuleGetByName("LogHttplog");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    tm_module = TmModuleGetByName("AlertUnifiedLog");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    tm_module = TmModuleGetByName("AlertUnifiedAlert");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    tm_module = TmModuleGetByName("AlertDebuglog");
+    if (tm_module == NULL) {
+        printf("ERROR: TmModuleGetByName failed\n");
+        exit(1);
+    }
+    TmVarSlotSetFuncAppend(tv_main,tm_module);
+
+    if (TmThreadSpawn(tv_main) != 0) {
+        printf("ERROR: TmThreadSpawn failed\n");
+        exit(1);
+    }
+
+#endif /* MANY_THREADS */
 
     ThreadVars tv_flowmgr;
     memset(&tv_flowmgr, 0, sizeof(ThreadVars));