Large update containing the first step to making the detection engine use rule groups. Address based rule groups are now implemented.

TODO

@@ -1 +1,19 @@
-Test more
+DETECTION ENGINE:
+- create a detection_engine_ctx for storing the sig list, packet entry point, other stuff
+- many group heads seem to have the same number of sigs. See if we can save memory by detecting similars... the savings are in the pattern matcher.
+ -> this is done, and works. However it can be taken further. It's not the group heads that should be compared, but the mpm contexts...
+
+- implement flow as a prefilter
+- implement protocol as a prefilter
+- implement src and dst ports as prefilters
+
+
+WU-MANBER:
+
+- Consider using dynamic/variable hash sizes. A wm_ctx is quite big (512kb) even for small pattern sets.
+
+
+MAIN:
+- move packet preallocation into it's own function
+- create a cleanup function
+

src/Makefile.am

@@ -18,6 +18,7 @@ flow-util.c flow-util.h \
 flow-var.c flow-var.h \
 host.c host.h \
 detect.c detect.h \
+detect-siggroup.c detect-siggroup.h \
 detect-parse.c detect-parse.h \
 detect-mpm.c detect-mpm.h \
 detect-content.c detect-content.h \

src/detect-address-ipv4.c

@@ -12,7 +12,7 @@
 
 #include "detect-address.h"
 
-int AddressCmpIPv4(DetectAddressData *a, DetectAddressData *b) {
+int DetectAddressCmpIPv4(DetectAddressData *a, DetectAddressData *b) {
     u_int32_t a_ip1 = ntohl(a->ip[0]);
     u_int32_t a_ip2 = ntohl(a->ip2[0]);
     u_int32_t b_ip1 = ntohl(b->ip[0]);
@@ -23,7 +23,7 @@ int AddressCmpIPv4(DetectAddressData *a, DetectAddressData *b) {
         //printf("ADDRESS_EQ\n");
         return ADDRESS_EQ;
     /* ADDRESS_ES */
-    } else if (a_ip1 >= b_ip1 && a_ip1 < b_ip2 && a_ip2 <= b_ip2) {
+    } else if (a_ip1 >= b_ip1 && a_ip1 <= b_ip2 && a_ip2 <= b_ip2) {
         //printf("ADDRESS_ES\n");
         return ADDRESS_ES;
     /* ADDRESS_EB */
@@ -62,7 +62,7 @@ int AddressCmpIPv4(DetectAddressData *a, DetectAddressData *b) {
  * a = 1.2.3.4, b = 1.2.3.0/24
  * must result in: a == 1.2.3.0-1.2.3.3, b == 1.2.3.4, c == 1.2.3.5-1.2.3.255
  */
-int AddressCutIPv4(DetectAddressData *a, DetectAddressData *b, DetectAddressData **c) {
+int DetectAddressCutIPv4(DetectAddressData *a, DetectAddressData *b, DetectAddressData **c) {
     u_int32_t a_ip1 = ntohl(a->ip[0]);
     u_int32_t a_ip2 = ntohl(a->ip2[0]);
     u_int32_t b_ip1 = ntohl(b->ip[0]);
@@ -71,7 +71,7 @@ int AddressCutIPv4(DetectAddressData *a, DetectAddressData *b, DetectAddressData
     /* default to NULL */
     *c = NULL;
 
-    int r = AddressCmpIPv4(a,b);
+    int r = DetectAddressCmpIPv4(a,b);
     if (r != ADDRESS_ES && r != ADDRESS_EB && r != ADDRESS_LE && r != ADDRESS_GE) {
         goto error;
     }
@@ -227,7 +227,7 @@ error:
  * must result in: a == 0.0.0.0-255.255.255.254, b == NULL
  *
  */
-int AddressCutNotIPv4(DetectAddressData *a, DetectAddressData **b) {
+int DetectAddressCutNotIPv4(DetectAddressData *a, DetectAddressData **b) {
     u_int32_t a_ip1 = ntohl(a->ip[0]);
     u_int32_t a_ip2 = ntohl(a->ip2[0]);
 

src/detect-address-ipv4.h

@@ -6,9 +6,9 @@
 #ifndef __DETECT_ADDRESS_IPV4_H__
 #define __DETECT_ADDRESS_IPV4_H__
 
-int AddressCmpIPv4(DetectAddressData *, DetectAddressData *);
-int AddressCutIPv4(DetectAddressData *, DetectAddressData *, DetectAddressData **);
-int AddressCutNotIPv4(DetectAddressData *, DetectAddressData **);
+int DetectAddressCmpIPv4(DetectAddressData *, DetectAddressData *);
+int DetectAddressCutIPv4(DetectAddressData *, DetectAddressData *, DetectAddressData **);
+int DetectAddressCutNotIPv4(DetectAddressData *, DetectAddressData **);
 
 #endif /* __DETECT_ADDRESS_IPV4_H__ */
 

src/detect-address-ipv6.c

@@ -75,7 +75,7 @@ int AddressIPv6Ge(u_int32_t *a, u_int32_t *b) {
     return 0;
 }
 
-int AddressCmpIPv6(DetectAddressData *a, DetectAddressData *b) {
+int DetectAddressCmpIPv6(DetectAddressData *a, DetectAddressData *b) {
     /* ADDRESS_EQ */
     if (AddressIPv6Eq(a->ip, b->ip) == 1 &&
         AddressIPv6Eq(a->ip2, b->ip2) == 1) {
@@ -83,7 +83,7 @@ int AddressCmpIPv6(DetectAddressData *a, DetectAddressData *b) {
         return ADDRESS_EQ;
     /* ADDRESS_ES */
     } else if (AddressIPv6Ge(a->ip, b->ip) == 1 &&
-               AddressIPv6Lt(a->ip, b->ip2) == 1 &&
+               AddressIPv6Le(a->ip, b->ip2) == 1 &&
                AddressIPv6Le(a->ip2, b->ip2) == 1) {
         //printf("ADDRESS_ES\n");
         return ADDRESS_ES;
@@ -186,7 +186,7 @@ static void AddressCutIPv6Copy(u_int32_t *a, u_int32_t *b) {
     b[3] = htonl(a[3]);
 }
 
-int AddressCutIPv6(DetectAddressData *a, DetectAddressData *b, DetectAddressData **c) {
+int DetectAddressCutIPv6(DetectAddressData *a, DetectAddressData *b, DetectAddressData **c) {
     u_int32_t a_ip1[4] = { ntohl(a->ip[0]), ntohl(a->ip[1]), ntohl(a->ip[2]), ntohl(a->ip[3]) };
     u_int32_t a_ip2[4] = { ntohl(a->ip2[0]), ntohl(a->ip2[1]), ntohl(a->ip2[2]), ntohl(a->ip2[3]) };
     u_int32_t b_ip1[4] = { ntohl(b->ip[0]), ntohl(b->ip[1]), ntohl(b->ip[2]), ntohl(b->ip[3]) };
@@ -195,7 +195,7 @@ int AddressCutIPv6(DetectAddressData *a, DetectAddressData *b, DetectAddressData
     /* default to NULL */
     *c = NULL;
 
-    int r = AddressCmpIPv6(a,b);
+    int r = DetectAddressCmpIPv6(a,b);
     if (r != ADDRESS_ES && r != ADDRESS_EB && r != ADDRESS_LE && r != ADDRESS_GE) {
         goto error;
     }
@@ -350,7 +350,7 @@ error:
  * must result in: a == 0.0.0.0-255.255.255.254, b == NULL
  *
  */
-int AddressCutNotIPv6(DetectAddressData *a, DetectAddressData **b) {
+int DetectAddressCutNotIPv6(DetectAddressData *a, DetectAddressData **b) {
     u_int32_t a_ip1[4] = { ntohl(a->ip[0]), ntohl(a->ip[1]), ntohl(a->ip[2]), ntohl(a->ip[3]) };
     u_int32_t a_ip2[4] = { ntohl(a->ip2[0]), ntohl(a->ip2[1]), ntohl(a->ip2[2]), ntohl(a->ip2[3]) };
     u_int32_t ip_nul[4] = { 0x00000000, 0x00000000, 0x00000000, 0x00000000 };

src/detect-address-ipv6.h

@@ -12,9 +12,9 @@ int AddressIPv6Eq(u_int32_t *, u_int32_t *);
 int AddressIPv6Le(u_int32_t *, u_int32_t *);
 int AddressIPv6Ge(u_int32_t *, u_int32_t *);
 
-int AddressCutIPv6(DetectAddressData *, DetectAddressData *, DetectAddressData **);
-int AddressCutNotIPv6(DetectAddressData *, DetectAddressData **);
-int AddressCmpIPv6(DetectAddressData *, DetectAddressData *);
+int DetectAddressCutIPv6(DetectAddressData *, DetectAddressData *, DetectAddressData **);
+int DetectAddressCutNotIPv6(DetectAddressData *, DetectAddressData **);
+int DetectAddressCmpIPv6(DetectAddressData *, DetectAddressData *);
 
 void DetectAddressIPv6Tests(void);
 

src/detect-address.c

@@ -27,11 +27,9 @@ void DetectAddressRegister (void) {
 }
 
 /* prototypes */
-DetectAddressData *DetectAddressParse(char *);
 void DetectAddressDataPrint(DetectAddressData *);
-int AddressCmp(DetectAddressData *, DetectAddressData *);
-int AddressCut(DetectAddressData *, DetectAddressData *, DetectAddressData **);
-int AddressCutNot(DetectAddressData *, DetectAddressData **);
+int DetectAddressCut(DetectAddressData *, DetectAddressData *, DetectAddressData **);
+int DetectAddressCutNot(DetectAddressData *, DetectAddressData **);
 
 
 DetectAddressGroup *DetectAddressGroupInit(void) {
@@ -122,9 +120,11 @@ int DetectAddressGroupInsert(DetectAddressGroupsHead *gh, DetectAddressData *new
         int r = 0;
 
         for (cur = head; cur != NULL; cur = cur->next) {
-            r = AddressCmp(new,cur->ad);
+            r = DetectAddressCmp(new,cur->ad);
             if (r == ADDRESS_ER) {
-                printf("ADDRESS_ER\n");
+                printf("ADDRESS_ER DetectAddressCmp compared:\n");
+                DetectAddressDataPrint(new);
+                DetectAddressDataPrint(cur->ad);
                 goto error;
             } 
             /* if so, handle that */
@@ -178,22 +178,22 @@ int DetectAddressGroupInsert(DetectAddressGroupsHead *gh, DetectAddressData *new
 
             } else if (r == ADDRESS_ES) {
                 DetectAddressData *c = NULL;
-                r = AddressCut(cur->ad,new,&c);
+                r = DetectAddressCut(cur->ad,new,&c);
                 DetectAddressGroupInsert(gh, new);
                 if (c) DetectAddressGroupInsert(gh, c);
             } else if (r == ADDRESS_EB) {
                 DetectAddressData *c = NULL;
-                r = AddressCut(cur->ad,new,&c);
+                r = DetectAddressCut(cur->ad,new,&c);
                 DetectAddressGroupInsert(gh, new);
                 if (c) DetectAddressGroupInsert(gh, c);
             } else if (r == ADDRESS_LE) {
                 DetectAddressData *c = NULL;
-                r = AddressCut(cur->ad,new,&c);
+                r = DetectAddressCut(cur->ad,new,&c);
                 DetectAddressGroupInsert(gh, new);
                 if (c) DetectAddressGroupInsert(gh, c);
             } else if (r == ADDRESS_GE) {
                 DetectAddressData *c = NULL;
-                r = AddressCut(cur->ad,new,&c);
+                r = DetectAddressCut(cur->ad,new,&c);
                 DetectAddressGroupInsert(gh, new);
                 if (c) DetectAddressGroupInsert(gh, c);
             }
@@ -232,7 +232,7 @@ int DetectAddressGroupSetup(DetectAddressGroupsHead *gh, char *s) {
     if (ad->flags & ADDRESS_FLAG_NOT) {
         DetectAddressData *ad2 = NULL;
 
-        if (AddressCutNot(ad,&ad2) < 0) {
+        if (DetectAddressCutNot(ad,&ad2) < 0) {
             goto error;
         }
 
@@ -249,6 +249,22 @@ int DetectAddressGroupSetup(DetectAddressGroupsHead *gh, char *s) {
     if (DetectAddressGroupInsert(gh, ad) < 0)
         goto error;
 
+    /* if any, insert 0.0.0.0/0 and ::/0 as well */
+    if (ad->flags & ADDRESS_FLAG_ANY) {
+        ad = DetectAddressParse("0.0.0.0/0");
+        if (ad == NULL)
+            goto error;
+
+	if (DetectAddressGroupInsert(gh, ad) < 0)
+	    goto error;
+
+        ad = DetectAddressParse("::/0");
+        if (ad == NULL)
+            goto error;
+
+	if (DetectAddressGroupInsert(gh, ad) < 0)
+	    goto error;
+    }
     return 0;
 
 error:
@@ -267,42 +283,52 @@ DetectAddressGroupsHead *DetectAddressGroupsHeadInit(void) {
     return gh;
 }
 
-void DetectAddressGroupsHeadFree(DetectAddressGroupsHead *gh) {
+void DetectAddressGroupsHeadCleanup(DetectAddressGroupsHead *gh) {
     if (gh != NULL) {
+        DetectAddressGroupCleanupList(gh->any_head);
         DetectAddressGroupCleanupList(gh->ipv4_head);
         DetectAddressGroupCleanupList(gh->ipv6_head);
+    }
+}
+
+void DetectAddressGroupsHeadFree(DetectAddressGroupsHead *gh) {
+    if (gh != NULL) {
+        DetectAddressGroupsHeadCleanup(gh);
         free(gh);
     }
 }
 
-int AddressCut(DetectAddressData *a, DetectAddressData *b, DetectAddressData **c) {
+int DetectAddressCut(DetectAddressData *a, DetectAddressData *b, DetectAddressData **c) {
     if (a->family == AF_INET) {
-        return AddressCutIPv4(a,b,c);
+        return DetectAddressCutIPv4(a,b,c);
     } else if (a->family == AF_INET6) {
-        return AddressCutIPv6(a,b,c);
+        return DetectAddressCutIPv6(a,b,c);
     }
 
     return -1;
 }
 
-int AddressCutNot(DetectAddressData *a, DetectAddressData **b) {
+int DetectAddressCutNot(DetectAddressData *a, DetectAddressData **b) {
     if (a->family == AF_INET) {
-        return AddressCutNotIPv4(a,b);
+        return DetectAddressCutNotIPv4(a,b);
     } else if (a->family == AF_INET6) {
-        return AddressCutNotIPv6(a,b);
+        return DetectAddressCutNotIPv6(a,b);
     }
 
     return -1;
 }
 
-int AddressCmp(DetectAddressData *a, DetectAddressData *b) {
+int DetectAddressCmp(DetectAddressData *a, DetectAddressData *b) {
     if (a->family != b->family)
         return ADDRESS_ER;
 
-    if (a->family == AF_INET)
-        return AddressCmpIPv4(a,b);
+    /* check any */
+    if (a->flags & ADDRESS_FLAG_ANY && b->flags & ADDRESS_FLAG_ANY)
+        return ADDRESS_EQ;
+    else if (a->family == AF_INET)
+        return DetectAddressCmpIPv4(a,b);
     else if (a->family == AF_INET6)
-        return AddressCmpIPv6(a,b);
+        return DetectAddressCmpIPv6(a,b);
 
     return ADDRESS_ER;
 }
@@ -531,6 +557,18 @@ error:
     return NULL;
 }
 
+DetectAddressData *DetectAddressDataCopy(DetectAddressData *src) {
+    DetectAddressData *dst = DetectAddressDataInit();
+    if (dst == NULL) {
+        goto error;
+    }
+
+    memcpy(dst,src,sizeof(DetectAddressData));
+
+    return dst;
+error:
+    return NULL;
+}
 
 int DetectAddressSetup (Signature *s, SigMatch *m, char *addressstr)
 {
@@ -550,7 +588,9 @@ int DetectAddressSetup (Signature *s, SigMatch *m, char *addressstr)
 }
 
 void DetectAddressDataFree(DetectAddressData *dd) {
-    if (dd) free(dd);
+    if (dd != NULL) {
+        free(dd);
+    }
 }
 
 int DetectAddressMatch (DetectAddressData *dd, Address *a) {
@@ -559,8 +599,11 @@ int DetectAddressMatch (DetectAddressData *dd, Address *a) {
 
     switch (a->family) {
         case AF_INET:
-            if (a->addr_data32[0] >= dd->ip[0] &&
-                a->addr_data32[0] <= dd->ip2[0]) {
+            /* XXX figure out a way to not need to do this ntohl
+             * if we switch to Address inside DetectAddressData
+             * we can do u_int8_t checks */
+            if (ntohl(a->addr_data32[0]) >= ntohl(dd->ip[0]) &&
+                ntohl(a->addr_data32[0]) <= ntohl(dd->ip2[0])) {
                 return 1;
             } else {
                 return 0;
@@ -583,7 +626,9 @@ void DetectAddressDataPrint(DetectAddressData *ad) {
     if (ad == NULL)
         return;
 
-    if (ad->family == AF_INET) {
+    if (ad->flags & ADDRESS_FLAG_ANY) {
+        printf("ANY\n");
+    } else if (ad->family == AF_INET) {
         struct in_addr in;
         char s[16];
 
@@ -606,6 +651,28 @@ void DetectAddressDataPrint(DetectAddressData *ad) {
     }
 }
 
+/* find the group matching address in a group head */
+DetectAddressGroup *
+DetectAddressLookupGroup(DetectAddressGroupsHead *gh, Address *a) {
+    DetectAddressGroup *g;
+
+    /* XXX should we really do this check every time we run
+     * this function? */
+    if (a->family == AF_INET)
+        g = gh->ipv4_head;
+    else if (a->family == AF_INET6)
+        g = gh->ipv6_head;
+    else
+        g = gh->any_head;
+
+    for ( ; g != NULL; g = g->next) {
+        if (DetectAddressMatch(g->ad,a) == 1) {
+            return g;
+        }
+    }
+
+    return NULL;
+}
 
 /* TESTS */
 
@@ -1404,7 +1471,7 @@ int AddressTestCmp01 (void) {
     db = DetectAddressParse("192.168.0.0/255.255.255.0");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_EQ)
+    if (DetectAddressCmp(da,db) != ADDRESS_EQ)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1426,7 +1493,7 @@ int AddressTestCmp02 (void) {
     db = DetectAddressParse("192.168.0.0/255.255.255.0");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_EB)
+    if (DetectAddressCmp(da,db) != ADDRESS_EB)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1448,7 +1515,7 @@ int AddressTestCmp03 (void) {
     db = DetectAddressParse("192.168.0.0/255.255.0.0");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_ES)
+    if (DetectAddressCmp(da,db) != ADDRESS_ES)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1470,7 +1537,7 @@ int AddressTestCmp04 (void) {
     db = DetectAddressParse("192.168.1.0/255.255.255.0");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_LT)
+    if (DetectAddressCmp(da,db) != ADDRESS_LT)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1492,7 +1559,7 @@ int AddressTestCmp05 (void) {
     db = DetectAddressParse("192.168.0.0/255.255.255.0");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_GT)
+    if (DetectAddressCmp(da,db) != ADDRESS_GT)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1514,7 +1581,7 @@ int AddressTestCmp06 (void) {
     db = DetectAddressParse("192.168.0.0/255.255.0.0");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_EQ)
+    if (DetectAddressCmp(da,db) != ADDRESS_EQ)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1536,7 +1603,7 @@ int AddressTestCmpIPv407 (void) {
     db = DetectAddressParse("192.168.1.128-192.168.2.128");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_LE)
+    if (DetectAddressCmp(da,db) != ADDRESS_LE)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1558,7 +1625,7 @@ int AddressTestCmpIPv408 (void) {
     db = DetectAddressParse("192.168.1.0/255.255.255.0");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_GE)
+    if (DetectAddressCmp(da,db) != ADDRESS_GE)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1580,7 +1647,7 @@ int AddressTestCmp07 (void) {
     db = DetectAddressParse("2001::1/3");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_EQ)
+    if (DetectAddressCmp(da,db) != ADDRESS_EQ)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1602,7 +1669,7 @@ int AddressTestCmp08 (void) {
     db = DetectAddressParse("2001::/8");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_EB)
+    if (DetectAddressCmp(da,db) != ADDRESS_EB)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1624,7 +1691,7 @@ int AddressTestCmp09 (void) {
     db = DetectAddressParse("2001::/3");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_ES)
+    if (DetectAddressCmp(da,db) != ADDRESS_ES)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1646,7 +1713,7 @@ int AddressTestCmp10 (void) {
     db = DetectAddressParse("2001:1:2:4:0:0:0:0/64");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_LT)
+    if (DetectAddressCmp(da,db) != ADDRESS_LT)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1668,7 +1735,7 @@ int AddressTestCmp11 (void) {
     db = DetectAddressParse("2001:1:2:3:0:0:0:0/64");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_GT)
+    if (DetectAddressCmp(da,db) != ADDRESS_GT)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -1690,7 +1757,7 @@ int AddressTestCmp12 (void) {
     db = DetectAddressParse("2001:1:2:3:2:0:0:0/64");
     if (db == NULL) goto error;
 
-    if (AddressCmp(da,db) != ADDRESS_EQ)
+    if (DetectAddressCmp(da,db) != ADDRESS_EQ)
         result = 0;
 
     DetectAddressDataFree(da);
@@ -2536,12 +2603,18 @@ int AddressTestCutIPv401(void) {
     a = DetectAddressParse("1.2.3.0/255.255.255.0");
     b = DetectAddressParse("1.2.2.0-1.2.3.4");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2550,7 +2623,7 @@ int AddressTestCutIPv402(void) {
     a = DetectAddressParse("1.2.3.0/255.255.255.0");
     b = DetectAddressParse("1.2.2.0-1.2.3.4");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2558,8 +2631,14 @@ int AddressTestCutIPv402(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2568,7 +2647,7 @@ int AddressTestCutIPv403(void) {
     a = DetectAddressParse("1.2.3.0/255.255.255.0");
     b = DetectAddressParse("1.2.2.0-1.2.3.4");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2586,8 +2665,14 @@ int AddressTestCutIPv403(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2596,7 +2681,7 @@ int AddressTestCutIPv404(void) {
     a = DetectAddressParse("1.2.3.3-1.2.3.6");
     b = DetectAddressParse("1.2.3.0-1.2.3.5");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2614,8 +2699,14 @@ int AddressTestCutIPv404(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2624,7 +2715,7 @@ int AddressTestCutIPv405(void) {
     a = DetectAddressParse("1.2.3.3-1.2.3.6");
     b = DetectAddressParse("1.2.3.0-1.2.3.9");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2642,8 +2733,14 @@ int AddressTestCutIPv405(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2652,7 +2749,7 @@ int AddressTestCutIPv406(void) {
     a = DetectAddressParse("1.2.3.0-1.2.3.9");
     b = DetectAddressParse("1.2.3.3-1.2.3.6");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2670,8 +2767,14 @@ int AddressTestCutIPv406(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2680,7 +2783,7 @@ int AddressTestCutIPv407(void) {
     a = DetectAddressParse("1.2.3.0-1.2.3.6");
     b = DetectAddressParse("1.2.3.0-1.2.3.9");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2695,8 +2798,14 @@ int AddressTestCutIPv407(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2705,7 +2814,7 @@ int AddressTestCutIPv408(void) {
     a = DetectAddressParse("1.2.3.3-1.2.3.9");
     b = DetectAddressParse("1.2.3.0-1.2.3.9");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2720,8 +2829,14 @@ int AddressTestCutIPv408(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2730,7 +2845,7 @@ int AddressTestCutIPv409(void) {
     a = DetectAddressParse("1.2.3.0-1.2.3.9");
     b = DetectAddressParse("1.2.3.0-1.2.3.6");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2745,8 +2860,14 @@ int AddressTestCutIPv409(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 
@@ -2755,7 +2876,7 @@ int AddressTestCutIPv410(void) {
     a = DetectAddressParse("1.2.3.0-1.2.3.9");
     b = DetectAddressParse("1.2.3.3-1.2.3.9");
 
-    if (AddressCut(a,b,&c) == -1) {
+    if (DetectAddressCut(a,b,&c) == -1) {
         goto error;
     }
 
@@ -2770,8 +2891,14 @@ int AddressTestCutIPv410(void) {
         goto error;
     }
 
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 1;
 error:
+    DetectAddressDataFree(a);
+    DetectAddressDataFree(b);
+    DetectAddressDataFree(c);
     return 0;
 }
 

src/detect-address.h

@@ -17,6 +17,7 @@ enum {
 #define ADDRESS_FLAG_NOT 0x2
 
 typedef struct DetectAddressData_ {
+    /* XXX convert to use a Address datatype to replace family, ip,ip2*/
     u_int8_t family;
     u_int32_t ip[4];
     u_int32_t ip2[4];
@@ -28,7 +29,10 @@ typedef struct DetectAddressGroup_ {
     DetectAddressData *ad;
 
     /* XXX ptr to rules, or PortGroup or whatever */
+    struct DetectAddressGroupsHead_ *dst_gh;
 
+    /* signatures that belong in this group */
+    struct _SigGroupHead *sh;
 
     /* double linked list */
     struct DetectAddressGroup_ *prev;
@@ -46,8 +50,16 @@ typedef struct DetectAddressGroupsHead_ {
 void DetectAddressRegister (void);
 DetectAddressGroupsHead *DetectAddressGroupsHeadInit();
 void DetectAddressGroupsHeadFree(DetectAddressGroupsHead *);
+void DetectAddressGroupsHeadCleanup(DetectAddressGroupsHead *);
 DetectAddressData *DetectAddressDataInit(void);
 void DetectAddressDataFree(DetectAddressData *);
+void DetectAddressDataPrint(DetectAddressData *);
+DetectAddressData *DetectAddressDataCopy(DetectAddressData *);
+int DetectAddressGroupInsert(DetectAddressGroupsHead *, DetectAddressData *);
+int DetectAddressGroupSetup(DetectAddressGroupsHead *, char *);
+int DetectAddressCmp(DetectAddressData *, DetectAddressData *);
+DetectAddressData *DetectAddressParse(char *);
+DetectAddressGroup *DetectAddressLookupGroup(DetectAddressGroupsHead *, Address *);
 
 #endif /* __DETECT_ADDRESS_H__ */
 

src/detect-content.c

@@ -47,6 +47,10 @@ int DetectContentSetup (Signature *, SigMatch *, char *);
 u_int8_t nocasetable[256];
 #define _nc(c) nocasetable[(c)]
 
+/* we use a global id for content matches to be able to use
+ * just one pattern matcher thread context per thread. */
+static u_int32_t content_max_id = 0;
+
 void DetectContentRegister (void) {
     sigmatch_table[DETECT_CONTENT].name = "content";
     sigmatch_table[DETECT_CONTENT].Match = DetectContentMatch;
@@ -72,6 +76,12 @@ void DetectContentRegister (void) {
 #endif /* DEBUG */
 }
 
+/* pass on the content_max_id */
+u_int32_t DetectContentMaxId(void) {
+    printf("DetectContentMaxId: %u\n", content_max_id);
+    return content_max_id;
+}
+
 static inline int
 TestOffsetDepth(MpmMatch *m, DetectContentData *co, u_int16_t pktoff) {
     if (m->offset >= pktoff) {
@@ -105,7 +115,7 @@ TestWithinDistanceOffsetDepth(ThreadVars *t, PatternMatcherThread *pmt, MpmMatch
         return 1;
 
     DetectContentData *co = (DetectContentData *)nsm->ctx;
-    MpmMatch *nm = pmt->mpm_ctx[pmt->mpm_instance].match[co->id].top;
+    MpmMatch *nm = pmt->mtc.match[co->id].top;
 
     for (; nm; nm = nm->next) {
         //printf("TestWithinDistanceOffsetDepth: nm->offset %u, m->offset %u\n", nm->offset, m->offset);
@@ -150,7 +160,7 @@ DoDetectContent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signature *
     char match = 0;
 
     /* Get the top match, we already know we have one. */
-    MpmMatch *m = pmt->mpm_ctx[pmt->mpm_instance].match[co->id].top;
+    MpmMatch *m = pmt->mtc.match[co->id].top;
 
     /*  if we have within or distance coming up next, check this match
      *  for distance and/or within and check the rest of this match
@@ -243,7 +253,7 @@ int DetectContentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Sig
     DetectContentData *co = (DetectContentData *)m->ctx;
 
     /* see if we had a match */
-    len = pmt->mpm_ctx[pmt->mpm_instance].match[co->id].len;
+    len = pmt->mtc.match[co->id].len;
     if (len == 0)
         return 0;
 
@@ -251,7 +261,7 @@ int DetectContentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Sig
     printf("content \'%s\' matched %u time(s) at offsets: ", co->content, len);
 
     MpmMatch *tmpm = NULL;
-    for (tmpm = pmt->mpm_ctx[pmt->mpm_instance].match[co->id].top; tmpm != NULL; tmpm = tmpm->next) {
+    for (tmpm = pmt->mtc.match[co->id].top; tmpm != NULL; tmpm = tmpm->next) {
         printf("%u ", tmpm->offset);
     }
     printf("\n");
@@ -368,6 +378,9 @@ int DetectContentSetup (Signature *s, SigMatch *m, char *contentstr)
 
     SigMatchAppend(s,m,sm);
 
+    cd->id = content_max_id;
+    content_max_id++;
+
     if (dubbed) free(str);
     return 0;
 

src/detect-content.h

@@ -25,6 +25,7 @@ typedef struct _DetectContentData {
 
 /* prototypes */
 void DetectContentRegister (void);
+u_int32_t DetectContentMaxId(void);
 
 #endif /* __DETECT_CONTENT_H__ */
 

src/detect-mpm.c

@@ -7,7 +7,7 @@
 
 #include "decode.h"
 #include "detect.h"
-
+#include "detect-siggroup.h"
 #include "detect-mpm.h"
 #include "util-mpm.h"
 
@@ -18,30 +18,67 @@
 #include "detect-content.h"
 #include "detect-uricontent.h"
 
-MpmCtx mpm_ctx[MPM_INSTANCE_MAX];
-
 u_int32_t PacketPatternMatch(ThreadVars *t, PatternMatcherThread *pmt, Packet *p) {
     u_int32_t ret;
 
-    ret = mpm_ctx[pmt->mpm_instance].Search(&mpm_ctx[pmt->mpm_instance], &pmt->mpm_ctx[pmt->mpm_instance], p->tcp_payload, p->tcp_payload_len);
+    ret = pmt->mc->Search(pmt->mc, &pmt->mtc, p->tcp_payload, p->tcp_payload_len);
 
     //printf("PacketPatternMatch: ret %u\n", ret);
     return ret;
 }
 
 /* cleans up the mpm instance after a match */
-void PacketPatternCleanup(ThreadVars *t, PatternMatcherThread *pmt, u_int8_t instance) {
-    if (mpm_ctx[pmt->mpm_instance].Cleanup != NULL) {
-        mpm_ctx[pmt->mpm_instance].Cleanup(&pmt->mpm_ctx[instance]);
+void PacketPatternCleanup(ThreadVars *t, PatternMatcherThread *pmt) {
+    /* content */
+    if (pmt->mc != NULL && pmt->mc->Cleanup != NULL) {
+        pmt->mc->Cleanup(&pmt->mtc);
     }
+    /* uricontent */
+    if (pmt->mcu != NULL && pmt->mcu->Cleanup != NULL) {
+        pmt->mcu->Cleanup(&pmt->mtcu);
+    }
+}
+
+/* XXX remove this once we got rid of the global mpm_ctx */
+void PatternMatchDestroy(MpmCtx *mc) {
+    u_int8_t instance = 0;
+
+    for (instance = 0; instance < MPM_INSTANCE_MAX; instance++)
+        mc[instance].DestroyCtx(&mc[instance]);
 }
 
-void PatternMatchDestroy(void) {
-    u_int8_t instance;
+/* TODO remove this when we move to the rule groups completely */
+void PatternMatchPrepare(MpmCtx *mc)
+{
+    u_int8_t instance = 0;
 
     /* intialize contexes */
     for (instance = 0; instance < MPM_INSTANCE_MAX; instance++) {
-        mpm_ctx[instance].DestroyCtx(&mpm_ctx[instance]);
+        MpmInitCtx(&mc[instance], MPM_WUMANBER);
+    }
+}
+
+
+/* free the pattern matcher part of a SigGroupHead */
+void PatternMatchDestroyGroup(SigGroupHead *sh) {
+    /* content */
+    if (sh->flags & SIG_GROUP_HAVECONTENT && sh->mpm_ctx != NULL) {
+        sh->mpm_ctx->DestroyCtx(sh->mpm_ctx);
+        free(sh->mpm_ctx);
+
+        /* ready for reuse */
+        sh->mpm_ctx = NULL;
+        sh->flags &= ~SIG_GROUP_HAVECONTENT;
+    }
+
+    /* uricontent */
+    if (sh->flags & SIG_GROUP_HAVEURICONTENT && sh->mpm_uri_ctx != NULL) {
+        sh->mpm_uri_ctx->DestroyCtx(sh->mpm_uri_ctx);
+        free(sh->mpm_uri_ctx);
+
+        /* ready for reuse */
+        sh->mpm_uri_ctx = NULL;
+        sh->flags &= ~SIG_GROUP_HAVEURICONTENT;
     }
 }
 
@@ -51,143 +88,122 @@ void PatternMatchDestroy(void) {
  *  - determine if a content match can set the 'single' flag
  *
  *
+ * XXX do error checking
  */
-void PatternMatchPrepare(Signature *rootsig)
+int PatternMatchPrepareGroup(SigGroupHead *sh)
 {
     Signature *s;
-    u_int8_t instance = 0;
+    SigGroupContainer *sc;
+    u_int32_t co_cnt = 0;
+    u_int32_t ur_cnt = 0;
+    u_int32_t cnt = 0;
+
+    /* see if this head has content and/or uricontent */
+    for (sc = sh->head; sc != NULL; sc = sc->next) {
+        s = sc->s;
+
+        /* find flow setting of this rule */
+        SigMatch *sm;
 
-    u_int32_t id = 0;
-    u_int32_t depth = 0;
-    u_int32_t offset = 0;
-    u_int32_t within = 0;
-    u_int32_t distance = 0;
-    u_int32_t keywords = 0;
+        for (sm = s->match; sm != NULL; sm = sm->next) {
+            if (sm->type == DETECT_CONTENT) {
+                co_cnt++;
+            } else if (sm->type == DETECT_URICONTENT) {
+                ur_cnt++;
+            }
+        }
+    }
 
-    u_int32_t uri_id = 0;
-    u_int32_t uri_depth = 0;
-    u_int32_t uri_offset = 0;
-    u_int32_t uri_within = 0;
-    u_int32_t uri_distance = 0;
-    u_int32_t uri_keywords = 0;
+    if (co_cnt > 0) {
+        sh->flags |= SIG_GROUP_HAVECONTENT;
+    }
+    if (ur_cnt > 0) {
+        sh->flags |= SIG_GROUP_HAVEURICONTENT;
+    }
 
     /* intialize contexes */
-    for (instance = 0; instance < MPM_INSTANCE_MAX; instance++) {
-        MpmInitCtx(&mpm_ctx[instance], MPM_WUMANBER);
+    if (sh->flags & SIG_GROUP_HAVECONTENT) {
+        sh->mpm_ctx = malloc(sizeof(MpmCtx));
+        if (sh->mpm_ctx == NULL)
+            goto error;
+
+        MpmInitCtx(sh->mpm_ctx, MPM_WUMANBER);
     }
+    if (sh->flags & SIG_GROUP_HAVEURICONTENT) {
+        sh->mpm_uri_ctx = malloc(sizeof(MpmCtx));
+        if (sh->mpm_uri_ctx == NULL)
+            goto error;
 
-    for (s = rootsig; s != NULL; s = s->next) {
-        instance = MPM_INSTANCE_BOTH;
+        MpmInitCtx(sh->mpm_uri_ctx, MPM_WUMANBER);
+    }
 
-        SigMatch *sm;
-        for (sm = s->match; sm != NULL; sm = sm->next) {
-            if (sm->type == DETECT_FLOW) {
-                DetectFlowData *fd = (DetectFlowData *)sm->ctx;
-                if (fd->flags & FLOW_PKT_TOSERVER)
-                    instance = MPM_INSTANCE_TOSERVER;
-                else if (fd->flags & FLOW_PKT_TOCLIENT)
-                    instance = MPM_INSTANCE_TOCLIENT;
-
-                break;
-            }
-        }
-        //printf("Add sig %u to instance %u\n", s->id, instance);
+    /* for each signature in this group do */
+    for (sc = sh->head; sc != NULL; sc = sc->next) {
+        s = sc->s;
+        cnt++;
 
+        /* find flow setting of this rule */
+        SigMatch *sm;
         for (sm = s->match; sm != NULL; sm = sm->next) {
             if (sm->type == DETECT_CONTENT) {
                 DetectContentData *cd = (DetectContentData *)sm->ctx;
 
-                if (cd->depth) depth++;
-                if (cd->offset) offset++;
-                if (cd->within) within++;
-                if (cd->distance) distance++;
-
-                if (instance == MPM_INSTANCE_BOTH) { /* no flow setting in rule */
-                    if (cd->flags & DETECT_CONTENT_NOCASE) {
-                        mpm_ctx[MPM_INSTANCE_TOSERVER].AddPatternNocase(&mpm_ctx[MPM_INSTANCE_TOSERVER], cd->content, cd->content_len, id);
-                        mpm_ctx[MPM_INSTANCE_TOCLIENT].AddPatternNocase(&mpm_ctx[MPM_INSTANCE_TOCLIENT], cd->content, cd->content_len, id);
-                    } else {
-                        mpm_ctx[MPM_INSTANCE_TOSERVER].AddPattern(&mpm_ctx[MPM_INSTANCE_TOSERVER], cd->content, cd->content_len, id);
-                        mpm_ctx[MPM_INSTANCE_TOCLIENT].AddPattern(&mpm_ctx[MPM_INSTANCE_TOCLIENT], cd->content, cd->content_len, id);
-                    }
+                if (cd->flags & DETECT_CONTENT_NOCASE) {
+                    sh->mpm_ctx->AddPatternNocase(sh->mpm_ctx, cd->content, cd->content_len, cd->id);
                 } else {
-                    if (cd->flags & DETECT_CONTENT_NOCASE) {
-                        mpm_ctx[instance].AddPatternNocase(&mpm_ctx[instance], cd->content, cd->content_len, id);
-                    } else {
-                        mpm_ctx[instance].AddPattern(&mpm_ctx[instance], cd->content, cd->content_len, id);
-                    }
+                    sh->mpm_ctx->AddPattern(sh->mpm_ctx, cd->content, cd->content_len, cd->id);
                 }
-
-                cd->id = id;
-
-                id++;
-                keywords++;
             } else if (sm->type == DETECT_URICONTENT) {
                 DetectUricontentData *ud = (DetectUricontentData *)sm->ctx;
 
-                if (ud->depth) uri_depth++;
-                if (ud->offset) uri_offset++;
-                if (ud->within) uri_within++;
-                if (ud->distance) uri_distance++;
-
-                if (instance == MPM_INSTANCE_BOTH) { /* no flow setting in rule */
-                    if (ud->flags & DETECT_URICONTENT_NOCASE) {
-                        mpm_ctx[MPM_INSTANCE_TOSERVER + MPM_INSTANCE_URIOFFSET].AddPatternNocase(&mpm_ctx[MPM_INSTANCE_TOSERVER + MPM_INSTANCE_URIOFFSET], ud->uricontent, ud->uricontent_len, uri_id);
-                        mpm_ctx[MPM_INSTANCE_TOCLIENT + MPM_INSTANCE_URIOFFSET].AddPatternNocase(&mpm_ctx[MPM_INSTANCE_TOCLIENT + MPM_INSTANCE_URIOFFSET], ud->uricontent, ud->uricontent_len, uri_id);
-                    } else {
-                        mpm_ctx[MPM_INSTANCE_TOSERVER + MPM_INSTANCE_URIOFFSET].AddPattern(&mpm_ctx[MPM_INSTANCE_TOSERVER + MPM_INSTANCE_URIOFFSET], ud->uricontent, ud->uricontent_len, uri_id);
-                        mpm_ctx[MPM_INSTANCE_TOCLIENT + MPM_INSTANCE_URIOFFSET].AddPattern(&mpm_ctx[MPM_INSTANCE_TOCLIENT + MPM_INSTANCE_URIOFFSET], ud->uricontent, ud->uricontent_len, uri_id);
-                    }
+                if (ud->flags & DETECT_URICONTENT_NOCASE) {
+                    sh->mpm_uri_ctx->AddPatternNocase(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, ud->id);
                 } else {
-                    if (ud->flags & DETECT_URICONTENT_NOCASE) {
-                        mpm_ctx[instance + MPM_INSTANCE_URIOFFSET].AddPatternNocase(&mpm_ctx[instance + MPM_INSTANCE_URIOFFSET], ud->uricontent, ud->uricontent_len, uri_id);
-                    } else {
-                        mpm_ctx[instance + MPM_INSTANCE_URIOFFSET].AddPattern(&mpm_ctx[instance + MPM_INSTANCE_URIOFFSET], ud->uricontent, ud->uricontent_len, uri_id);
-                    }
+                    sh->mpm_uri_ctx->AddPattern(sh->mpm_uri_ctx, ud->uricontent, ud->uricontent_len, ud->id);
                 }
-
-                ud->id = uri_id;
-
-                uri_id++;
-                uri_keywords++;
             }
         }
     }
 
-    for (instance = 0; instance < MPM_INSTANCE_MAX; instance++) {
-        if (mpm_ctx[instance].Prepare != NULL) {
-            mpm_ctx[instance].Prepare(&mpm_ctx[instance]);
+    /* content */
+    if (sh->flags & SIG_GROUP_HAVECONTENT) {
+        if (sh->mpm_ctx->Prepare != NULL) {
+            sh->mpm_ctx->Prepare(sh->mpm_ctx);
+        }
+    }
+    /* uricontent */
+    if (sh->flags & SIG_GROUP_HAVEURICONTENT) {
+        if (sh->mpm_uri_ctx->Prepare != NULL) {
+            sh->mpm_uri_ctx->Prepare(sh->mpm_uri_ctx);
         }
     }
 
     //printf("Printing info...\n");
-    //for (instance = 0; instance < MPM_INSTANCE_MAX; instance++) {
-    //    mpm_ctx[instance].PrintCtx(&mpm_ctx[instance]);
-    //}
+    //sh->mpm_ctx.PrintCtx(&sh->mpm_ctx);
+    //sh->mpm_uri_ctx.PrintCtx(&sh->mpm_uri_ctx);
+    //printf("mpm_ctx %p\n", &sh->mpm_uri_ctx);
 
-#ifdef DEBUG
-    for (instance = 0; instance < MPM_INSTANCE_MAX; instance++) {
-        printf("Case sensitive:\n");
-        MpmPrintTree(&mpm_ctx[instance].root);
-        printf("Case INsensitive:\n");
-        MpmPrintTree(&mpm_ctx[instance].nocase_root);
-    }
-#endif /* DEBUG */
+    return 0;
+error:
+    /* XXX */
+    return -1;
 }
 
 int PatternMatcherThreadInit(ThreadVars *t, void **data) {
-    u_int8_t mpm_instance = 0;
-
     PatternMatcherThread *pmt = malloc(sizeof(PatternMatcherThread));
     if (pmt == NULL) {
         return -1;
     }
     memset(pmt, 0, sizeof(PatternMatcherThread));
 
-    /* intialize contexes */
-    for (mpm_instance = 0; mpm_instance < MPM_INSTANCE_MAX; mpm_instance++) {
-        mpm_ctx[mpm_instance].InitThreadCtx(&mpm_ctx[mpm_instance], &pmt->mpm_ctx[mpm_instance]);
-    }
+    /* XXX we still depend on the global mpm_ctx here
+     *
+     * Initialize the thread pattern match ctx with the max size
+     * of the content and uricontent id's so our match lookup
+     * table is always big enough
+     */
+    mpm_ctx[0].InitThreadCtx(&mpm_ctx[0], &pmt->mtc, DetectContentMaxId());
+    mpm_ctx[0].InitThreadCtx(&mpm_ctx[0], &pmt->mtcu, DetectUricontentMaxId());
 
     *data = (void *)pmt;
     //printf("PatternMatcherThreadInit: data %p pmt %p\n", *data, pmt);
@@ -196,23 +212,18 @@ int PatternMatcherThreadInit(ThreadVars *t, void **data) {
 
 int PatternMatcherThreadDeinit(ThreadVars *t, void *data) {
     PatternMatcherThread *pmt = (PatternMatcherThread *)data;
-    u_int8_t instance;
 
-    /* intialize contexes */
-    for (instance = 0; instance < MPM_INSTANCE_MAX; instance++) {
-        mpm_ctx[instance].DestroyThreadCtx(&mpm_ctx[instance], &pmt->mpm_ctx[instance]);
-    }
+    /* XXX */
+    mpm_ctx[0].DestroyThreadCtx(&mpm_ctx[0], &pmt->mtc);
+    mpm_ctx[0].DestroyThreadCtx(&mpm_ctx[0], &pmt->mtcu);
 
     return 0;
 }
 
 
 void PatternMatcherThreadInfo(ThreadVars *t, PatternMatcherThread *pmt) {
-    u_int8_t mpm_instance = 0;
-
-    /* intialize contexes */
-    for (mpm_instance = 0; mpm_instance < MPM_INSTANCE_MAX; mpm_instance++) {
-        mpm_ctx[mpm_instance].PrintThreadCtx(&pmt->mpm_ctx[mpm_instance]);
-    }
+    /* XXX */
+    mpm_ctx[0].PrintThreadCtx(&pmt->mtc);
+    mpm_ctx[0].PrintThreadCtx(&pmt->mtcu);
 }
 

src/detect-mpm.h

@@ -1,12 +1,16 @@
 #ifndef __DETECT_MPM_H__
 #define __DETECT_MPM_H__
 
+MpmCtx mpm_ctx[MPM_INSTANCE_MAX];
+
 u_int32_t PacketPatternMatch(ThreadVars *, PatternMatcherThread *, Packet *);
 int PacketPatternScan(ThreadVars *t, Packet *p, u_int8_t mpm_instance);
-void PacketPatternCleanup(ThreadVars *, PatternMatcherThread *, u_int8_t);
-void PatternMatchPrepare(Signature *);
+void PacketPatternCleanup(ThreadVars *, PatternMatcherThread *);
+void PatternMatchPrepare(MpmCtx *);
+int PatternMatchPrepareGroup(SigGroupHead *);
 void PatternMatcherThreadInfo(ThreadVars *, PatternMatcherThread *);
-void PatternMatchDestroy(void);
+void PatternMatchDestroy(MpmCtx *);
+void PatternMatchDestroyGroup(SigGroupHead *);
 
 int PatternMatcherThreadInit(ThreadVars *, void **);
 int PatternMatcherThreadDeinit(ThreadVars *, void *);

src/detect-parse.c

@@ -7,6 +7,8 @@
 #include "detect.h"
 #include "flow.h"
 
+#include "util-unittest.h"
+
 static pcre *config_pcre = NULL;
 static pcre *option_pcre = NULL;
 static pcre_extra *config_pcre_extra = NULL;
@@ -141,8 +143,8 @@ int SigParseOptions(Signature *s, SigMatch *m, char *optstr) {
         printf("pcre_exec failed: ret %d, optstr \"%s\"\n", ret, optstr);
         goto error;
     }
-    //printf("SigParseOptions: pcre_exec returned %d\n", ret);
 
+    /* extract the substrings */
     for (i = 1; i <= ret-1; i++) {
         pcre_get_substring(optstr, ov, MAX_SUBSTRINGS, i, &arr[i-1]);
         //printf("SigParseOptions: arr[%d] = \"%s\"\n", i-1, arr[i-1]);
@@ -165,32 +167,43 @@ int SigParseOptions(Signature *s, SigMatch *m, char *optstr) {
     } else {
         optname = (char *)arr[0];
         optvalue = (char *)arr[1];
-        if (ret > 3) optmore = (char *)arr[2];
+        if (ret == 4) optmore = (char *)arr[2];
         else optmore = NULL;
     }
 
     /* setup may or may not add a new SigMatch to the list */
     if (st->Setup(s, m, optvalue) < 0)
         goto error;
-    //printf("SigParseOptions: s->match:%p,m:%p\n", s->match, m);
+
     /* thats why we check for that here */
-    if (m && m->next)
+    if (m != NULL && m->next != NULL)
         m = m->next;
     else if (m == NULL && s->match != NULL)
         m = s->match;
 
-    //printf("SigParseOptions: s->match:%p,m:%p\n", s->match, m);
     if (ret == 4 && optmore != NULL) {
         //printf("SigParseOptions: recursive call for more options... (s:%p,m:%p)\n", s, m);
 
+        if (optname) pcre_free_substring(optname);
+        if (optvalue) pcre_free_substring(optvalue);
+        if (optstr) free(optstr);
+        //if (optmore) pcre_free_substring(optmore);
         if (arr != NULL) free(arr);
-        return(SigParseOptions(s, m, optmore));
+        return SigParseOptions(s, m, optmore);
     }
 
+    if (optname) pcre_free_substring(optname);
+    if (optvalue) pcre_free_substring(optvalue);
+    if (optmore) pcre_free_substring(optmore);
+    if (optstr) free(optstr);
     if (arr != NULL) free(arr);
     return 0;
 
 error:
+    if (optname) pcre_free_substring(optname);
+    if (optvalue) pcre_free_substring(optvalue);
+    if (optmore) pcre_free_substring(optmore);
+    if (optstr) free(optstr);
     if (arr != NULL) free(arr);
     return -1;
 }
@@ -206,16 +219,34 @@ int SigParseAddress(Signature *s, const char *addrstr, char flag) {
     } else if (strcmp(addrstr,"$EXTERNAL_NET") == 0) {
         addr = "!192.168.0.0/16";
     } else if (strcmp(addrstr,"$HTTP_SERVERS") == 0) {
+        addr = "192.168.0.1-192.168.0.16";
     } else if (strcmp(addrstr,"$SMTP_SERVERS") == 0) {
+        addr = "192.168.0.17-192.168.0.32";
     } else if (strcmp(addrstr,"$SQL_SERVERS") == 0) {
+        addr = "192.168.0.33-192.168.0.48";
     } else if (strcmp(addrstr,"$DNS_SERVERS") == 0) {
+        addr = "any";
     } else if (strcmp(addrstr,"any") == 0) {
-
+        addr = "any";
     } else {
+        addr = (char *)addrstr;
         printf("addr \"%s\"\n", addrstr);
     }
 
+    if (flag == 0) {
+        if (DetectAddressGroupSetup(&s->src,addr) < 0) {
+            goto error;
+        }
+    } else {
+        if (DetectAddressGroupSetup(&s->dst,addr) < 0) {
+            goto error;
+        }
+    }
+
+
     return 0;
+error:
+    return -1;
 }
 
 int SigParseProto(Signature *s, const char *protostr) {
@@ -341,7 +372,7 @@ int SigParse(Signature *s, char *sigstr) {
 
     /* we can have no options, so make sure we have them */
     if (basics[CONFIG_OPTS] != NULL) {
-        ret = SigParseOptions(s, NULL, basics[CONFIG_OPTS]);
+        ret = SigParseOptions(s, NULL, strdup(basics[CONFIG_OPTS]));
     }
 
     /* cleanup */
@@ -376,7 +407,11 @@ void SigFree(Signature *s) {
         sm = nsm;
     }
 
-    if (s->msg) free(s->msg);
+    DetectAddressGroupsHeadCleanup(&s->src);
+    DetectAddressGroupsHeadCleanup(&s->dst);
+
+    if (s->msg != NULL) free(s->msg);
+
     free(s);
 }
 
@@ -396,3 +431,28 @@ error:
 
 }
 
+/*
+ * TESTS
+ */
+
+int SigParseTest01 (void) {
+    int result = 1;
+    Signature *sig = NULL;
+
+    SigParsePrepare();
+
+    sig = SigInit("alert tcp 1.2.3.4 any -> !1.2.3.4 any (msg:\"SigParseTest01\"; sid:1;)");
+    if (sig == NULL) {
+        result = 0;
+        goto end;
+    }
+
+    SigFree(sig);
+end:
+    return result;
+}
+
+void SigParseRegisterTests(void) {
+    UtRegisterTest("SigParseTest01", SigParseTest01, 1);
+}
+

src/detect-parse.h

@@ -7,6 +7,7 @@ Signature *SigAlloc (void);
 void SigFree(Signature *s);
 Signature *SigInit(char *sigstr);
 void SigParsePrepare(void);
+void SigParseRegisterTests(void);
 
 #endif /* __DETECT_PARSE_H__ */
 

src/detect-pcre.c

@@ -109,7 +109,7 @@ int DetectPcreMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Signat
             if (ret) {
                 if (strcmp(pe->capname,"http_uri") == 0) {
                     if (pmt->de_scanned_httpuri == 1)
-                        PacketPatternCleanup(t, pmt, pmt->mpm_instance+MPM_INSTANCE_URIOFFSET);
+                        PacketPatternCleanup(t, pmt);
 
                     pmt->de_have_httpuri = 1;
                     pmt->de_scanned_httpuri = 0;
@@ -164,6 +164,10 @@ int DetectPcreSetup (Signature *s, SigMatch *m, char *regexstr)
     //printf("DetectPcreSetup: \'%s\'\n", capture_str_ptr ? capture_str_ptr : "NULL");
 
     ret = pcre_exec(parse_regex, parse_regex_study, regexstr, strlen(regexstr), 0, 0, ov, MAX_SUBSTRINGS);
+    if (ret < 0) {
+        goto error;
+    }
+
     if (ret > 1) {
         const char *str_ptr;
         res = pcre_get_substring((char *)regexstr, ov, MAX_SUBSTRINGS, 1, &str_ptr);
@@ -199,47 +203,49 @@ int DetectPcreSetup (Signature *s, SigMatch *m, char *regexstr)
     }
     //printf("DetectPcreSetup: pd->capname %s\n", pd->capname ? pd->capname : "NULL");
 
-    while (*op) {
-        DEBUGPRINT("DetectPcreSetup: regex option %c", *op);
-
-        switch (*op) {
-            case 'A':
-                opts |= PCRE_ANCHORED;
-                break;
-            case 'E':
-                opts |= PCRE_DOLLAR_ENDONLY;
-                break;
-            case 'G':
-                opts |= PCRE_UNGREEDY;
-                break;
-
-            case 'i':
-                opts |= PCRE_CASELESS;
-                break;
-            case 'm':
-                opts |= PCRE_MULTILINE;
-                break;
-            case 's':
-                opts |= PCRE_DOTALL;
-                break;
-            case 'x':
-                opts |= PCRE_EXTENDED;
-                break;
-
-            case 'B': /* snort's option */
-                pd->flags |= DETECT_PCRE_RAWBYTES;
-                break;
-            case 'R': /* snort's option */
-                pd->flags |= DETECT_PCRE_RELATIVE;
-                break;
-            case 'U': /* snort's option */
-                pd->flags |= DETECT_PCRE_URI;
-                break;
-            default:
-                printf("DetectPcreSetup: unknown regex modifier '%c'\n", *op);
-                break;
+    if (op != NULL) {
+        while (*op) {
+            DEBUGPRINT("DetectPcreSetup: regex option %c", *op);
+
+            switch (*op) {
+                case 'A':
+                    opts |= PCRE_ANCHORED;
+                    break;
+                case 'E':
+                    opts |= PCRE_DOLLAR_ENDONLY;
+                    break;
+                case 'G':
+                    opts |= PCRE_UNGREEDY;
+                    break;
+
+                case 'i':
+                    opts |= PCRE_CASELESS;
+                    break;
+                case 'm':
+                    opts |= PCRE_MULTILINE;
+                    break;
+                case 's':
+                    opts |= PCRE_DOTALL;
+                    break;
+                case 'x':
+                    opts |= PCRE_EXTENDED;
+                    break;
+
+                case 'B': /* snort's option */
+                    pd->flags |= DETECT_PCRE_RAWBYTES;
+                    break;
+                case 'R': /* snort's option */
+                    pd->flags |= DETECT_PCRE_RELATIVE;
+                    break;
+                case 'U': /* snort's option */
+                    pd->flags |= DETECT_PCRE_URI;
+                    break;
+                default:
+                    printf("DetectPcreSetup: unknown regex modifier '%c'\n", *op);
+                    break;
+            }
+            op++;
         }
-        op++;
     }
 
     //printf("DetectPcreSetup: \"%s\"\n", re);

src/detect-siggroup.c

@@ -0,0 +1,140 @@
+
+#include "decode.h"
+#include "detect.h"
+#include "flow-var.h"
+
+#include "util-cidr.h"
+#include "util-unittest.h"
+
+#include "detect.h"
+#include "detect-address.h"
+#include "detect-mpm.h"
+
+int SigGroupHeadCmp(SigGroupHead *, SigGroupHead *);
+
+static SigGroupHead *sgh_list = NULL;
+
+/* return the first SigGroupHead that matches
+ * the lookup one. */
+SigGroupHead* SigGroupHeadListGet(SigGroupHead *a) {
+    SigGroupHead *b = sgh_list;
+
+    for ( ; b != NULL; b = b->next) {
+        if (SigGroupHeadCmp(a,b) == 1 && a != b) {
+            return b;
+        }
+    }
+    return NULL;
+}
+
+/* basically just reset the prt as the list items
+ * themselves are removed elsewhere */
+void SigGroupHeadListClean(void) {
+    sgh_list = NULL;
+}
+
+void SigGroupHeadList(void) {
+    SigGroupHead *sh;
+
+    printf("SigGroupHeadList: start\n");
+    for (sh = sgh_list; sh != NULL; sh = sh->next) {
+        printf("%p sig_cnt %u\n", sh, sh->sig_cnt);
+    }
+    printf("SigGroupHeadList: end\n");
+}
+
+/* put this head in the list */
+void SigGroupHeadAppend(SigGroupHead *sh) {
+    if (sgh_list == NULL) {
+        sgh_list = sh;
+    } else {
+        SigGroupHead *list = sgh_list;
+
+        while (list->next != NULL)
+            list = list->next;
+
+        list->next = sh;
+    }
+}
+
+int SigGroupAppend(DetectAddressGroup *ag, Signature *s) {
+    SigGroupContainer *sg = NULL, *tmp_sg = NULL;
+
+    sg = malloc(sizeof(SigGroupContainer));
+    if (sg == NULL) {
+        goto error;
+    }
+    memset(sg,0,sizeof(SigGroupContainer));
+
+    /* connect the signature to the container */
+    sg->s = s;
+
+    /* see if we have a head already */
+    if (ag->sh == NULL) {
+        ag->sh = malloc(sizeof(SigGroupHead));
+        if (ag->sh == NULL) {
+            goto error;
+        }
+        memset(ag->sh, 0, sizeof(SigGroupHead));
+    }
+
+    if (ag->sh->head == NULL) {
+        /* put it as first in the list */
+        ag->sh->head = sg;
+    } else {
+        /* append to the list */
+        tmp_sg = ag->sh->head;
+        while (tmp_sg->next != NULL) {
+            tmp_sg = tmp_sg->next;
+        }
+
+        tmp_sg->next = sg;
+    }
+    ag->sh->sig_cnt++;
+    return 0;
+error:
+    return -1;
+}
+
+/* XXX function name */
+int SigGroupClean(DetectAddressGroup *ag) {
+    SigGroupContainer *sg = NULL, *next_sg = NULL;
+
+    if (ag->sh == NULL)
+        return 0;
+
+    if (!(ag->sh->flags & SIG_GROUP_COPY))
+        PatternMatchDestroyGroup(ag->sh);
+
+    sg = ag->sh->head;
+
+    while (sg != NULL) {
+        next_sg = sg->next;
+
+        sg->s->rulegroup_refcnt--;
+        sg->s = NULL;
+        free(sg);
+
+        sg = next_sg;        
+    }
+
+    free(ag->sh);
+    return 0;
+}
+
+int SigGroupHeadCmp(SigGroupHead *a, SigGroupHead *b) {
+    SigGroupContainer *sg_a = NULL, *sg_b = NULL;
+
+    if (a->sig_cnt != b->sig_cnt)
+        return 0;
+
+    for (sg_a = a->head, sg_b = b->head;
+         sg_a != NULL && sg_b != NULL;
+         sg_a = sg_a->next, sg_b = sg_b->next) {
+        if (sg_a->s != sg_b->s)
+            return 0;
+    }
+
+    return 1;
+}
+

src/detect-siggroup.h

@@ -0,0 +1,11 @@
+#ifndef __DETECT_SIGGROUP_H__
+#define __DETECT_SIGGROUP_H__
+
+int SigGroupAppend(DetectAddressGroup *, Signature *);
+int SigGroupClean(DetectAddressGroup *ag);
+SigGroupHead* SigGroupHeadListGet(SigGroupHead *a);
+void SigGroupHeadListClean(void);
+void SigGroupHeadAppend(SigGroupHead *);
+
+#endif /* __DETECT_SIGGROUP_H__ */
+

src/detect-uricontent.c

@@ -30,8 +30,6 @@
 
 #include "util-unittest.h"
 
-MpmCtx mpm_ctx[MPM_INSTANCE_MAX];
-
 int DetectUricontentMatch (ThreadVars *, PatternMatcherThread *, Packet *, Signature *, SigMatch *);
 int DetectUricontentSetup (Signature *, SigMatch *, char *);
 void HttpUriRegisterTests(void);
@@ -39,6 +37,10 @@ void HttpUriRegisterTests(void);
 u_int8_t nocasetable[256];
 #define _nc(c) nocasetable[(c)]
 
+/* we use a global id for uricontent matches to be able to
+ * use just one pattern matcher thread context per thread. */
+static u_int32_t uricontent_max_id = 0;
+
 void DetectUricontentRegister (void) {
     sigmatch_table[DETECT_URICONTENT].name = "uricontent";
     sigmatch_table[DETECT_URICONTENT].Match = DetectUricontentMatch;
@@ -64,6 +66,12 @@ void DetectUricontentRegister (void) {
 #endif /* DEBUG */
 }
 
+/* pass on the uricontent_max_id */
+u_int32_t DetectUricontentMaxId(void) {
+    printf("DetectUricontentMaxId: %u\n", uricontent_max_id);
+    return uricontent_max_id;
+}
+
 /* Normalize http buffer
  *
  * Returns 0: on ok
@@ -120,7 +128,7 @@ TestWithinDistanceOffsetDepth(ThreadVars *t, PatternMatcherThread *pmt, MpmMatch
         return 1;
 
     DetectUricontentData *co = (DetectUricontentData *)nsm->ctx;
-    MpmMatch *nm = pmt->mpm_ctx[pmt->mpm_instance + MPM_INSTANCE_URIOFFSET].match[co->id].top;
+    MpmMatch *nm = pmt->mtcu.match[co->id].top;
 
     for (; nm; nm = nm->next) {
         //printf("test_nextsigmatch: (nm->offset+1) %u, (m->offset+1) %u\n", (nm->offset+1), (m->offset+1));
@@ -152,7 +160,7 @@ DoDetectUricontent(ThreadVars *t, PatternMatcherThread *pmt, Packet *p, SigMatch
     char match = 0;
 
     /* Get the top match, we already know we have one. */
-    MpmMatch *m = pmt->mpm_ctx[pmt->mpm_instance + MPM_INSTANCE_URIOFFSET].match[co->id].top;
+    MpmMatch *m = pmt->mtcu.match[co->id].top;
 
     /*  if we have within or distance coming up next, check this match
      *  for distance and/or within and check the rest of this match
@@ -224,8 +232,6 @@ int DetectUricontentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p,
 {
     u_int32_t len = 0;
     u_int32_t ret = 0;
-    u_int8_t instance = pmt->mpm_instance + MPM_INSTANCE_URIOFFSET;
-    //printf("instance %u\n", instance);
 
     //printf("scanning uricontent have %u scan %u\n", pmt->de_have_httpuri, pmt->de_scanned_httpuri);
 
@@ -241,8 +247,8 @@ int DetectUricontentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p,
         u_int8_t i;
         for (i = 0; i <= p->http_uri.cnt; i++) {
             //printf("p->http_uri.raw_size[%u] %u, %p, %s\n", i, p->http_uri.raw_size[i], p->http_uri.raw[i], p->http_uri.raw[i]);
-
-            ret += mpm_ctx[instance].Search(&mpm_ctx[instance], &pmt->mpm_ctx[instance], p->http_uri.raw[i], p->http_uri.raw_size[i]);
+            //printf("pmt->mcu %p\n", pmt->mcu);
+            ret += pmt->mcu->Search(pmt->mcu, &pmt->mtcu, p->http_uri.raw[i], p->http_uri.raw_size[i]);
             //printf("DetectUricontentMatch: ret %u\n", ret);
         }
         pmt->de_scanned_httpuri = 1;
@@ -255,7 +261,7 @@ int DetectUricontentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p,
     DetectUricontentData *co = (DetectUricontentData *)m->ctx;
 
     /* see if we had a match */
-    len = pmt->mpm_ctx[instance].match[co->id].len;
+    len = pmt->mtcu.match[co->id].len;
     if (len == 0)
         return 0;
 
@@ -263,7 +269,7 @@ int DetectUricontentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p,
     printf("uricontent \'%s\' matched %u time(s) at offsets: ", co->uricontent, len);
 
     MpmMatch *tmpm = NULL;
-    for (tmpm = pmt->mpm_ctx[mpm_instance].match[co->id].top; tmpm != NULL; tmpm = tmpm->next) {
+    for (tmpm = pmt->mtcu.match[co->id].top; tmpm != NULL; tmpm = tmpm->next) {
         printf("%u ", tmpm->offset);
     }
     printf("\n");
@@ -383,6 +389,9 @@ int DetectUricontentSetup (Signature *s, SigMatch *m, char *contentstr)
 
     SigMatchAppend(s,m,sm);
 
+    cd->id = uricontent_max_id;
+    uricontent_max_id++;
+
     if (dubbed) free(str);
     return 0;
 

src/detect-uricontent.h

@@ -24,6 +24,7 @@ typedef struct _DetectUricontentData {
 
 /* prototypes */
 void DetectUricontentRegister (void);
+u_int32_t DetectUricontentMaxId(void);
 
 #endif /* __DETECT_URICONTENT_H__ */
 

src/detect.h

@@ -1,6 +1,8 @@
 #ifndef __DETECT_H__
 #define __DETECT_H__
 
+#include "detect-address.h"
+
 #define SIG_FLAG_RECURSIVE 0x01
 #define SIG_FLAG_SP_ANY    0x02
 #define SIG_FLAG_DP_ANY    0x04
@@ -11,16 +13,22 @@ typedef struct _PatternMatcherThread {
     u_int16_t pkt_off;
     u_int8_t pkt_cnt;
 
-    /* multipattern matcher ctx */
-    MpmThreadCtx mpm_ctx[MPM_INSTANCE_MAX];
     char de_checking_distancewithin;
 
     /* http_uri stuff for uricontent */
     char de_have_httpuri;
     char de_scanned_httpuri;
 
-    /* instance of the mpm */
-    u_int8_t mpm_instance;
+    /* pointer to the current mpm ctx that is stored
+     * in a rule group head -- can be either a content
+     * or uricontent ctx.
+     *
+     * XXX rename to mpm_ctx as soon as the threading
+     * thing above is renamed as well  */
+    MpmCtx *mc;
+    MpmCtx *mcu;
+    MpmThreadCtx mtc;
+    MpmThreadCtx mtcu;
 } PatternMatcherThread;
 
 /* for now typedef them to known types, we will implement
@@ -34,9 +42,10 @@ typedef struct _Signature {
     char *msg;
     u_int8_t flags;
     u_int8_t action; 
-    SigAddress src, dst;
+    DetectAddressGroupsHead src, dst;
     SigPort sp, dp;
 
+    u_int32_t rulegroup_refcnt;
     struct _SigMatch *match;
     struct _Signature *next;
 } Signature;
@@ -58,6 +67,61 @@ typedef struct SigTableElmt {
     u_int8_t flags;
 } SigTableElmt;
 
+#define SIGGROUP_PROTO 1
+#define SIGGROUP_SP    2
+#define SIGGROUP_DP    3
+#define SIGGROUP_SRC   4
+#define SIGGROUP_DST   5
+#define SIGGROUP_FLOW  6
+#define SIGGROUP_DSIZE 7
+/* XXX more? */
+
+/* list container for signatures in the rule groups */
+typedef struct _SigGroupContainer {
+    /* ptr to the signature */
+    Signature *s;
+
+    /* list */
+    struct _SigGroupContainer *next;
+} SigGroupContainer;
+
+typedef struct _SigGroupType {
+    u_int8_t type;
+} SigGroupType;
+
+#define SIG_GROUP_HAVECONTENT    0x1
+#define SIG_GROUP_HAVEURICONTENT 0x2
+#define SIG_GROUP_INITIALIZED    0x4
+#define SIG_GROUP_COPY           0x8
+
+/* head of the list of containers, contains
+ * the pattern matcher context for the sigs
+ * that follow. */
+typedef struct _SigGroupHead {
+    u_int8_t type;
+
+    /* pattern matcher instance */
+    MpmCtx *mpm_ctx;
+    MpmCtx *mpm_uri_ctx;
+    u_int8_t flags;
+
+    /* list of signature containers */
+    SigGroupContainer *head;
+    u_int32_t sig_cnt;
+    u_int32_t refcnt;
+
+    struct _SigGroupHead *next;
+} SigGroupHead;
+
+typedef struct _SigGroupAddress {
+    u_int8_t type;
+    DetectAddressGroupsHead gh;
+} SigGroupAddress;
+
+typedef struct _SigGroupEntry {
+    SigGroupType *next;
+} SigGroupEntry;
+
 #define SIGMATCH_NOOPT  0x01
 
 void SigLoadSignatures (void);
@@ -102,5 +166,7 @@ void SigTableRegisterTests(void);
 void SigRegisterTests(void);
 void TmModuleDetectRegister (void);
 
+int SigGroupBuild(Signature *);
+
 #endif /* __DETECT_H__ */
 

src/util-mpm-trie.c

@@ -24,7 +24,7 @@
 
 /* prototypes to be exported */
 void TrieInitCtx(MpmCtx *mpm_ctx);
-void TrieThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx);
+void TrieThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int32_t);
 int TrieAddPattern(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t id);
 int TrieAddPatternNocase(MpmCtx *mpm_ctx, u_int8_t *key, u_int16_t keylen, u_int32_t id);
 u_int32_t TrieSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf, u_int16_t buflen);
@@ -784,7 +784,7 @@ void TrieDestroyCtx(MpmCtx *mpm_ctx) {
     }
 }
 
-void TrieThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx) {
+void TrieThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int32_t max_id) {
     memset(mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
 
     mpm_thread_ctx->ctx = malloc(sizeof(TrieThreadCtx));
@@ -800,7 +800,8 @@ void TrieThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx) {
      * this is done so the detect engine won't have to care about
      * what instance it's looking up in. The matches all have a
      * unique id and is the array lookup key at the same time */
-    u_int32_t keys = mpm_ctx->max_pattern_id + 1;
+    //u_int32_t keys = mpm_ctx->max_pattern_id + 1;
+    u_int32_t keys = max_id + 1;
     if (keys) {
         mpm_thread_ctx->match = malloc(keys * sizeof(MpmMatchBucket));
         if (mpm_thread_ctx->match == NULL) {
@@ -881,7 +882,7 @@ int TrieTestThreadInitCtx01 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     if (mpm_thread_ctx.memory_cnt == 2)
         result = 1;
@@ -897,7 +898,7 @@ int TrieTestThreadInitCtx02 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     TrieThreadCtx *trie_thread_ctx = (TrieThreadCtx *)mpm_thread_ctx.ctx;
 
@@ -915,7 +916,7 @@ int TrieTestInitAddPattern01 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     int ret = TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
     if (ret == 0)
@@ -932,7 +933,7 @@ int TrieTestInitAddPattern02 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
@@ -951,7 +952,7 @@ int TrieTestInitAddPattern03 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
@@ -970,7 +971,7 @@ int TrieTestInitAddPattern04 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
     TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
@@ -989,7 +990,7 @@ int TrieTestInitAddPattern05 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"Abcd", 4, 1234);
@@ -1008,7 +1009,7 @@ int TrieTestInitAddPattern06 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     TrieCtx *trie_ctx = (TrieCtx *)mpm_ctx.ctx;
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
@@ -1030,7 +1031,7 @@ int TrieTestInitAddPattern07 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1234);
 
@@ -1050,7 +1051,7 @@ int TrieTestSearch01 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcd", 4);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1070,7 +1071,7 @@ int TrieTestSearch02 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abce", 4);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1090,7 +1091,7 @@ int TrieTestSearch03 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1110,7 +1111,7 @@ int TrieTestSearch04 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"bcde", 4, 0);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1130,7 +1131,7 @@ int TrieTestSearch05 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"efgh", 4, 0);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1150,7 +1151,7 @@ int TrieTestSearch06 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
     TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"eFgH", 4, 0);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdEfGh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1171,7 +1172,7 @@ int TrieTestSearch07 (void) {
 
     TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"abcd", 4, 0);
     TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"eFgH", 4, 1);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdEfGh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1192,7 +1193,7 @@ int TrieTestSearch08 (void) {
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"abcde", 5, 0);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"bcde",  4, 1);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1212,7 +1213,7 @@ int TrieTestSearch09 (void) {
     MpmInitCtx(&mpm_ctx, MPM_TRIE);
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"ab", 2, 0);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"ab", 2);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1233,7 +1234,7 @@ int TrieTestSearch10 (void) {
 
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"bc", 2, 0);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"gh", 2, 1);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1255,7 +1256,7 @@ int TrieTestSearch11 (void) {
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"a", 1, 0);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"d", 1, 1);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"h", 1, 2);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1277,7 +1278,7 @@ int TrieTestSearch12 (void) {
     TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"d", 1, 1);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1299,7 +1300,7 @@ int TrieTestSearch13 (void) {
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"a", 1, 0);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"h", 1, 2);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1321,7 +1322,7 @@ int TrieTestSearch14 (void) {
     TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     u_int32_t cnt = TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1343,7 +1344,7 @@ int TrieTestSearch15 (void) {
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"A", 1, 0);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
 
@@ -1368,7 +1369,7 @@ int TrieTestSearch16 (void) {
     TrieAddPatternNocase(&mpm_ctx, (u_int8_t *)"A", 1, 0);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1);
     TrieAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 2);
-    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    TrieThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     TrieSearch(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
 

src/util-mpm-wumanber.c

@@ -20,7 +20,7 @@
 #include "util-unittest.h"
 
 void WmInitCtx (MpmCtx *mpm_ctx);
-void WmThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx);
+void WmThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int32_t);
 void WmDestroyCtx(MpmCtx *mpm_ctx);
 void WmThreadDestroyCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx);
 int WmAddPatternCI(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, u_int32_t id);
@@ -218,6 +218,8 @@ void WmFreePattern(MpmCtx *mpm_ctx, WmPattern *p) {
 int WmAddPattern(MpmCtx *mpm_ctx, u_int8_t *pat, u_int16_t patlen, char nocase, u_int32_t id) {
     WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx;
 
+    //printf("WmAddPattern: ctx %p \"", mpm_ctx); prt(pat, patlen); printf("\" id %u\n", id);
+
     if (patlen == 0)
         return 0;
 
@@ -422,7 +424,7 @@ int WmPreparePatterns(MpmCtx *mpm_ctx) {
     wm_ctx->parray = malloc(mpm_ctx->pattern_cnt * sizeof(WmPattern));
     if (wm_ctx->parray == NULL) goto error;
     memset(wm_ctx->parray, 0, mpm_ctx->pattern_cnt * sizeof(WmPattern));
-
+    //printf("mpm_ctx %p, parray %p\n", mpm_ctx,wm_ctx->parray);
     mpm_ctx->memory_cnt++;
     mpm_ctx->memory_size += (mpm_ctx->pattern_cnt * sizeof(WmPattern));
 
@@ -507,7 +509,6 @@ u_int32_t WmSearch2(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf
             prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2)));
             prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2));
             //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf);
-
             for (thi = hi; thi != NULL; thi = thi->nxt) {
                 p = &wm_ctx->parray[thi->idx];
 
@@ -624,6 +625,8 @@ u_int32_t WmSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf
 }
 
 void WmInitCtx (MpmCtx *mpm_ctx) {
+    //printf("WmInitCtx: mpm_ctx %p\n", mpm_ctx);
+
     memset(mpm_ctx, 0, sizeof(MpmCtx));
 
     mpm_ctx->ctx = malloc(sizeof(WmCtx));
@@ -661,7 +664,7 @@ void WmDestroyCtx(MpmCtx *mpm_ctx) {
     }
 }
 
-void WmThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx) {
+void WmThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int32_t matchsize) {
     memset(mpm_thread_ctx, 0, sizeof(MpmThreadCtx));
 
     mpm_thread_ctx->ctx = malloc(sizeof(WmThreadCtx));
@@ -677,7 +680,8 @@ void WmThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx) {
      * this is done so the detect engine won't have to care about
      * what instance it's looking up in. The matches all have a
      * unique id and is the array lookup key at the same time */
-    u_int32_t keys = mpm_ctx->max_pattern_id + 1;
+    //u_int32_t keys = mpm_ctx->max_pattern_id + 1;
+    u_int32_t keys = matchsize + 1;
     if (keys) {
         mpm_thread_ctx->match = malloc(keys * sizeof(MpmMatchBucket));
         if (mpm_thread_ctx->match == NULL) {
@@ -758,7 +762,7 @@ int WmTestThreadInitCtx01 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     if (mpm_thread_ctx.memory_cnt == 2)
         result = 1;
@@ -774,7 +778,7 @@ int WmTestThreadInitCtx02 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx.ctx;
 
@@ -792,7 +796,7 @@ int WmTestInitAddPattern01 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     int ret = WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 1234);
     if (ret == 0)
@@ -809,7 +813,7 @@ int WmTestInitAddPattern02 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx;
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 1234);
@@ -828,7 +832,7 @@ int WmTestInitAddPattern03 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx;
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 1234);
@@ -849,7 +853,7 @@ int WmTestInitAddPattern04 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx;
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 1234);
@@ -870,7 +874,7 @@ int WmTestInitAddPattern05 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx;
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 1234);
@@ -891,7 +895,7 @@ int WmTestInitAddPattern06 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
     WmCtx *wm_ctx = (WmCtx *)mpm_ctx.ctx;
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 1234);
@@ -912,7 +916,7 @@ int WmTestInitAddPattern07 (void) {
     MpmThreadCtx mpm_thread_ctx;
 
     MpmInitCtx(&mpm_ctx, MPM_WUMANBER);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 1234);
 
@@ -948,7 +952,7 @@ int WmTestSearch01 (void) {
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 0);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcd", 4);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -969,7 +973,7 @@ int WmTestSearch02 (void) {
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 0);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abce", 4);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -990,7 +994,7 @@ int WmTestSearch03 (void) {
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 1, 0);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1011,7 +1015,7 @@ int WmTestSearch04 (void) {
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"bcde", 4, 1, 0);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1032,7 +1036,7 @@ int WmTestSearch05 (void) {
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"efgh", 4, 1, 0);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1053,7 +1057,7 @@ int WmTestSearch06 (void) {
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"eFgH", 4, 1, 0);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdEfGh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1075,7 +1079,7 @@ int WmTestSearch07 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcd", 4, 0, 0);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"eFgH", 4, 1, 1);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdEfGh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1097,7 +1101,7 @@ int WmTestSearch08 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"abcde", 5, 1, 0);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"bcde",  4, 1, 1);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1118,7 +1122,7 @@ int WmTestSearch09 (void) {
 
     WmAddPattern(&mpm_ctx, (u_int8_t *)"ab", 2, 1, 0);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 1);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"ab", 2);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1140,7 +1144,7 @@ int WmTestSearch10 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"bc", 2, 1, 0);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"gh", 2, 1, 1);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 2);
 
     u_int32_t cnt = WmSearch2(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1163,7 +1167,7 @@ int WmTestSearch11 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"d", 1, 1, 1);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"h", 1, 1, 2);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     u_int32_t cnt = WmSearch1(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1186,7 +1190,7 @@ int WmTestSearch12 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"d", 1, 1, 1);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 1, 2);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     u_int32_t cnt = WmSearch1(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1209,7 +1213,7 @@ int WmTestSearch13 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1, 1);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"h", 1, 1, 2);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     u_int32_t cnt = WmSearch1(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1232,7 +1236,7 @@ int WmTestSearch14 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1, 1);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 1, 2);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     u_int32_t cnt = WmSearch1(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
     MpmMatchCleanup(&mpm_thread_ctx);
@@ -1255,7 +1259,7 @@ int WmTestSearch15 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1, 1);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 1, 2);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     WmSearch1(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
 
@@ -1281,7 +1285,7 @@ int WmTestSearch16 (void) {
     WmAddPattern(&mpm_ctx, (u_int8_t *)"de",2, 1, 1);
     WmAddPattern(&mpm_ctx, (u_int8_t *)"Z", 1, 1, 2);
     WmPreparePatterns(&mpm_ctx);
-    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx);
+    WmThreadInitCtx(&mpm_ctx, &mpm_thread_ctx, 3);
 
     WmSearch1(&mpm_ctx, &mpm_thread_ctx, (u_int8_t *)"abcdefgh", 8);
 

src/util-mpm.h

@@ -3,6 +3,13 @@
 #ifndef __UTIL_MPM_H__
 #define __UTIL_MPM_H__
 
+#define MPM_INSTANCE_CONTENT      0 /* content match */
+#define MPM_INSTANCE_URICONTENT   1
+#define MPM_INSTANCE_MAX          2
+
+#define MPM_INSTANCE_URIOFFSET    1
+
+#if 0
 #define MPM_INSTANCE_TOSERVER     0 /* content match */
 #define MPM_INSTANCE_TOCLIENT     1
 #define MPM_INSTANCE_URI_TOSERVER 2 /* uricontent match */
@@ -11,7 +18,7 @@
 #define MPM_INSTANCE_BOTH         5
 
 #define MPM_INSTANCE_URIOFFSET    2
-
+#endif
 #define MPM_ENDMATCH_SINGLE 0x01 /* A single match is sufficient */
 #define MPM_ENDMATCH_OFFSET 0x02 /* has offset setting */
 #define MPM_ENDMATCH_DEPTH  0x04 /* has depth setting */
@@ -68,7 +75,7 @@ typedef struct _MpmCtx {
     void *ctx;
 
     void (*InitCtx)(struct _MpmCtx *);
-    void (*InitThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *);
+    void (*InitThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *, u_int32_t);
     void (*DestroyCtx)(struct _MpmCtx *);
     void (*DestroyThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *);
     int  (*AddPattern)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int32_t);
@@ -99,7 +106,7 @@ typedef struct _MpmCtx {
 typedef struct MpmTableElmt {
     char *name;
     void (*InitCtx)(struct _MpmCtx *);
-    void (*InitThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *);
+    void (*InitThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *, u_int32_t);
     void (*DestroyCtx)(struct _MpmCtx *);
     void (*DestroyThreadCtx)(struct _MpmCtx *, struct _MpmThreadCtx *);
     int  (*AddPattern)(struct _MpmCtx *, u_int8_t *, u_int16_t, u_int32_t);

src/vips.c

@@ -22,6 +22,7 @@
 #include "threadvars.h"
 #include "util-binsearch.h"
 
+#include "detect-parse.h"
 #include "detect-mpm.h"
 
 #include "tm-queuehandlers.h"
@@ -288,6 +289,8 @@ int main(int argc, char **argv)
 
     BinSearchInit();
     CIDRInit();
+    SigParsePrepare();
+    PatternMatchPrepare(mpm_ctx);
 
     TmModuleReceiveNFQRegister();
     TmModuleVerdictNFQRegister();
@@ -307,8 +310,8 @@ int main(int argc, char **argv)
     MpmRegisterTests();
     SigTableRegisterTests();
     SigRegisterTests();
-    UtRunTests();
-    exit(1);
+    //UtRunTests();
+    //exit(1);
 
     //LoadConfig();
     //exit(1);
@@ -316,7 +319,6 @@ int main(int argc, char **argv)
     /* initialize packet queues */
     memset(&packet_q,0,sizeof(packet_q));
     memset(&trans_q, 0,sizeof(trans_q));
-    //memset(&th_v, 0,sizeof(th_v));
 
     /* pre allocate packets */
     printf("Preallocating packets... packet size %u\n", sizeof(Packet));
@@ -345,6 +347,7 @@ int main(int argc, char **argv)
 
     SigLoadSignatures();
 
+    /* create the threads */
     ThreadVars *tv_receivenfq = TmThreadCreate("ReceiveNFQ","packetpool","packetpool","pickup-queue","simple","1slot_noinout");
     if (tv_receivenfq == NULL) {
         printf("ERROR: TmThreadsCreate failed\n");
@@ -530,7 +533,6 @@ int main(int argc, char **argv)
     }
     TmThreadAppend(&tv_flowmgr);
 
-
     while(1) {
         if (sigflags) {
             printf("signal received\n");
@@ -573,6 +575,7 @@ int main(int argc, char **argv)
     FlowPrintFlows();
     FlowShutdown();
 
+    SigGroupCleanup();
     SigCleanSignatures();
 
     pthread_exit(NULL);