From 37442a8a840d91163cc03f67483f4e93907cc6a4 Mon Sep 17 00:00:00 2001
From: Victor Julien <victor@inliniac.net>
Date: Thu, 17 Jun 2010 14:04:59 +0200
Subject: [PATCH] Prefilter signatures before fully scanning them.

---
 src/decode.h                 |  98 +++++++++++--------
 src/detect-content.c         |  10 ++
 src/detect-distance.c        |   4 +-
 src/detect-engine-proto.c    |  17 +++-
 src/detect-engine-siggroup.c | 179 +++++++++++++++++++++++------------
 src/detect-engine-siggroup.h |   2 +
 src/detect-parse.c           |  14 +++
 src/detect.c                 | 171 ++++++++++++++++++---------------
 src/detect.h                 | 107 ++++++++++++++-------
 src/util-unittest-helper.c   |  12 ++-
 10 files changed, 391 insertions(+), 223 deletions(-)

diff --git a/src/decode.h b/src/decode.h
index 6524278f1f..9b94703def 100644
--- a/src/decode.h
+++ b/src/decode.h
@@ -222,6 +222,22 @@ typedef struct PktVar_ {
 /* forward declartion since Packet struct definition requires this */
 struct PacketQueue_;
 
+/* sizes of the members:
+ * src: 17 bytes
+ * dst: 17 bytes
+ * sp/type: 1 byte
+ * dp/code: 1 byte
+ * proto: 1 byte
+ * recurs: 1 byte
+ *
+ * sum of above: 38 bytes
+ *
+ * flow ptr: 4/8 bytes
+ * flags: 1 byte
+ * flowflags: 1 byte
+ *
+ * sum of above 44/48 bytes
+ */
 typedef struct Packet_
 {
     /* Addresses, Ports and protocol
@@ -242,51 +258,35 @@ typedef struct Packet_
      * has the exact same tuple as the lower levels */
     uint8_t recursion_level;
 
-    struct timeval ts;
+    /*Pkt Flags*/
+    uint8_t flags;
+    /* flow */
+    uint8_t flowflags;
+    struct Flow_ *flow;
 
-    /* ready to set verdict counter, only set in root */
-    uint8_t rtv_cnt;
-    /* tunnel packet ref count */
-    uint8_t tpr_cnt;
-    SCMutex mutex_rtv_cnt;
-    /* tunnel stuff */
-    uint8_t tunnel_proto;
-    /* tunnel XXX convert to bitfield*/
-    char tunnel_pkt;
-    char tunnel_verdicted;
+    struct timeval ts;
 
-    /* nfq stuff */
+    union {
+        /* nfq stuff */
 #ifdef NFQ
-    NFQPacketVars nfq_v;
+        NFQPacketVars nfq_v;
 #endif /* NFQ */
 
-    /** libpcap vars: shared by Pcap Live mode and Pcap File mode */
-    PcapPacketVars pcap_v;
+        /** libpcap vars: shared by Pcap Live mode and Pcap File mode */
+        PcapPacketVars pcap_v;
+    };
 
     /** data linktype in host order */
     int datalink;
 
-    /* storage: maximum ip packet size + link header */
-    uint8_t pkt[IPV6_HEADER_LEN + 65536 + 28];
-    uint32_t pktlen;
-
-    /* flow */
-    struct Flow_ *flow;
-    uint8_t flowflags;
-
-    /*Pkt Flags*/
-    uint8_t flags;
+    /* IPS action to take */
+    uint8_t action;
 
     /* pkt vars */
     PktVar *pktvar;
 
     /* header pointers */
     EthernetHdr *ethh;
-    PPPHdr *ppph;
-    PPPOESessionHdr *pppoesh;
-    PPPOEDiscoveryHdr *pppoedh;
-    GREHdr *greh;
-    VLANHdr *vlanh;
 
     IPV4Hdr *ip4h;
     IPV4Vars ip4vars;
@@ -297,6 +297,14 @@ typedef struct Packet_
     IPV6Cache ip6c;
     IPV6ExtHdrs ip6eh;
 
+    TCPHdr *tcph;
+    TCPVars tcpvars;
+    TCPCache tcpc;
+
+    UDPHdr *udph;
+    UDPVars udpvars;
+    UDPCache udpc;
+
     ICMPV4Hdr *icmpv4h;
     ICMPV4Cache icmpv4c;
     ICMPV4Vars icmpv4vars;
@@ -305,27 +313,28 @@ typedef struct Packet_
     ICMPV6Cache icmpv6c;
     ICMPV6Vars icmpv6vars;
 
-    TCPHdr *tcph;
-    TCPVars tcpvars;
-    TCPCache tcpc;
+    PPPHdr *ppph;
+    PPPOESessionHdr *pppoesh;
+    PPPOEDiscoveryHdr *pppoedh;
 
-    UDPHdr *udph;
-    UDPVars udpvars;
-    UDPCache udpc;
+    GREHdr *greh;
+
+    VLANHdr *vlanh;
 
     /* ptr to the payload of the packet
      * with it's length. */
     uint8_t *payload;
     uint16_t payload_len;
 
+    /* storage: maximum ip packet size + link header */
+    uint8_t pkt[IPV6_HEADER_LEN + 65536 + 28];
+    uint32_t pktlen;
+
     /* decoder events: review how many events we have */
     uint8_t events[(DECODE_EVENT_MAX / 8) + 1];
 
     PacketAlerts alerts;
 
-    /* IPS action to take */
-    uint8_t action;
-
     /** packet number in the pcap file, matches wireshark */
     uint64_t pcap_cnt;
 
@@ -333,6 +342,17 @@ typedef struct Packet_
     struct Packet_ *next;
     struct Packet_ *prev;
 
+    /* ready to set verdict counter, only set in root */
+    uint8_t rtv_cnt;
+    /* tunnel packet ref count */
+    uint8_t tpr_cnt;
+    SCMutex mutex_rtv_cnt;
+    /* tunnel stuff */
+    uint8_t tunnel_proto;
+    /* tunnel XXX convert to bitfield*/
+    char tunnel_pkt;
+    char tunnel_verdicted;
+
     /* tunnel/encapsulation handling */
     struct Packet_ *root; /* in case of tunnel this is a ptr
                            * to the 'real' packet, the one we
diff --git a/src/detect-content.c b/src/detect-content.c
index 249b3617bb..cc4a316959 100644
--- a/src/detect-content.c
+++ b/src/detect-content.c
@@ -30,9 +30,11 @@
 #include "detect-uricontent.h"
 #include "detect-engine-mpm.h"
 #include "detect-engine.h"
+#include "detect-engine-state.h"
 #include "detect-parse.h"
 #include "util-mpm.h"
 #include "flow.h"
+#include "flow-util.h"
 #include "flow-var.h"
 #include "detect-flow.h"
 #include "app-layer.h"
@@ -1743,10 +1745,16 @@ static int SigTest76TestBug134(void)
     uint16_t buflen = strlen((char *)buf);
     Packet *p = UTHBuildPacket( buf, buflen, IPPROTO_TCP);
     int result = 0;
+    Flow f;
+
+    memset(&f, 0, sizeof(Flow));
+    FLOW_INITIALIZE(&f);
 
     p->dp = 515;
     p->flowflags |= FLOW_PKT_ESTABLISHED;
     p->flowflags |= FLOW_PKT_TOSERVER;
+    p->flow = &f;
+
     char sig[] = "alert tcp any any -> any 515 "
             "(msg:\"detect IFS\"; flow:to_server,established; content:\"${IFS}\";"
             " depth:50; offset:0; sid:900091; rev:1;)";
@@ -1759,6 +1767,8 @@ static int SigTest76TestBug134(void)
 end:
     if (p != NULL)
         UTHFreePacket(p);
+
+    FLOW_DESTROY(&f);
     return result;
 }
 
diff --git a/src/detect-distance.c b/src/detect-distance.c
index 9b1a294888..fa3c6fd392 100644
--- a/src/detect-distance.c
+++ b/src/detect-distance.c
@@ -278,8 +278,8 @@ int DetectDistanceTestPacket01 (void) {
     if (p == NULL)
         goto end;
 
-    char sig[] = "alert tcp any any -> any any (msg:\"suricata test\"; flow:"
-                 "from_server,established; byte_jump:1,2; content:\"|00|\"; "
+    char sig[] = "alert tcp any any -> any any (msg:\"suricata test\"; "
+                    "byte_jump:1,2; content:\"|00|\"; "
                     "within:1; distance:2; sid:98711212; rev:1;)";
 
     p->flowflags = FLOW_PKT_ESTABLISHED | FLOW_PKT_TOCLIENT;
diff --git a/src/detect-engine-proto.c b/src/detect-engine-proto.c
index a904785ca2..62f3c320f9 100644
--- a/src/detect-engine-proto.c
+++ b/src/detect-engine-proto.c
@@ -29,15 +29,20 @@
 
 #include "decode.h"
 #include "detect.h"
+
+#include "app-layer-parser.h"
+
+#include "flow-util.h"
 #include "flow-var.h"
 
+#include "detect-engine-siggroup.h"
+#include "detect-engine-state.h"
+
 #include "util-cidr.h"
 #include "util-byte.h"
 #include "util-unittest.h"
 #include "util-debug.h"
 
-#include "detect-engine-siggroup.h"
-
 /*Prototypes*/
 void DetectProtoTests (void);
 
@@ -349,9 +354,15 @@ static int DetectProtoTestSig01(void) {
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx;
     int result = 0;
+    Flow f;
 
+    memset(&f, 0, sizeof(Flow));
     memset(&th_v, 0, sizeof(th_v));
     memset(&p, 0, sizeof(p));
+
+    FLOW_INITIALIZE(&f);
+
+    p.flow = &f;
     p.src.family = AF_INET;
     p.dst.family = AF_INET;
     p.proto = IPPROTO_TCP;
@@ -400,6 +411,8 @@ static int DetectProtoTestSig01(void) {
     result = 1;
 
 cleanup:
+    FLOW_DESTROY(&f);
+
     SigGroupCleanup(de_ctx);
     SigCleanSignatures(de_ctx);
 
diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c
index 4dc17ac723..0a0cc790a7 100644
--- a/src/detect-engine-siggroup.c
+++ b/src/detect-engine-siggroup.c
@@ -65,18 +65,6 @@ static uint32_t detect_siggroup_matcharray_memory = 0;
 static uint32_t detect_siggroup_matcharray_init_cnt = 0;
 static uint32_t detect_siggroup_matcharray_free_cnt = 0;
 
-static SigGroupHeadInitData *SigGroupHeadInitDataAlloc(uint32_t size) {
-    SigGroupHeadInitData *sghid = SCMalloc(sizeof(SigGroupHeadInitData));
-    if (sghid == NULL)
-        return NULL;
-
-    memset(sghid, 0x00, sizeof(SigGroupHeadInitData));
-
-    detect_siggroup_head_initdata_init_cnt++;
-    detect_siggroup_head_initdata_memory += sizeof(SigGroupHeadInitData);
-    return sghid;
-}
-
 void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid) {
     if (sghid->content_array != NULL) {
         SCFree(sghid->content_array);
@@ -88,12 +76,59 @@ void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid) {
         sghid->uri_content_array = NULL;
         sghid->uri_content_size = 0;
     }
+    if (sghid->sig_array != NULL) {
+        SCFree(sghid->sig_array);
+        sghid->sig_array = NULL;
+
+        detect_siggroup_sigarray_free_cnt++;
+        detect_siggroup_sigarray_memory -= sghid->sig_size;
+    }
     SCFree(sghid);
 
     detect_siggroup_head_initdata_free_cnt++;
     detect_siggroup_head_initdata_memory -= sizeof(SigGroupHeadInitData);
 }
 
+static SigGroupHeadInitData *SigGroupHeadInitDataAlloc(uint32_t size) {
+    SigGroupHeadInitData *sghid = SCMalloc(sizeof(SigGroupHeadInitData));
+    if (sghid == NULL)
+        return NULL;
+
+    memset(sghid, 0x00, sizeof(SigGroupHeadInitData));
+
+    detect_siggroup_head_initdata_init_cnt++;
+    detect_siggroup_head_initdata_memory += sizeof(SigGroupHeadInitData);
+
+    /* initialize the signature bitarray */
+    sghid->sig_size = size;
+    if ( (sghid->sig_array = SCMalloc(sghid->sig_size)) == NULL)
+        goto error;
+
+    memset(sghid->sig_array, 0, sghid->sig_size);
+
+    detect_siggroup_sigarray_init_cnt++;
+    detect_siggroup_sigarray_memory += sghid->sig_size;
+
+    return sghid;
+error:
+    SigGroupHeadInitDataFree(sghid);
+    return NULL;
+}
+
+void SigGroupHeadStore(DetectEngineCtx *de_ctx, SigGroupHead *sgh) {
+    //printf("de_ctx->sgh_array_cnt %u, de_ctx->sgh_array_size %u, de_ctx->sgh_array %p\n", de_ctx->sgh_array_cnt, de_ctx->sgh_array_size, de_ctx->sgh_array);
+    if (de_ctx->sgh_array_cnt < de_ctx->sgh_array_size) {
+        de_ctx->sgh_array[de_ctx->sgh_array_cnt] = sgh;
+    } else {
+        de_ctx->sgh_array = SCRealloc(de_ctx->sgh_array, sizeof(SigGroupHead *) * (16 + de_ctx->sgh_array_size));
+        if (de_ctx->sgh_array == NULL)
+            return;
+        de_ctx->sgh_array_size += 10;
+        de_ctx->sgh_array[de_ctx->sgh_array_cnt] = sgh;
+    }
+    de_ctx->sgh_array_cnt++;
+}
+
 /**
  * \brief Alloc a SigGroupHead and its signature bit_array.
  *
@@ -103,7 +138,7 @@ void SigGroupHeadInitDataFree(SigGroupHeadInitData *sghid) {
  * \retval sgh Pointer to the newly init SigGroupHead on success; or NULL in
  *             case of error.
  */
-static SigGroupHead *SigGroupHeadAlloc(uint32_t size)
+static SigGroupHead *SigGroupHeadAlloc(DetectEngineCtx *de_ctx, uint32_t size)
 {
     SigGroupHead *sgh = SCMalloc(sizeof(SigGroupHead));
     if (sgh == NULL)
@@ -117,15 +152,6 @@ static SigGroupHead *SigGroupHeadAlloc(uint32_t size)
     detect_siggroup_head_init_cnt++;
     detect_siggroup_head_memory += sizeof(SigGroupHead);
 
-    /* initialize the signature bitarray */
-    sgh->sig_size = size;
-    if ( (sgh->sig_array = SCMalloc(sgh->sig_size)) == NULL)
-        goto error;
-    memset(sgh->sig_array, 0, sgh->sig_size);
-
-    detect_siggroup_sigarray_init_cnt++;
-    detect_siggroup_sigarray_memory += sgh->sig_size;
-
     return sgh;
 
 error:
@@ -148,14 +174,6 @@ void SigGroupHeadFree(SigGroupHead *sgh)
 
     PatternMatchDestroyGroup(sgh);
 
-    if (sgh->sig_array != NULL) {
-        SCFree(sgh->sig_array);
-        sgh->sig_array = NULL;
-
-        detect_siggroup_sigarray_free_cnt++;
-        detect_siggroup_sigarray_memory -= sgh->sig_size;
-    }
-
     if (sgh->match_array != NULL) {
         detect_siggroup_matcharray_free_cnt++;
         detect_siggroup_matcharray_memory -= (sgh->sig_cnt * sizeof(Signature *));
@@ -575,11 +593,11 @@ uint32_t SigGroupHeadHashFunc(HashListTable *ht, void *data, uint16_t datalen)
 
     SCLogDebug("hashing sgh %p (mpm_content_maxlen %u)", sgh, sgh->mpm_content_maxlen);
 
-    for (b = 0; b < sgh->sig_size; b++)
-        hash += sgh->sig_array[b];
+    for (b = 0; b < sgh->init->sig_size; b++)
+        hash += sgh->init->sig_array[b];
 
     hash %= ht->array_size;
-    SCLogDebug("hash %"PRIu32" (sig_size %"PRIu32")", hash, sgh->sig_size);
+    SCLogDebug("hash %"PRIu32" (sig_size %"PRIu32")", hash, sgh->init->sig_size);
     return hash;
 }
 
@@ -604,10 +622,10 @@ char SigGroupHeadCompareFunc(void *data1, uint16_t len1, void *data2,
     if (data1 == NULL || data2 == NULL)
         return 0;
 
-    if (sgh1->sig_size != sgh2->sig_size)
+    if (sgh1->init->sig_size != sgh2->init->sig_size)
         return 0;
 
-    if (memcmp(sgh1->sig_array, sgh2->sig_array, sgh1->sig_size) != 0)
+    if (memcmp(sgh1->init->sig_array, sgh2->init->sig_array, sgh1->init->sig_size) != 0)
         return 0;
 
     return 1;
@@ -866,16 +884,17 @@ static void SigGroupHeadFreeSigArraysHash2(DetectEngineCtx *de_ctx,
 
     for (htb = HashListTableGetListHead(ht);
          htb != NULL;
-         htb = HashListTableGetListNext(htb)) {
+         htb = HashListTableGetListNext(htb))
+    {
         sgh = (SigGroupHead *)HashListTableGetListData(htb);
 
-        if (sgh->sig_array != NULL) {
+        if (sgh->init->sig_array != NULL) {
             detect_siggroup_sigarray_free_cnt++;
-            detect_siggroup_sigarray_memory -= sgh->sig_size;
+            detect_siggroup_sigarray_memory -= sgh->init->sig_size;
 
-            SCFree(sgh->sig_array);
-            sgh->sig_array = NULL;
-            sgh->sig_size = 0;
+            SCFree(sgh->init->sig_array);
+            sgh->init->sig_array = NULL;
+            sgh->init->sig_size = 0;
         }
 
         if (sgh->init != NULL) {
@@ -905,15 +924,6 @@ static void SigGroupHeadFreeSigArraysHash(DetectEngineCtx *de_ctx,
          htb = HashListTableGetListNext(htb)) {
         sgh = (SigGroupHead *)HashListTableGetListData(htb);
 
-        if (sgh->sig_array != NULL) {
-            detect_siggroup_sigarray_free_cnt++;
-            detect_siggroup_sigarray_memory -= sgh->sig_size;
-
-            SCFree(sgh->sig_array);
-            sgh->sig_array = NULL;
-            sgh->sig_size = 0;
-        }
-
         if (sgh->init != NULL) {
             SigGroupHeadInitDataFree(sgh->init);
             sgh->init = NULL;
@@ -987,13 +997,13 @@ int SigGroupHeadAppendSig(DetectEngineCtx *de_ctx, SigGroupHead **sgh,
 
     /* see if we have a head already */
     if (*sgh == NULL) {
-        *sgh = SigGroupHeadAlloc(DetectEngineGetMaxSigId(de_ctx) / 8 + 1);
+        *sgh = SigGroupHeadAlloc(de_ctx, DetectEngineGetMaxSigId(de_ctx) / 8 + 1);
         if (*sgh == NULL)
             goto error;
     }
 
     /* enable the sig in the bitarray */
-    (*sgh)->sig_array[s->num / 8] |= 1 << (s->num % 8);
+    (*sgh)->init->sig_array[s->num / 8] |= 1 << (s->num % 8);
 
     /* update maxlen for mpm */
     if (s->flags & SIG_FLAG_MPM) {
@@ -1037,8 +1047,8 @@ int SigGroupHeadClearSigs(SigGroupHead *sgh)
     if (sgh == NULL)
         return 0;
 
-    if (sgh->sig_array != NULL)
-        memset(sgh->sig_array, 0, sgh->sig_size);
+    if (sgh->init->sig_array != NULL)
+        memset(sgh->init->sig_array, 0, sgh->init->sig_size);
 
     sgh->sig_cnt = 0;
 
@@ -1064,14 +1074,14 @@ int SigGroupHeadCopySigs(DetectEngineCtx *de_ctx, SigGroupHead *src, SigGroupHea
         return 0;
 
     if (*dst == NULL) {
-        *dst = SigGroupHeadAlloc(DetectEngineGetMaxSigId(de_ctx) / 8 + 1);
+        *dst = SigGroupHeadAlloc(de_ctx, DetectEngineGetMaxSigId(de_ctx) / 8 + 1);
         if (*dst == NULL)
             goto error;
     }
 
     /* do the copy */
-    for (idx = 0; idx < src->sig_size; idx++)
-        (*dst)->sig_array[idx] = (*dst)->sig_array[idx] | src->sig_array[idx];
+    for (idx = 0; idx < src->init->sig_size; idx++)
+        (*dst)->init->sig_array[idx] = (*dst)->init->sig_array[idx] | src->init->sig_array[idx];
 
     if (src->mpm_content_maxlen != 0) {
         if ((*dst)->mpm_content_maxlen == 0)
@@ -1111,7 +1121,7 @@ void SigGroupHeadSetSigCnt(SigGroupHead *sgh, uint32_t max_idx)
 
     sgh->sig_cnt = 0;
     for (sig = 0; sig < max_idx + 1; sig++) {
-        if (sgh->sig_array[sig / 8] & (1 << (sig % 8)))
+        if (sgh->init->sig_array[sig / 8] & (1 << (sig % 8)))
             sgh->sig_cnt++;
     }
 
@@ -1192,8 +1202,8 @@ void SigGroupHeadPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
     uint32_t u;
 
     SCLogDebug("The Signatures present in this SigGroupHead are: ");
-    for (u = 0; u < (sgh->sig_size * 8); u++) {
-        if (sgh->sig_array[u / 8] & (1 << (u % 8))) {
+    for (u = 0; u < (sgh->init->sig_size * 8); u++) {
+        if (sgh->init->sig_array[u / 8] & (1 << (u % 8))) {
             SCLogDebug("%" PRIu32, u);
             printf("s->num %"PRIu16" ", u);
         }
@@ -1532,7 +1542,7 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
     detect_siggroup_matcharray_memory += (sgh->sig_cnt * sizeof(Signature *));
 
     for (sig = 0; sig < max_idx + 1; sig++) {
-        if (!(sgh->sig_array[(sig / 8)] & (1 << (sig % 8))) )
+        if (!(sgh->init->sig_array[(sig / 8)] & (1 << (sig % 8))) )
             continue;
 
         s = de_ctx->sig_array[sig];
@@ -1546,6 +1556,49 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
     return 0;
 }
 
+int SigGroupHeadBuildHeadArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
+{
+    Signature *s = NULL;
+    uint32_t idx = 0;
+    uint32_t sig = 0;
+
+    if (sgh == NULL)
+        return 0;
+
+    BUG_ON(sgh->head_array != NULL);
+
+    sgh->head_array = SCMalloc(sgh->sig_cnt * sizeof(SignatureHeader));
+    if (sgh->head_array == NULL)
+        return -1;
+
+    memset(sgh->head_array, 0, sgh->sig_cnt * sizeof(SignatureHeader));
+
+    detect_siggroup_matcharray_init_cnt++;
+    detect_siggroup_matcharray_memory += (sgh->sig_cnt * sizeof(SignatureHeader *));
+
+    for (sig = 0; sig < sgh->sig_cnt; sig++) {
+        s = sgh->match_array[sig];
+        if (s == NULL)
+            continue;
+
+        sgh->head_array[idx].flags = s->flags;
+        sgh->head_array[idx].mpm_pattern_id = s->mpm_pattern_id;
+        sgh->head_array[idx].alproto = s->alproto;
+        sgh->head_array[idx].num = s->num;
+        sgh->head_array[idx].full_sig = s;
+
+        BUG_ON(s->flags != sgh->head_array[idx].flags);
+        BUG_ON(s->alproto != sgh->head_array[idx].alproto);
+        BUG_ON(s->mpm_pattern_id != sgh->head_array[idx].mpm_pattern_id);
+        BUG_ON(s->num != sgh->head_array[idx].num);
+        BUG_ON(s != sgh->head_array[idx].full_sig);
+
+        idx++;
+    }
+
+    return 0;
+}
+
 /**
  * \brief Check if a SigGroupHead contains a Signature, whose sid is sent as an
  *        argument.
@@ -1572,12 +1625,12 @@ int SigGroupHeadContainsSigId(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
     }
 
     for (sig = 0; sig < max_sid; sig++) {
-        if (sgh->sig_array == NULL) {
+        if (sgh->init->sig_array == NULL) {
             SCReturnInt(0);
         }
 
         /* Check if the SigGroupHead has an entry for the sid */
-        if ( !(sgh->sig_array[sig / 8] & (1 << (sig % 8))) )
+        if ( !(sgh->init->sig_array[sig / 8] & (1 << (sig % 8))) )
             continue;
 
         /* If we have reached here, we have an entry for sid in the SigGrouHead.
diff --git a/src/detect-engine-siggroup.h b/src/detect-engine-siggroup.h
index b86aaafa71..0d33a91279 100644
--- a/src/detect-engine-siggroup.h
+++ b/src/detect-engine-siggroup.h
@@ -83,4 +83,6 @@ int SigGroupHeadContainsSigId (DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 void SigGroupHeadRegisterTests(void);
 void SigGroupHeadPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh);
 
+void SigGroupHeadStore(DetectEngineCtx *, SigGroupHead *);
+int SigGroupHeadBuildHeadArray(DetectEngineCtx *, SigGroupHead *);
 #endif /* __DETECT_ENGINE_SIGGROUP_H__ */
diff --git a/src/detect-parse.c b/src/detect-parse.c
index 476487cf43..a3c96c7c3c 100644
--- a/src/detect-parse.c
+++ b/src/detect-parse.c
@@ -1089,6 +1089,13 @@ Signature *SigInit(DetectEngineCtx *de_ctx, char *sigstr) {
         }
     }
 
+    if (sig->umatch)
+        sig->flags |= SIG_FLAG_UMATCH;
+    if (sig->dmatch)
+        sig->flags |= SIG_FLAG_AMATCH;
+    if (sig->amatch)
+        sig->flags |= SIG_FLAG_AMATCH;
+
     SCLogDebug("sig %"PRIu32" SIG_FLAG_APPLAYER: %s, SIG_FLAG_PACKET: %s",
         sig->id, sig->flags & SIG_FLAG_APPLAYER ? "set" : "not set",
         sig->flags & SIG_FLAG_PACKET ? "set" : "not set");
@@ -1267,6 +1274,13 @@ Signature *SigInitReal(DetectEngineCtx *de_ctx, char *sigstr) {
         }
     }
 
+    if (sig->umatch)
+        sig->flags |= SIG_FLAG_UMATCH;
+    if (sig->dmatch)
+        sig->flags |= SIG_FLAG_AMATCH;
+    if (sig->amatch)
+        sig->flags |= SIG_FLAG_AMATCH;
+
     SCLogDebug("sig %"PRIu32" SIG_FLAG_APPLAYER: %s, SIG_FLAG_PACKET: %s",
         sig->id, sig->flags & SIG_FLAG_APPLAYER ? "set" : "not set",
         sig->flags & SIG_FLAG_PACKET ? "set" : "not set");
diff --git a/src/detect.c b/src/detect.c
index b8c8b93de6..2a8a3ba8cd 100644
--- a/src/detect.c
+++ b/src/detect.c
@@ -204,28 +204,6 @@ void DetectExitPrintStats(ThreadVars *tv, void *data) {
         (float)(det_ctx->pkts_uri_searched/(float)(det_ctx->uris)*100));
 }
 
-int SghHasSig(DetectEngineCtx *de_ctx, SigGroupHead *sgh, uint32_t sid) {
-    if (sgh == NULL) {
-        return 0;
-    }
-
-    uint32_t sig;
-    for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
-        if (!(sgh->sig_array[(sig/8)] & (1<<(sig%8))))
-            continue;
-
-        Signature *s = de_ctx->sig_array[sig];
-        if (s == NULL)
-            continue;
-
-        if (sid == s->id) {
-            return 1;
-        }
-    }
-
-    return 0;
-}
-
 /** \brief Create the path if default-rule-path was specified
  *  \param sig_file The name of the file
  *  \retval str Pointer to the string path + sig_file
@@ -441,6 +419,12 @@ int SigLoadSignatures (DetectEngineCtx *de_ctx, char *sig_file)
  *  \param de_state_start flag to indicate if we're at the start of a stateful run
  *  \param p packet
  *  \param alproto application layer protocol
+ *
+ *  Order of SignatureHeader access:
+ *  1. flags
+ *  2. alproto
+ *  3. mpm_pattern_id
+ *  4. num
  */
 static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx,
         DetectEngineThreadCtx *det_ctx, char de_state_start, Packet *p,
@@ -452,26 +436,10 @@ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx,
     det_ctx->match_array_cnt = 0;
 
     for (i = 0; i < det_ctx->sgh->sig_cnt; i++) {
-        Signature *s = det_ctx->sgh->match_array[i];
+        SignatureHeader *s = &det_ctx->sgh->head_array[i];
 
-        if (s->flags & SIG_FLAG_MPM) {
-            /* filter out sigs that want pattern matches, but
-             * have no matches */
-            if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id / 8)] & (1<<(s->mpm_pattern_id % 8))) &&
-                    (s->flags & SIG_FLAG_MPM) && !(s->flags & SIG_FLAG_MPM_NEGCONTENT)) {
-                SCLogDebug("mpm sig without matches (pat id check in content).");
-                continue;
-            }
-        }
-
-        /* de_state check, filter out all signatures that already had a match before
-         * or just partially match */
-        if (de_state_start == FALSE) {
-            if (s->amatch != NULL || s->umatch != NULL || s->dmatch != NULL) {
-                if (det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NEW) {
-                    continue;
-                }
-            }
+        if (s->flags & SIG_FLAG_FLOW && !p->flow) {
+            continue;
         }
 
         /* filter out the sigs that inspect the payload, if packet
@@ -488,43 +456,29 @@ static void SigMatchSignaturesBuildMatchArray(DetectEngineCtx *de_ctx,
             }
         }
 
-        /* check the source & dst port in the sig */
-        if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) {
-            if (!(s->flags & SIG_FLAG_DP_ANY)) {
-                DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp);
-                if (dport == NULL) {
-                    SCLogDebug("dport didn't match.");
-                    continue;
-                }
-            }
-            if (!(s->flags & SIG_FLAG_SP_ANY)) {
-                DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp);
-                if (sport == NULL) {
-                    SCLogDebug("sport didn't match.");
-                    continue;
-                }
+        if (s->flags & SIG_FLAG_MPM && !(s->flags & SIG_FLAG_MPM_NEGCONTENT)) {
+            /* filter out sigs that want pattern matches, but
+             * have no matches */
+            if (!(det_ctx->pmq.pattern_id_bitarray[(s->mpm_pattern_id / 8)] & (1<<(s->mpm_pattern_id % 8)))) {
+                SCLogDebug("mpm sig without matches (pat id %"PRIu32" check in content).", s->mpm_pattern_id);
+                continue;
             }
         }
 
-        /* check the destination address */
-        if (!(s->flags & SIG_FLAG_DST_ANY)) {
-            DetectAddress *daddr = DetectAddressLookupInHead(&s->dst,&p->dst);
-            if (daddr == NULL) {
-                SCLogDebug("dst addr didn't match.");
-                    continue;
-            }
-        }
-        /* check the source address */
-        if (!(s->flags & SIG_FLAG_SRC_ANY)) {
-            DetectAddress *saddr = DetectAddressLookupInHead(&s->src,&p->src);
-            if (saddr == NULL) {
-                SCLogDebug("src addr didn't match.");
+        /* de_state check, filter out all signatures that already had a match before
+         * or just partially match */
+        if (s->flags & SIG_FLAG_AMATCH || s->flags & SIG_FLAG_UMATCH ||
+                s->flags & SIG_FLAG_DMATCH)
+        {
+            if (de_state_start == FALSE) {
+                if (det_ctx->de_state_sig_array[s->num] != DE_STATE_MATCH_NEW) {
                     continue;
+                }
             }
         }
 
         /* okay, store it */
-        det_ctx->match_array[det_ctx->match_array_cnt] = s;
+        det_ctx->match_array[det_ctx->match_array_cnt] = s->full_sig;
         det_ctx->match_array_cnt++;
     }
 }
@@ -759,6 +713,41 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
         s = det_ctx->match_array[idx];
         SCLogDebug("inspecting signature id %"PRIu32"", s->id);
 
+        /* check the source & dst port in the sig */
+        if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) {
+            if (!(s->flags & SIG_FLAG_DP_ANY)) {
+                DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp);
+                if (dport == NULL) {
+                    SCLogDebug("dport didn't match.");
+                    continue;
+                }
+            }
+            if (!(s->flags & SIG_FLAG_SP_ANY)) {
+                DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp);
+                if (sport == NULL) {
+                    SCLogDebug("sport didn't match.");
+                    continue;
+                }
+            }
+        }
+
+        /* check the destination address */
+        if (!(s->flags & SIG_FLAG_DST_ANY)) {
+            DetectAddress *daddr = DetectAddressLookupInHead(&s->dst,&p->dst);
+            if (daddr == NULL) {
+                SCLogDebug("dst addr didn't match.");
+                    continue;
+            }
+        }
+        /* check the source address */
+        if (!(s->flags & SIG_FLAG_SRC_ANY)) {
+            DetectAddress *saddr = DetectAddressLookupInHead(&s->src,&p->src);
+            if (saddr == NULL) {
+                SCLogDebug("src addr didn't match.");
+                    continue;
+            }
+        }
+
         SCLogDebug("s->amatch %p, s->umatch %p", s->amatch, s->umatch);
         if ((s->amatch != NULL || s->umatch != NULL || s->dmatch != NULL) && p->flow != NULL) {
             if (de_state_start == TRUE) {
@@ -1891,7 +1880,7 @@ int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressHead *hea
          * and build the temporary destination address list for it */
         uint32_t sig;
         for (sig = 0; sig < de_ctx->sig_array_len; sig++) {
-            if (!(gr->sh->sig_array[(sig/8)] & (1<<(sig%8))))
+            if (!(gr->sh->init->sig_array[(sig/8)] & (1<<(sig%8))))
                 continue;
 
             tmp_s = de_ctx->sig_array[sig];
@@ -2033,6 +2022,7 @@ int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressHead *hea
                 }
 
                 SigGroupHeadHashAdd(de_ctx, sgr->sh);
+                SigGroupHeadStore(de_ctx, sgr->sh);
                 de_ctx->gh_unique++;
             } else {
                 SCLogDebug("calling SigGroupHeadFree sgr %p, sgr->sh %p", sgr, sgr->sh);
@@ -2083,7 +2073,7 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd
          * and build the temporary destination address list for it */
         uint32_t sig;
         for (sig = 0; sig < de_ctx->sig_array_len; sig++) {
-            if (!(src_gr->sh->sig_array[(sig/8)] & (1<<(sig%8))))
+            if (!(src_gr->sh->init->sig_array[(sig/8)] & (1<<(sig%8))))
                 continue;
 
             tmp_s = de_ctx->sig_array[sig];
@@ -2145,7 +2135,7 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd
 
                 uint32_t sig2;
                 for (sig2 = 0; sig2 < max_idx+1; sig2++) {
-                    if (!(dst_gr->sh->sig_array[(sig2/8)] & (1<<(sig2%8))))
+                    if (!(dst_gr->sh->init->sig_array[(sig2/8)] & (1<<(sig2%8))))
                         continue;
 
                     Signature *s = de_ctx->sig_array[sig2];
@@ -2201,7 +2191,7 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd
                         DetectPortDpHashReset(de_ctx);
                         uint32_t sig2;
                         for (sig2 = 0; sig2 < max_idx+1; sig2++) {
-                            if (!(sp->sh->sig_array[(sig2/8)] & (1<<(sig2%8))))
+                            if (!(sp->sh->init->sig_array[(sig2/8)] & (1<<(sig2%8))))
                                 continue;
 
                             Signature *s = de_ctx->sig_array[sig2];
@@ -2323,6 +2313,7 @@ int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAdd
                                 }
 
                                 SigGroupHeadDPortHashAdd(de_ctx, dp->sh);
+                                SigGroupHeadStore(de_ctx, dp->sh);
                                 de_ctx->gh_unique++;
                             } else {
                                 SCLogDebug("dp %p dp->sh %p is a copy", dp, dp->sh);
@@ -2537,14 +2528,14 @@ void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh) {
 }
 
 void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh) {
-    if (sgh == NULL) {
+    if (sgh == NULL || sgh->init == NULL) {
         printf("\n");
         return;
     }
 
     uint32_t sig;
     for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
-        if (sgh->sig_array[(sig/8)] & (1<<(sig%8))) {
+        if (sgh->init->sig_array[(sig/8)] & (1<<(sig%8))) {
             printf("%" PRIu32 " ", de_ctx->sig_array[sig]->id);
         }
     }
@@ -2552,14 +2543,14 @@ void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh) {
 }
 
 void DbgSghContainsSig(DetectEngineCtx *de_ctx, SigGroupHead *sgh, uint32_t sid) {
-    if (sgh == NULL) {
+    if (sgh == NULL || sgh->init == NULL) {
         printf("\n");
         return;
     }
 
     uint32_t sig;
     for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) {
-        if (!(sgh->sig_array[(sig/8)] & (1<<(sig%8))))
+        if (!(sgh->init->sig_array[(sig/8)] & (1<<(sig%8))))
             continue;
 
         Signature *s = de_ctx->sig_array[sig];
@@ -2573,6 +2564,29 @@ void DbgSghContainsSig(DetectEngineCtx *de_ctx, SigGroupHead *sgh, uint32_t sid)
     printf("\n");
 }
 
+/** \brief finalize preparing sgh's */
+int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) {
+    SCEnter();
+
+    //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt);
+
+    uint32_t idx = 0;
+
+    for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) {
+        SigGroupHead *sgh = de_ctx->sgh_array[idx];
+        if (sgh == NULL)
+            continue;
+
+        SigGroupHeadBuildHeadArray(de_ctx, sgh);
+    }
+
+    SCFree(de_ctx->sgh_array);
+    de_ctx->sgh_array_cnt = 0;
+    de_ctx->sgh_array_size = 0;
+
+    SCReturnInt(0);
+}
+
 /* shortcut for debugging. If enabled Stage5 will
  * print sigid's for all groups */
 #define PRINTSIGS
@@ -2921,6 +2935,7 @@ int SigGroupBuild (DetectEngineCtx *de_ctx) {
 #endif
 
     SigAddressPrepareStage3(de_ctx);
+    SigAddressPrepareStage4(de_ctx);
 
 #ifdef __SC_CUDA_SUPPORT__
     unsigned int cuda_free_after_alloc = 0;
diff --git a/src/detect.h b/src/detect.h
index 0518273c4e..7474ec95b7 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -55,7 +55,6 @@ struct SCSigSignatureWrapper_;
 
    For TCP/UDP
 
-   Packet data size (dsize)
    - Flow direction
    -- Protocol
    -=- Src address
@@ -65,7 +64,6 @@ struct SCSigSignatureWrapper_;
 
    For the other protocols
 
-   Packet data size (dsize)
    - Flow direction
    -- Protocol
    -=- Src address
@@ -200,6 +198,10 @@ typedef struct DetectPort_ {
 #define SIG_FLAG_BIDIREC        0x00010000  /**< signature has bidirectional operator */
 #define SIG_FLAG_PACKET         0x00020000  /**< signature has matches against a packet (as opposed to app layer) */
 
+#define SIG_FLAG_UMATCH         0x00040000
+#define SIG_FLAG_AMATCH         0x00080000
+#define SIG_FLAG_DMATCH         0x00100000
+
 /* Detection Engine flags */
 #define DE_QUIET           0x01     /**< DE is quiet (esp for unittests) */
 
@@ -219,33 +221,43 @@ typedef struct IPOnlyCIDRItem_ {
 
 } IPOnlyCIDRItem;
 
-/** \brief Signature container */
-typedef struct Signature_ {
+/** \brief Subset of the Signature for cache efficient prefiltering
+ */
+typedef struct SignatureHeader_ {
     uint32_t flags;
 
-    uint8_t rev;
-    int prio;
+    /* app layer signature stuff */
+    uint16_t alproto;
+
+    /** pattern in the mpm matcher */
+    uint32_t mpm_pattern_id;
 
-    uint32_t gid; /**< generator id */
     SigIntId num; /**< signature number, internal id */
-    uint32_t id;  /**< sid, set by the 'sid' rule keyword */
-    uint8_t nchunk_groups; /**< Internal chunk grp id (for splitted patterns) */
-    char *msg;
 
-    /** classification id **/
-    uint8_t class;
+    /** pointer to the full signature */
+    struct Signature_ *full_sig;
+} SignatureHeader;
 
-    /** classification message */
-    char *class_msg;
+/** \brief Signature container */
+typedef struct Signature_ {
+    uint32_t flags;
 
-    /** Reference */
-    Reference *references;
+    /* app layer signature stuff */
+    uint16_t alproto;
 
-    /** addresses, ports and proto this sig matches on */
+    /** pattern in the mpm matcher */
+    uint32_t mpm_pattern_id;
+
+    SigIntId num; /**< signature number, internal id */
+
+    /** address settings for this signature */
     DetectAddressHead src, dst;
-    DetectProto proto;
+    /** port settings for this signature */
     DetectPort *sp, *dp;
 
+    /** addresses, ports and proto this sig matches on */
+    DetectProto proto;
+
     /** netblocks and hosts specified at the sid, in CIDR format */
     IPOnlyCIDRItem *CidrSrc, *CidrDst;
 
@@ -260,6 +272,7 @@ typedef struct Signature_ {
     struct SigMatch_ *amatch_tail; /* general app layer  matches, tail of the list */
     struct SigMatch_ *dmatch; /* dce app layer matches */
     struct SigMatch_ *dmatch_tail; /* dce app layer matches, tail of the list */
+
     /** ptr to the next sig in the list */
     struct Signature_ *next;
 
@@ -270,18 +283,30 @@ typedef struct Signature_ {
     uint16_t mpm_content_maxlen;
     uint16_t mpm_uricontent_maxlen;
 
-    /* app layer signature stuff */
-    uint8_t alproto;
-
     /** number of sigmatches in the match and pmatch list */
     uint16_t sm_cnt;
 
     SigIntId order_id;
 
     /** pattern in the mpm matcher */
-    uint32_t mpm_pattern_id;
     uint32_t mpm_uripattern_id;
 
+    uint8_t rev;
+    int prio;
+
+    uint32_t gid; /**< generator id */
+    uint32_t id;  /**< sid, set by the 'sid' rule keyword */
+    char *msg;
+
+    /** classification id **/
+    uint8_t class;
+
+    /** classification message */
+    char *class_msg;
+
+    /** Reference */
+    Reference *references;
+
 #ifdef PROFILING
     uint16_t profiling_id;
 #endif
@@ -439,6 +464,12 @@ typedef struct DetectEngineCtx_ {
     /** hash table for looking up patterns for
      *  id sharing and id tracking. */
     MpmPatternIdStore *mpm_pattern_id_store;
+
+    /* array containing all sgh's in use so we can loop
+     * through it in Stage4. */
+    struct SigGroupHead_ **sgh_array;
+    uint32_t sgh_array_cnt;
+    uint32_t sgh_array_size;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */
@@ -588,29 +619,37 @@ typedef struct SigGroupHeadInitData_ {
     uint8_t *stream_content_array;
     uint32_t stream_content_size;
 
+    /* "Normal" detection uses these only at init, but ip-only
+     * uses it during runtime as well, thus not in init... */
+    uint8_t *sig_array; /**< bit array of sig nums (internal id's) */
+    uint32_t sig_size; /**< size in bytes */
+
     /* port ptr */
     struct DetectPort_ *port;
 } SigGroupHeadInitData;
 
-/** \brief head of the list of containers. */
+/** \brief Container for matching data for a signature group */
 typedef struct SigGroupHead_ {
     uint8_t flags;
 
-    /* pattern matcher instance */
-    MpmCtx *mpm_ctx;
-    uint16_t mpm_content_maxlen;
-    MpmCtx *mpm_uri_ctx;
-    uint16_t mpm_uricontent_maxlen;
-    MpmCtx *mpm_stream_ctx;
-    uint16_t mpm_streamcontent_maxlen;
+    uint8_t pad0;
+    uint16_t pad1;
 
     /* number of sigs in this head */
     uint32_t sig_cnt;
 
-    /* "Normal" detection uses these only at init, but ip-only
-     * uses it during runtime as well, thus not in init... */
-    uint8_t *sig_array; /**< bit array of sig nums (internal id's) */
-    uint32_t sig_size; /**< size in bytes */
+    /** chunk of memory containing the "header" part of each
+     *  signature ordered as an array. Used to pre-filter the
+     *  signatures to be inspected in a cache efficient way. */
+    SignatureHeader *head_array;
+
+    /* pattern matcher instances */
+    MpmCtx *mpm_ctx;
+    MpmCtx *mpm_stream_ctx;
+    uint16_t mpm_content_maxlen;
+    uint16_t mpm_streamcontent_maxlen;
+    MpmCtx *mpm_uri_ctx;
+    uint16_t mpm_uricontent_maxlen;
 
     /** Array with sig ptrs... size is sig_cnt * sizeof(Signature *) */
     Signature **match_array;
diff --git a/src/util-unittest-helper.c b/src/util-unittest-helper.c
index 549f93667d..56e1de50ec 100644
--- a/src/util-unittest-helper.c
+++ b/src/util-unittest-helper.c
@@ -25,18 +25,21 @@
  */
 
 #include "suricata-common.h"
+
 #include "decode.h"
+
+#include "flow-private.h"
+#include "flow-util.h"
+
 #include "detect.h"
 #include "detect-parse.h"
+#include "detect-engine.h"
+
 #include "util-debug.h"
 #include "util-time.h"
 #include "util-error.h"
 #include "util-unittest.h"
 #include "util-unittest-helper.h"
-#include <stdarg.h>
-#include "detect-engine.h"
-#include "flow-private.h"
-#include "flow-util.h"
 
 /**
  * \brief UTHBuildPacketReal is a function that create tcp/udp packets for unittests
@@ -586,7 +589,6 @@ int UTHPacketMatchSigMpm(Packet *p, char *sig, uint16_t mpm_type) {
     int result = 0;
 
     DecodeThreadVars dtv;
-
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx = NULL;