prefilter: implement prefilter keyword

Introduce prefilter keyword to force a keyword to be used as prefilter.

e.g.
alert tcp any any -> any any (content:"A"; flags:R; prefilter; sid:1;)
alert tcp any any -> any any (content:"A"; flags:R; sid:2;)
alert tcp any any -> any any (content:"A"; dsize:1; prefilter; sid:3;)
alert tcp any any -> any any (content:"A"; dsize:1; sid:4;)

In sid 2 and 4 the content keyword is used in the MPM engine.
In sid 1 and 3 the flags and dsize keywords will be used.

src/Makefile.am

@@ -191,6 +191,7 @@ detect-parse.c detect-parse.h \
 detect-pcre.c detect-pcre.h \
 detect-pkt-data.c detect-pkt-data.h \
 detect-pktvar.c detect-pktvar.h \
+detect-prefilter.c detect-prefilter.h \
 detect-priority.c detect-priority.h \
 detect-rawbytes.c detect-rawbytes.h \
 detect-reference.c detect-reference.h \

src/detect-engine-mpm.c

@@ -1283,11 +1283,16 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
      * true size, since duplicates are removed below, but counted here.
      */
     for (s = de_ctx->sig_list; s != NULL; s = s->next) {
+        if (s->flags & SIG_FLAG_PREFILTER)
+            continue;
+
         RetrieveFPForSig(s);
         if (s->mpm_sm != NULL) {
             DetectContentData *cd = (DetectContentData *)s->mpm_sm->ctx;
             struct_total_size += sizeof(DetectFPAndItsId);
             content_total_size += cd->content_len;
+
+            s->flags |= SIG_FLAG_PREFILTER;
         }
     }
 

src/detect-engine-siggroup.c

@@ -699,7 +699,7 @@ int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sg
         if (s == NULL)
             continue;
 
-        if (s->mpm_sm == NULL || (s->flags & SIG_FLAG_MPM_NEG)) {
+        if (!(s->flags & SIG_FLAG_PREFILTER) || (s->flags & SIG_FLAG_MPM_NEG)) {
             if (!(DetectFlagsSignatureNeedsSynPackets(s))) {
                 non_pf++;
             }
@@ -730,7 +730,7 @@ int SigGroupHeadBuildNonPrefilterArray(DetectEngineCtx *de_ctx, SigGroupHead *sg
         if (s == NULL)
             continue;
 
-        if (s->mpm_sm == NULL || (s->flags & SIG_FLAG_MPM_NEG)) {
+        if (!(s->flags & SIG_FLAG_PREFILTER) || (s->flags & SIG_FLAG_MPM_NEG)) {
             if (!(DetectFlagsSignatureNeedsSynPackets(s))) {
                 BUG_ON(sgh->non_pf_other_store_cnt >= non_pf);
                 BUG_ON(sgh->non_pf_other_store_array == NULL);

src/detect-prefilter.c

@@ -0,0 +1,104 @@
+/* Copyright (C) 2007-2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ *
+ * Implements the prefilter keyword
+ */
+
+#include "suricata-common.h"
+#include "decode.h"
+
+#include "detect.h"
+#include "detect-parse.h"
+#include "detect-content.h"
+
+#include "util-debug.h"
+
+static int DetectPrefilterSetup (DetectEngineCtx *, Signature *, char *);
+
+void DetectPrefilterRegister(void)
+{
+    sigmatch_table[DETECT_PREFILTER].name = "prefilter";
+    sigmatch_table[DETECT_PREFILTER].desc = "force a condition to be used as prefilter";
+    sigmatch_table[DETECT_PREFILTER].Match = NULL;
+    sigmatch_table[DETECT_PREFILTER].Setup = DetectPrefilterSetup;
+    sigmatch_table[DETECT_PREFILTER].Free  = NULL;
+    sigmatch_table[DETECT_PREFILTER].RegisterTests = NULL;
+
+    sigmatch_table[DETECT_PREFILTER].flags |= SIGMATCH_NOOPT;
+}
+
+/**
+ *  \internal
+ *  \brief Apply the prefilter keyword to the last match
+ *  \param det_ctx detection engine ctx
+ *  \param s signature
+ *  \param nullstr should be null
+ *  \retval 0 ok
+ *  \retval -1 failure
+ */
+static int DetectPrefilterSetup (DetectEngineCtx *de_ctx, Signature *s, char *nullstr)
+{
+    SCEnter();
+
+    SigMatch *sm = NULL;
+    int ret = -1;
+
+    if (nullstr != NULL) {
+        SCLogError(SC_ERR_INVALID_VALUE, "prefilter has value");
+        goto end;
+    }
+
+    if (s->flags & SIG_FLAG_PREFILTER) {
+        SCLogError(SC_ERR_INVALID_SIGNATURE, "prefilter already set");
+        goto end;
+    }
+
+    sm = SigMatchGetLastSM(s);
+    if (sm == NULL) {
+        SCLogError(SC_ERR_INVALID_SIGNATURE, "prefilter needs preceding match");
+        goto end;
+    }
+
+    s->prefilter_sm = sm;
+    s->flags |= SIG_FLAG_PREFILTER;
+
+    /* if the sig match is content, prefilter should act like
+     * 'fast_pattern' w/o options. */
+    if (sm->type == DETECT_CONTENT) {
+        DetectContentData *cd = (DetectContentData *)sm->ctx;
+        if ((cd->flags & DETECT_CONTENT_NEGATED) &&
+                ((cd->flags & DETECT_CONTENT_DISTANCE) ||
+                 (cd->flags & DETECT_CONTENT_WITHIN) ||
+                 (cd->flags & DETECT_CONTENT_OFFSET) ||
+                 (cd->flags & DETECT_CONTENT_DEPTH)))
+        {
+            SCLogError(SC_ERR_INVALID_SIGNATURE, "prefilter; cannot be "
+                    "used with negated content, along with relative modifiers");
+            goto end;
+        }
+        cd->flags |= DETECT_CONTENT_FAST_PATTERN;
+    }
+
+    ret = 0;
+ end:
+    SCReturnInt(ret);
+}

src/detect-prefilter.h

@@ -0,0 +1,30 @@
+/* Copyright (C) 2007-2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Victor Julien <victor@inliniac.net>
+ */
+
+#ifndef __DETECT_PREFILTER_H__
+#define __DETECT_PREFILTER_H__
+
+/* prototypes */
+void DetectPrefilterRegister (void);
+
+#endif /* __DETECT_PREFILTER_H__ */

src/detect.c

@@ -89,6 +89,7 @@
 #include "detect-distance.h"
 #include "detect-offset.h"
 #include "detect-sid.h"
+#include "detect-prefilter.h"
 #include "detect-priority.h"
 #include "detect-classtype.h"
 #include "detect-reference.h"
@@ -3419,6 +3420,24 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
 
         RuleSetWhitelist(tmp_s);
 
+        if (!(tmp_s->flags & SIG_FLAG_PREFILTER)) {
+            int i;
+            for (i = 0; i < DETECT_SM_LIST_DETECT_MAX; i++) {
+                SigMatch *sm = tmp_s->sm_lists[i];
+                while (sm != NULL) {
+                    if (sigmatch_table[sm->type].SupportsPrefilter != NULL) {
+                        if (sigmatch_table[sm->type].SupportsPrefilter(tmp_s) == TRUE) {
+                            tmp_s->prefilter_sm = sm;
+                            tmp_s->flags |= SIG_FLAG_PREFILTER;
+                            SCLogConfig("sid %u: prefilter is on \"%s\"", tmp_s->id, sigmatch_table[sm->type].name);
+                            break;
+                        }
+                    }
+                    sm = sm->next;
+                }
+            }
+        }
+
         de_ctx->sig_cnt++;
     }
 
@@ -3812,6 +3831,15 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
         SCLogDebug("filestore count %u", sgh->filestore_cnt);
 
         BUG_ON(PatternMatchPrepareGroup(de_ctx, sgh) != 0);
+
+        int i = 0;
+        for (i = 0; i < DETECT_TBLSIZE; i++)
+        {
+            if (sigmatch_table[i].SetupPrefilter != NULL) {
+                sigmatch_table[i].SetupPrefilter(sgh);
+            }
+        }
+
         SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh);
 
         sgh->id = idx;
@@ -4103,6 +4131,7 @@ void SigTableSetup(void)
 
     DetectSidRegister();
     DetectPriorityRegister();
+    DetectPrefilterRegister();
     DetectRevRegister();
     DetectClasstypeRegister();
     DetectReferenceRegister();

src/detect.h

@@ -277,6 +277,9 @@ typedef struct DetectPort_ {
 #define SIG_FLAG_TLSSTORE               (1<<21)
 
 #define SIG_FLAG_BYPASS                (1<<22)
+
+#define SIG_FLAG_PREFILTER              (1<<23) /**< sig is part of a prefilter engine */
+
 /* signature init flags */
 #define SIG_FLAG_INIT_DEONLY         1  /**< decode event only signature */
 #define SIG_FLAG_INIT_PACKET         (1<<1)  /**< signature has matches against a packet (as opposed to app layer) */
@@ -439,6 +442,8 @@ typedef struct Signature_ {
     SigMatch *dsize_sm;
     /* the fast pattern added from this signature */
     SigMatch *mpm_sm;
+    /* used to speed up init of prefilter */
+    SigMatch *prefilter_sm;
 
     /* SigMatch list used for adding content and friends. E.g. file_data; */
     int list;
@@ -905,6 +910,9 @@ typedef struct SigTableElmt_ {
     /** keyword setup function pointer */
     int (*Setup)(DetectEngineCtx *, Signature *, char *);
 
+    _Bool (*SupportsPrefilter)(const Signature *s);
+    int (*SetupPrefilter)(struct SigGroupHead_ *sgh);
+
     void (*Free)(void *);
     void (*RegisterTests)(void);
 
@@ -1221,6 +1229,8 @@ enum {
 
     DETECT_BYPASS,
 
+    DETECT_PREFILTER,
+
     /* make sure this stays last */
     DETECT_TBLSIZE,
 };