mpm: add Hyperscan integration

This adds an MPM implementation that uses the Hyperscan regex engine
library from Intel, accessible as the "hs" mpm-algo.

configure.ac

@@ -585,6 +585,44 @@
         AC_MSG_RESULT(no)
     fi
 
+  # libhs
+    enable_hyperscan="no"
+
+    # Try pkg-config first:
+    PKG_CHECK_MODULES([libhs], libhs,, [with_pkgconfig_libhs=no])
+    if test "$with_pkgconfig_libhs" != "no"; then
+        CPPFLAGS="${CPPFLAGS} ${libhs_CFLAGS}"
+        LIBS="${LIBS} ${libhs_LIBS}"
+    fi
+
+    AC_ARG_WITH(libhs_includes,
+            [  --with-libhs-includes=DIR  libhs include directory],
+            [with_libhs_includes="$withval"],[with_libhs_includes=no])
+    AC_ARG_WITH(libhs_libraries,
+            [  --with-libhs-libraries=DIR    libhs library directory],
+            [with_libhs_libraries="$withval"],[with_libhs_libraries="no"])
+
+    if test "$with_libhs_includes" != "no"; then
+        CPPFLAGS="${CPPFLAGS} -I${with_libhs_includes}"
+    fi
+    AC_CHECK_HEADER(hs.h,HYPERSCAN="yes",HYPERSCAN="no")
+    if test "$HYPERSCAN" = "yes"; then
+        if test "$with_libhs_libraries" != "no"; then
+            LDFLAGS="${LDFLAGS}  -L${with_libhs_libraries}"
+        fi
+
+        AC_CHECK_LIB(hs,hs_compile,,HYPERSCAN="no")
+        enable_hyperscan="yes"
+        if test "$HYPERSCAN" = "no"; then
+            echo
+            echo "   Hyperscan headers are present, but link test failed."
+            echo "   Check that you have a shared library and C++ linkage available."
+            echo
+            enable_hyperscan="no"
+        fi
+    fi
+    AS_IF([test "x$enable_hyperscan" = "xyes"], [AC_DEFINE([BUILD_HYPERSCAN], [1], [Intel Hyperscan support enabled])])
+
   # libyaml
     AC_ARG_WITH(libyaml_includes,
             [  --with-libyaml-includes=DIR  libyaml include directory],
@@ -1896,6 +1934,7 @@ SURICATA_BUILD_CONF="Suricata Configuration:
   Non-bundled htp:                         ${enable_non_bundled_htp}
   Old barnyard2 support:                   ${enable_old_barnyard2}
   CUDA enabled:                            ${enable_cuda}
+  Hyperscan support:                       ${enable_hyperscan}
 
   Suricatasc install:                      ${enable_python}
 

src/Makefile.am

@@ -372,6 +372,7 @@ util-mpm-ac-tile.c util-mpm-ac-tile.h \
 util-mpm-ac-tile-small.c \
 util-mpm-b2g.c util-mpm-b2g.h \
 util-mpm-b3g.c util-mpm-b3g.h \
+util-mpm-hs.c util-mpm-hs.h \
 util-mpm.c util-mpm.h \
 util-mpm-wumanber.c util-mpm-wumanber.h \
 util-optimize.h \

src/detect-engine.c

@@ -1068,6 +1068,9 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx)
         /* for now, since we still haven't implemented any intelligence into
          * understanding the patterns and distributing mpm_ctx across sgh */
         if (de_ctx->mpm_matcher == DEFAULT_MPM || de_ctx->mpm_matcher == MPM_AC_GFBS ||
+#ifdef BUILD_HYPERSCAN
+            de_ctx->mpm_matcher == MPM_HS ||
+#endif
 #ifdef __SC_CUDA_SUPPORT__
             de_ctx->mpm_matcher == MPM_AC_BS || de_ctx->mpm_matcher == MPM_AC_CUDA) {
 #else

src/runmode-unittests.c

@@ -108,6 +108,7 @@
 #include "util-memrchr.h"
 
 #include "util-mpm-ac.h"
+#include "util-mpm-hs.h"
 #include "detect-engine-mpm.h"
 
 #include "util-decode-asn1.h"
@@ -288,6 +289,9 @@ void RunUnittests(int list_unittests, char *regex_arg)
         uint32_t failed = UtRunTests(regex_arg);
         PacketPoolDestroy();
         UtCleanup();
+#ifdef BUILD_HYPERSCAN
+        MpmHSGlobalCleanup();
+#endif
 #ifdef __SC_CUDA_SUPPORT__
         if (PatternMatchDefaultMatcher() == MPM_AC_CUDA)
             MpmCudaBufferDeSetup();

src/suricata.c

@@ -194,6 +194,7 @@
 #include "util-cuda-buffer.h"
 #include "util-mpm-ac.h"
 #endif
+#include "util-mpm-hs.h"
 #include "util-storage.h"
 #include "host-storage.h"
 
@@ -2615,6 +2616,10 @@ int main(int argc, char **argv)
 
     SC_ATOMIC_DESTROY(engine_stage);
 
+#ifdef BUILD_HYPERSCAN
+    MpmHSGlobalCleanup();
+#endif
+
 #ifdef __SC_CUDA_SUPPORT__
     if (PatternMatchDefaultMatcher() == MPM_AC_CUDA)
         MpmCudaBufferDeSetup();

src/util-mpm-hs.h

@@ -0,0 +1,76 @@
+/* Copyright (C) 2007-2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jim Xu <jim.xu@windriver.com>
+ * \author Justin Viiret <justin.viiret@intel.com>
+ *
+ * MPM pattern matcher that calls the Hyperscan regex matcher.
+ */
+
+#ifndef __UTIL_MPM_HS__H__
+#define __UTIL_MPM_HS__H__
+
+typedef struct SCHSPattern_ {
+    /* length of the pattern */
+    uint16_t len;
+    /* flags describing the pattern */
+    uint8_t flags;
+    /* holds the original pattern that was added */
+    uint8_t *original_pat;
+    /* pattern id */
+    uint32_t id;
+
+    uint16_t offset;
+    uint16_t depth;
+
+    /* sid(s) for this pattern */
+    uint32_t sids_size;
+    SigIntId *sids;
+
+    /* only used at ctx init time, when this structure is part of a hash
+     * table. */
+    struct SCHSPattern_ *next;
+} SCHSPattern;
+
+typedef struct SCHSCtx_ {
+    /* hash used during ctx initialization */
+    SCHSPattern **init_hash;
+
+    /* pattern database and pattern arrays. */
+    void *pattern_db;
+
+    /* size of database, for accounting. */
+    size_t hs_db_size;
+} SCHSCtx;
+
+typedef struct SCHSThreadCtx_ {
+    /* Hyperscan scratch space region for this thread, capable of handling any
+     * database that has been compiled. */
+    void *scratch;
+
+    /* size of scratch space, for accounting. */
+    size_t scratch_size;
+} SCHSThreadCtx;
+
+void MpmHSRegister(void);
+
+void MpmHSGlobalCleanup(void);
+
+#endif /* __UTIL_MPM_HS__H__ */

src/util-mpm.c

@@ -35,6 +35,7 @@
 #include "util-mpm-ac-gfbs.h"
 #include "util-mpm-ac-bs.h"
 #include "util-mpm-ac-tile.h"
+#include "util-mpm-hs.h"
 #include "util-hashlist.h"
 
 #include "detect-engine.h"
@@ -674,6 +675,9 @@ void MpmTableSetup(void)
     MpmACBSRegister();
     MpmACGfbsRegister();
     MpmACTileRegister();
+#ifdef BUILD_HYPERSCAN
+    MpmHSRegister();
+#endif /* BUILD_HYPERSCAN */
 #ifdef __SC_CUDA_SUPPORT__
     MpmACCudaRegister();
 #endif /* __SC_CUDA_SUPPORT__ */

src/util-mpm.h

@@ -72,6 +72,7 @@ enum {
     MPM_AC_GFBS,
     MPM_AC_BS,
     MPM_AC_TILE,
+    MPM_HS,
     /* table size */
     MPM_TABLE_SIZE,
 };