spm: add Hyperscan implementation

src/Makefile.am

@@ -350,6 +350,7 @@ util-hashlist.c util-hashlist.h \
 util-hash-lookup3.c util-hash-lookup3.h \
 util-host-os-info.c util-host-os-info.h \
 util-host-info.c util-host-info.h \
+util-hyperscan.c util-hyperscan.h \
 util-ioctl.h util-ioctl.c \
 util-ip.h util-ip.c \
 util-logopenfile.h util-logopenfile.c \
@@ -397,6 +398,7 @@ util-signal.c util-signal.h \
 util-spm-bm.c util-spm-bm.h \
 util-spm-bs2bm.c util-spm-bs2bm.h \
 util-spm-bs.c util-spm-bs.h \
+util-spm-hs.c util-spm-hs.h \
 util-spm.c util-spm.h util-clock.h \
 util-storage.c util-storage.h \
 util-strlcatu.c \

src/util-hyperscan.c

@@ -0,0 +1,59 @@
+/* Copyright (C) 2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Justin Viiret <justin.viiret@intel.com>
+ *
+ * Support functions for Hyperscan library integration.
+ */
+
+#include "suricata-common.h"
+#include "suricata.h"
+
+#ifdef BUILD_HYPERSCAN
+
+/**
+ * \internal
+ * \brief Convert a pattern into a regex string accepted by the Hyperscan
+ * compiler.
+ *
+ * For simplicity, we just take each byte of the original pattern and render it
+ * with a hex escape (i.e. ' ' -> "\x20")/
+ */
+char *HSRenderPattern(const uint8_t *pat, uint16_t pat_len)
+{
+    if (pat == NULL) {
+        return NULL;
+    }
+    const size_t hex_len = (pat_len * 4) + 1;
+    char *str = SCMalloc(hex_len);
+    if (str == NULL) {
+        return NULL;
+    }
+    memset(str, 0, hex_len);
+    char *sp = str;
+    for (uint16_t i = 0; i < pat_len; i++) {
+        snprintf(sp, 5, "\\x%02x", pat[i]);
+        sp += 4;
+    }
+    *sp = '\0';
+    return str;
+}
+
+#endif /* BUILD_HYPERSCAN */

src/util-hyperscan.h

@@ -0,0 +1,31 @@
+/* Copyright (C) 2007-2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Justin Viiret <justin.viiret@intel.com>
+ *
+ * Support functions for Hyperscan library integration.
+ */
+
+#ifndef __UTIL_HYPERSCAN__H__
+#define __UTIL_HYPERSCAN__H__
+
+char *HSRenderPattern(const uint8_t *pat, uint16_t pat_len);
+
+#endif /* __UTIL_HYPERSCAN__H__ */

src/util-mpm-hs.c

@@ -40,6 +40,7 @@
 #include "util-memcpy.h"
 #include "util-hash.h"
 #include "util-hash-lookup3.h"
+#include "util-hyperscan.h"
 
 #ifdef BUILD_HYPERSCAN
 
@@ -111,34 +112,6 @@ static void SCHSSetAllocators(void)
     }
 }
 
-/**
- * \internal
- * \brief Convert a pattern into a regex string accepted by the Hyperscan
- * compiler.
- *
- * For simplicity, we just take each byte of the original pattern and render it
- * with a hex escape (i.e. ' ' -> "\x20")/
- */
-static char *SCHSRenderPattern(uint8_t *pat, uint16_t pat_len)
-{
-    if (pat == NULL) {
-        return NULL;
-    }
-    const size_t hex_len = (pat_len * 4) + 1;
-    char *str = SCMalloc(hex_len);
-    if (str == NULL) {
-        return NULL;
-    }
-    memset(str, 0, hex_len);
-    char *sp = str;
-    for (uint16_t i = 0; i < pat_len; i++) {
-        snprintf(sp, 5, "\\x%02x", pat[i]);
-        sp += 4;
-    }
-    *sp = '\0';
-    return str;
-}
-
 /**
  * \internal
  * \brief Creates a hash of the pattern.  We use it for the hashing process
@@ -679,7 +652,7 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx)
             cd->flags[i] |= HS_FLAG_CASELESS;
         }
 
-        cd->expressions[i] = SCHSRenderPattern(p->original_pat, p->len);
+        cd->expressions[i] = HSRenderPattern(p->original_pat, p->len);
 
         if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) {
             cd->ext[i] = SCMalloc(sizeof(hs_expr_ext_t));

src/util-spm-hs.c

@@ -0,0 +1,237 @@
+/* Copyright (C) 2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Justin Viiret <justin.viiret@intel.com>
+ *
+ * Single pattern matcher that uses the Hyperscan regex matcher.
+ */
+
+#include "suricata-common.h"
+#include "suricata.h"
+
+#include "util-hyperscan.h"
+#include "util-spm-hs.h"
+
+#ifdef BUILD_HYPERSCAN
+
+#include <hs.h>
+
+/**
+ * \internal
+ * \brief Hyperscan match callback, called by hs_scan.
+ */
+static int MatchEvent(unsigned int id, unsigned long long from,
+                      unsigned long long to, unsigned int flags, void *context)
+{
+    uint64_t *match_offset = context;
+    BUG_ON(*match_offset != UINT64_MAX);
+    *match_offset = to;
+    return 1; /* Terminate matching. */
+}
+
+typedef struct SpmHsCtx_ {
+    hs_database_t *db;
+    uint16_t needle_len;
+} SpmHsCtx;
+
+static void HSDestroyCtx(SpmCtx *ctx)
+{
+    if (ctx == NULL) {
+        return;
+    }
+    SpmHsCtx *sctx = ctx->ctx;
+    if (sctx) {
+        hs_free_database(sctx->db);
+        SCFree(sctx);
+    }
+    SCFree(ctx);
+}
+
+static int HSBuildDatabase(const uint8_t *needle, uint16_t needle_len,
+                            int nocase, SpmHsCtx *sctx,
+                            SpmGlobalThreadCtx *global_thread_ctx)
+{
+    char *expr = HSRenderPattern(needle, needle_len);
+    if (expr == NULL) {
+        SCLogDebug("HSRenderPattern returned NULL");
+        return -1;
+    }
+
+    unsigned flags = nocase ? HS_FLAG_CASELESS : 0;
+
+    hs_database_t *db = NULL;
+    hs_compile_error_t *compile_err = NULL;
+    hs_error_t err = hs_compile(expr, flags, HS_MODE_BLOCK, NULL, &db,
+                                &compile_err);
+    if (err != HS_SUCCESS) {
+        SCLogError(SC_ERR_FATAL, "Unable to compile '%s' with Hyperscan, "
+                                 "returned %d.", expr, err);
+        exit(EXIT_FAILURE);
+    }
+
+    SCFree(expr);
+
+    /* Update scratch for this database. */
+    hs_scratch_t *scratch = global_thread_ctx->ctx;
+    err = hs_alloc_scratch(db, &scratch);
+    if (err != HS_SUCCESS) {
+        /* If scratch allocation failed, this is not recoverable:  other SPM
+         * contexts may need this scratch space. */
+        SCLogError(SC_ERR_FATAL,
+                   "Unable to alloc scratch for Hyperscan, returned %d.", err);
+        exit(EXIT_FAILURE);
+    }
+    global_thread_ctx->ctx = scratch;
+    sctx->db = db;
+    sctx->needle_len = needle_len;
+
+    return 0;
+}
+
+static SpmCtx *HSInitCtx(const uint8_t *needle, uint16_t needle_len, int nocase,
+                         SpmGlobalThreadCtx *global_thread_ctx)
+{
+    SpmCtx *ctx = SCMalloc(sizeof(SpmCtx));
+    if (ctx == NULL) {
+        SCLogDebug("Unable to alloc SpmCtx.");
+        return NULL;
+    }
+    memset(ctx, 0, sizeof(SpmCtx));
+    ctx->matcher = SPM_HS;
+
+    SpmHsCtx *sctx = SCMalloc(sizeof(SpmHsCtx));
+    if (sctx == NULL) {
+        SCLogDebug("Unable to alloc SpmHsCtx.");
+        SCFree(ctx);
+        return NULL;
+    }
+    ctx->ctx = sctx;
+
+    memset(sctx, 0, sizeof(SpmHsCtx));
+    if (HSBuildDatabase(needle, needle_len, nocase, sctx,
+                        global_thread_ctx) != 0) {
+        SCLogDebug("HSBuildDatabase failed.");
+        HSDestroyCtx(ctx);
+        return NULL;
+    }
+
+    return ctx;
+}
+
+static uint8_t *HSScan(const SpmCtx *ctx, SpmThreadCtx *thread_ctx,
+                       const uint8_t *haystack, uint16_t haystack_len)
+{
+    const SpmHsCtx *sctx = ctx->ctx;
+    hs_scratch_t *scratch = thread_ctx->ctx;
+
+    uint64_t match_offset = UINT64_MAX;
+    hs_error_t err = hs_scan(sctx->db, (const char *)haystack, haystack_len, 0,
+                             scratch, MatchEvent, &match_offset);
+    if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
+        /* An error value (other than HS_SCAN_TERMINATED) from hs_scan()
+         * indicates that it was passed an invalid database or scratch region,
+         * which is not something we can recover from at scan time. */
+        SCLogError(SC_ERR_FATAL, "Hyperscan returned fatal error %d.", err);
+        exit(EXIT_FAILURE);
+    }
+
+    if (match_offset == UINT64_MAX) {
+        return NULL;
+    }
+
+    BUG_ON(match_offset < sctx->needle_len);
+    BUG_ON(match_offset > UINT16_MAX); /* haystack_len is a uint16_t */
+
+    /* Note: existing API returns non-const ptr */
+    return (uint8_t *)haystack + (match_offset - sctx->needle_len);
+}
+
+static SpmGlobalThreadCtx *HSInitGlobalThreadCtx(void)
+{
+    SpmGlobalThreadCtx *global_thread_ctx = SCMalloc(sizeof(SpmGlobalThreadCtx));
+    if (global_thread_ctx == NULL) {
+        SCLogDebug("Unable to alloc SpmGlobalThreadCtx.");
+        return NULL;
+    }
+    memset(global_thread_ctx, 0, sizeof(*global_thread_ctx));
+    global_thread_ctx->matcher = SPM_HS;
+
+    /* We store scratch in the HS-specific ctx. This will be initialized as
+     * patterns are compiled by SpmInitCtx. */
+    global_thread_ctx->ctx = NULL;
+
+    return global_thread_ctx;
+}
+
+static void HSDestroyGlobalThreadCtx(SpmGlobalThreadCtx *global_thread_ctx)
+{
+    if (global_thread_ctx == NULL) {
+        return;
+    }
+    hs_free_scratch(global_thread_ctx->ctx);
+    SCFree(global_thread_ctx);
+}
+
+static void HSDestroyThreadCtx(SpmThreadCtx *thread_ctx)
+{
+    if (thread_ctx == NULL) {
+        return;
+    }
+    hs_free_scratch(thread_ctx->ctx);
+    SCFree(thread_ctx);
+}
+
+static SpmThreadCtx *HSMakeThreadCtx(const SpmGlobalThreadCtx *global_thread_ctx)
+{
+    SpmThreadCtx *thread_ctx = SCMalloc(sizeof(SpmThreadCtx));
+    if (thread_ctx == NULL) {
+        SCLogDebug("Unable to alloc SpmThreadCtx.");
+        return NULL;
+    }
+    memset(thread_ctx, 0, sizeof(*thread_ctx));
+    thread_ctx->matcher = SPM_HS;
+
+    if (global_thread_ctx->ctx != NULL) {
+        hs_scratch_t *scratch = NULL;
+        hs_error_t err = hs_clone_scratch(global_thread_ctx->ctx, &scratch);
+        if (err != HS_SUCCESS) {
+            SCLogError(SC_ERR_FATAL, "Unable to clone scratch (error %d).",
+                       err);
+            exit(EXIT_FAILURE);
+        }
+        thread_ctx->ctx = scratch;
+    }
+
+    return thread_ctx;
+}
+
+void SpmHSRegister(void)
+{
+    spm_table[SPM_HS].name = "hs";
+    spm_table[SPM_HS].InitGlobalThreadCtx = HSInitGlobalThreadCtx;
+    spm_table[SPM_HS].DestroyGlobalThreadCtx = HSDestroyGlobalThreadCtx;
+    spm_table[SPM_HS].MakeThreadCtx = HSMakeThreadCtx;
+    spm_table[SPM_HS].DestroyThreadCtx = HSDestroyThreadCtx;
+    spm_table[SPM_HS].InitCtx = HSInitCtx;
+    spm_table[SPM_HS].DestroyCtx = HSDestroyCtx;
+    spm_table[SPM_HS].Scan = HSScan;
+}
+
+#endif /* BUILD_HYPERSCAN */

src/util-spm-hs.h

@@ -0,0 +1,31 @@
+/* Copyright (C) 2016 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Justin Viiret <justin.viiret@intel.com>
+ *
+ * Single pattern matcher that uses the Hyperscan regex matcher.
+ */
+
+#ifndef __UTIL_SPM_HS_H__
+#define __UTIL_SPM_HS_H__
+
+void SpmHSRegister(void);
+
+#endif /* __UTIL_SPM_HS_H__ */

src/util-spm.c

@@ -53,6 +53,7 @@
 #include "util-spm-bs.h"
 #include "util-spm-bs2bm.h"
 #include "util-spm-bm.h"
+#include "util-spm-hs.h"
 #include "util-clock.h"
 
 /**
@@ -89,6 +90,9 @@ void SpmTableSetup(void)
     memset(spm_table, 0, sizeof(spm_table));
 
     SpmBMRegister();
+#ifdef BUILD_HYPERSCAN
+    SpmHSRegister();
+#endif
 }
 
 SpmGlobalThreadCtx *SpmInitGlobalThreadCtx(uint16_t matcher)

src/util-spm.h

@@ -30,6 +30,7 @@
 
 enum {
     SPM_BM, /* Boyer-Moore */
+    SPM_HS, /* Hyperscan */
     /* Other SPM matchers will go here. */
     SPM_TABLE_SIZE
 };