diff --git a/src/Makefile.am b/src/Makefile.am index 4d0490a1f3..e2369c70f7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -350,6 +350,7 @@ util-hashlist.c util-hashlist.h \ util-hash-lookup3.c util-hash-lookup3.h \ util-host-os-info.c util-host-os-info.h \ util-host-info.c util-host-info.h \ +util-hyperscan.c util-hyperscan.h \ util-ioctl.h util-ioctl.c \ util-ip.h util-ip.c \ util-logopenfile.h util-logopenfile.c \ @@ -397,6 +398,7 @@ util-signal.c util-signal.h \ util-spm-bm.c util-spm-bm.h \ util-spm-bs2bm.c util-spm-bs2bm.h \ util-spm-bs.c util-spm-bs.h \ +util-spm-hs.c util-spm-hs.h \ util-spm.c util-spm.h util-clock.h \ util-storage.c util-storage.h \ util-strlcatu.c \ diff --git a/src/util-hyperscan.c b/src/util-hyperscan.c new file mode 100644 index 0000000000..edcf73f665 --- /dev/null +++ b/src/util-hyperscan.c @@ -0,0 +1,59 @@ +/* Copyright (C) 2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Justin Viiret + * + * Support functions for Hyperscan library integration. + */ + +#include "suricata-common.h" +#include "suricata.h" + +#ifdef BUILD_HYPERSCAN + +/** + * \internal + * \brief Convert a pattern into a regex string accepted by the Hyperscan + * compiler. + * + * For simplicity, we just take each byte of the original pattern and render it + * with a hex escape (i.e. ' ' -> "\x20")/ + */ +char *HSRenderPattern(const uint8_t *pat, uint16_t pat_len) +{ + if (pat == NULL) { + return NULL; + } + const size_t hex_len = (pat_len * 4) + 1; + char *str = SCMalloc(hex_len); + if (str == NULL) { + return NULL; + } + memset(str, 0, hex_len); + char *sp = str; + for (uint16_t i = 0; i < pat_len; i++) { + snprintf(sp, 5, "\\x%02x", pat[i]); + sp += 4; + } + *sp = '\0'; + return str; +} + +#endif /* BUILD_HYPERSCAN */ diff --git a/src/util-hyperscan.h b/src/util-hyperscan.h new file mode 100644 index 0000000000..67bcabf49d --- /dev/null +++ b/src/util-hyperscan.h @@ -0,0 +1,31 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Justin Viiret + * + * Support functions for Hyperscan library integration. + */ + +#ifndef __UTIL_HYPERSCAN__H__ +#define __UTIL_HYPERSCAN__H__ + +char *HSRenderPattern(const uint8_t *pat, uint16_t pat_len); + +#endif /* __UTIL_HYPERSCAN__H__ */ diff --git a/src/util-mpm-hs.c b/src/util-mpm-hs.c index c7659974a7..8b993ae160 100644 --- a/src/util-mpm-hs.c +++ b/src/util-mpm-hs.c @@ -40,6 +40,7 @@ #include "util-memcpy.h" #include "util-hash.h" #include "util-hash-lookup3.h" +#include "util-hyperscan.h" #ifdef BUILD_HYPERSCAN @@ -111,34 +112,6 @@ static void SCHSSetAllocators(void) } } -/** - * \internal - * \brief Convert a pattern into a regex string accepted by the Hyperscan - * compiler. - * - * For simplicity, we just take each byte of the original pattern and render it - * with a hex escape (i.e. ' ' -> "\x20")/ - */ -static char *SCHSRenderPattern(uint8_t *pat, uint16_t pat_len) -{ - if (pat == NULL) { - return NULL; - } - const size_t hex_len = (pat_len * 4) + 1; - char *str = SCMalloc(hex_len); - if (str == NULL) { - return NULL; - } - memset(str, 0, hex_len); - char *sp = str; - for (uint16_t i = 0; i < pat_len; i++) { - snprintf(sp, 5, "\\x%02x", pat[i]); - sp += 4; - } - *sp = '\0'; - return str; -} - /** * \internal * \brief Creates a hash of the pattern. We use it for the hashing process @@ -679,7 +652,7 @@ int SCHSPreparePatterns(MpmCtx *mpm_ctx) cd->flags[i] |= HS_FLAG_CASELESS; } - cd->expressions[i] = SCHSRenderPattern(p->original_pat, p->len); + cd->expressions[i] = HSRenderPattern(p->original_pat, p->len); if (p->flags & (MPM_PATTERN_FLAG_OFFSET | MPM_PATTERN_FLAG_DEPTH)) { cd->ext[i] = SCMalloc(sizeof(hs_expr_ext_t)); diff --git a/src/util-spm-hs.c b/src/util-spm-hs.c new file mode 100644 index 0000000000..5ca0d333d0 --- /dev/null +++ b/src/util-spm-hs.c @@ -0,0 +1,237 @@ +/* Copyright (C) 2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Justin Viiret + * + * Single pattern matcher that uses the Hyperscan regex matcher. + */ + +#include "suricata-common.h" +#include "suricata.h" + +#include "util-hyperscan.h" +#include "util-spm-hs.h" + +#ifdef BUILD_HYPERSCAN + +#include + +/** + * \internal + * \brief Hyperscan match callback, called by hs_scan. + */ +static int MatchEvent(unsigned int id, unsigned long long from, + unsigned long long to, unsigned int flags, void *context) +{ + uint64_t *match_offset = context; + BUG_ON(*match_offset != UINT64_MAX); + *match_offset = to; + return 1; /* Terminate matching. */ +} + +typedef struct SpmHsCtx_ { + hs_database_t *db; + uint16_t needle_len; +} SpmHsCtx; + +static void HSDestroyCtx(SpmCtx *ctx) +{ + if (ctx == NULL) { + return; + } + SpmHsCtx *sctx = ctx->ctx; + if (sctx) { + hs_free_database(sctx->db); + SCFree(sctx); + } + SCFree(ctx); +} + +static int HSBuildDatabase(const uint8_t *needle, uint16_t needle_len, + int nocase, SpmHsCtx *sctx, + SpmGlobalThreadCtx *global_thread_ctx) +{ + char *expr = HSRenderPattern(needle, needle_len); + if (expr == NULL) { + SCLogDebug("HSRenderPattern returned NULL"); + return -1; + } + + unsigned flags = nocase ? HS_FLAG_CASELESS : 0; + + hs_database_t *db = NULL; + hs_compile_error_t *compile_err = NULL; + hs_error_t err = hs_compile(expr, flags, HS_MODE_BLOCK, NULL, &db, + &compile_err); + if (err != HS_SUCCESS) { + SCLogError(SC_ERR_FATAL, "Unable to compile '%s' with Hyperscan, " + "returned %d.", expr, err); + exit(EXIT_FAILURE); + } + + SCFree(expr); + + /* Update scratch for this database. */ + hs_scratch_t *scratch = global_thread_ctx->ctx; + err = hs_alloc_scratch(db, &scratch); + if (err != HS_SUCCESS) { + /* If scratch allocation failed, this is not recoverable: other SPM + * contexts may need this scratch space. */ + SCLogError(SC_ERR_FATAL, + "Unable to alloc scratch for Hyperscan, returned %d.", err); + exit(EXIT_FAILURE); + } + global_thread_ctx->ctx = scratch; + sctx->db = db; + sctx->needle_len = needle_len; + + return 0; +} + +static SpmCtx *HSInitCtx(const uint8_t *needle, uint16_t needle_len, int nocase, + SpmGlobalThreadCtx *global_thread_ctx) +{ + SpmCtx *ctx = SCMalloc(sizeof(SpmCtx)); + if (ctx == NULL) { + SCLogDebug("Unable to alloc SpmCtx."); + return NULL; + } + memset(ctx, 0, sizeof(SpmCtx)); + ctx->matcher = SPM_HS; + + SpmHsCtx *sctx = SCMalloc(sizeof(SpmHsCtx)); + if (sctx == NULL) { + SCLogDebug("Unable to alloc SpmHsCtx."); + SCFree(ctx); + return NULL; + } + ctx->ctx = sctx; + + memset(sctx, 0, sizeof(SpmHsCtx)); + if (HSBuildDatabase(needle, needle_len, nocase, sctx, + global_thread_ctx) != 0) { + SCLogDebug("HSBuildDatabase failed."); + HSDestroyCtx(ctx); + return NULL; + } + + return ctx; +} + +static uint8_t *HSScan(const SpmCtx *ctx, SpmThreadCtx *thread_ctx, + const uint8_t *haystack, uint16_t haystack_len) +{ + const SpmHsCtx *sctx = ctx->ctx; + hs_scratch_t *scratch = thread_ctx->ctx; + + uint64_t match_offset = UINT64_MAX; + hs_error_t err = hs_scan(sctx->db, (const char *)haystack, haystack_len, 0, + scratch, MatchEvent, &match_offset); + if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) { + /* An error value (other than HS_SCAN_TERMINATED) from hs_scan() + * indicates that it was passed an invalid database or scratch region, + * which is not something we can recover from at scan time. */ + SCLogError(SC_ERR_FATAL, "Hyperscan returned fatal error %d.", err); + exit(EXIT_FAILURE); + } + + if (match_offset == UINT64_MAX) { + return NULL; + } + + BUG_ON(match_offset < sctx->needle_len); + BUG_ON(match_offset > UINT16_MAX); /* haystack_len is a uint16_t */ + + /* Note: existing API returns non-const ptr */ + return (uint8_t *)haystack + (match_offset - sctx->needle_len); +} + +static SpmGlobalThreadCtx *HSInitGlobalThreadCtx(void) +{ + SpmGlobalThreadCtx *global_thread_ctx = SCMalloc(sizeof(SpmGlobalThreadCtx)); + if (global_thread_ctx == NULL) { + SCLogDebug("Unable to alloc SpmGlobalThreadCtx."); + return NULL; + } + memset(global_thread_ctx, 0, sizeof(*global_thread_ctx)); + global_thread_ctx->matcher = SPM_HS; + + /* We store scratch in the HS-specific ctx. This will be initialized as + * patterns are compiled by SpmInitCtx. */ + global_thread_ctx->ctx = NULL; + + return global_thread_ctx; +} + +static void HSDestroyGlobalThreadCtx(SpmGlobalThreadCtx *global_thread_ctx) +{ + if (global_thread_ctx == NULL) { + return; + } + hs_free_scratch(global_thread_ctx->ctx); + SCFree(global_thread_ctx); +} + +static void HSDestroyThreadCtx(SpmThreadCtx *thread_ctx) +{ + if (thread_ctx == NULL) { + return; + } + hs_free_scratch(thread_ctx->ctx); + SCFree(thread_ctx); +} + +static SpmThreadCtx *HSMakeThreadCtx(const SpmGlobalThreadCtx *global_thread_ctx) +{ + SpmThreadCtx *thread_ctx = SCMalloc(sizeof(SpmThreadCtx)); + if (thread_ctx == NULL) { + SCLogDebug("Unable to alloc SpmThreadCtx."); + return NULL; + } + memset(thread_ctx, 0, sizeof(*thread_ctx)); + thread_ctx->matcher = SPM_HS; + + if (global_thread_ctx->ctx != NULL) { + hs_scratch_t *scratch = NULL; + hs_error_t err = hs_clone_scratch(global_thread_ctx->ctx, &scratch); + if (err != HS_SUCCESS) { + SCLogError(SC_ERR_FATAL, "Unable to clone scratch (error %d).", + err); + exit(EXIT_FAILURE); + } + thread_ctx->ctx = scratch; + } + + return thread_ctx; +} + +void SpmHSRegister(void) +{ + spm_table[SPM_HS].name = "hs"; + spm_table[SPM_HS].InitGlobalThreadCtx = HSInitGlobalThreadCtx; + spm_table[SPM_HS].DestroyGlobalThreadCtx = HSDestroyGlobalThreadCtx; + spm_table[SPM_HS].MakeThreadCtx = HSMakeThreadCtx; + spm_table[SPM_HS].DestroyThreadCtx = HSDestroyThreadCtx; + spm_table[SPM_HS].InitCtx = HSInitCtx; + spm_table[SPM_HS].DestroyCtx = HSDestroyCtx; + spm_table[SPM_HS].Scan = HSScan; +} + +#endif /* BUILD_HYPERSCAN */ diff --git a/src/util-spm-hs.h b/src/util-spm-hs.h new file mode 100644 index 0000000000..e362198819 --- /dev/null +++ b/src/util-spm-hs.h @@ -0,0 +1,31 @@ +/* Copyright (C) 2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Justin Viiret + * + * Single pattern matcher that uses the Hyperscan regex matcher. + */ + +#ifndef __UTIL_SPM_HS_H__ +#define __UTIL_SPM_HS_H__ + +void SpmHSRegister(void); + +#endif /* __UTIL_SPM_HS_H__ */ diff --git a/src/util-spm.c b/src/util-spm.c index aa09b98bc8..6e1192a892 100644 --- a/src/util-spm.c +++ b/src/util-spm.c @@ -53,6 +53,7 @@ #include "util-spm-bs.h" #include "util-spm-bs2bm.h" #include "util-spm-bm.h" +#include "util-spm-hs.h" #include "util-clock.h" /** @@ -89,6 +90,9 @@ void SpmTableSetup(void) memset(spm_table, 0, sizeof(spm_table)); SpmBMRegister(); +#ifdef BUILD_HYPERSCAN + SpmHSRegister(); +#endif } SpmGlobalThreadCtx *SpmInitGlobalThreadCtx(uint16_t matcher) diff --git a/src/util-spm.h b/src/util-spm.h index 310285695c..a29da29d32 100644 --- a/src/util-spm.h +++ b/src/util-spm.h @@ -30,6 +30,7 @@ enum { SPM_BM, /* Boyer-Moore */ + SPM_HS, /* Hyperscan */ /* Other SPM matchers will go here. */ SPM_TABLE_SIZE };