From caea596ce5682fadcb113a6a7c8ab4a2bc15a1b2 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 19 Oct 2015 17:39:57 +0200 Subject: [PATCH] profiling: output post-prefilter matches Dump a json record containing all sigs that need to be inspected after prefilter. Part of profiling. Only dump if threshold is met, which is currently set by: --set detect.profiling.inspect-logging-threshold=200 A file called packet_inspected_rules.json is created in the default log dir. --- src/Makefile.am | 1 + src/detect-engine-profile.c | 136 ++++++++++++++++++++++++++++++++++++ src/detect-engine-profile.h | 29 ++++++++ src/detect-engine.c | 5 ++ src/detect.c | 7 +- src/detect.h | 1 + suricata.yaml.in | 7 ++ 7 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 src/detect-engine-profile.c create mode 100644 src/detect-engine-profile.h diff --git a/src/Makefile.am b/src/Makefile.am index c3c7c679c1..4d0490a1f3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -121,6 +121,7 @@ detect-engine-mpm.c detect-engine-mpm.h \ detect-engine-payload.c detect-engine-payload.h \ detect-engine-port.c detect-engine-port.h \ detect-engine-proto.c detect-engine-proto.h \ +detect-engine-profile.c detect-engine-profile.h \ detect-engine-siggroup.c detect-engine-siggroup.h \ detect-engine-sigorder.c detect-engine-sigorder.h \ detect-engine-state.c detect-engine-state.h \ diff --git a/src/detect-engine-profile.c b/src/detect-engine-profile.c new file mode 100644 index 0000000000..f512765723 --- /dev/null +++ b/src/detect-engine-profile.c @@ -0,0 +1,136 @@ +/* Copyright (C) 2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + */ + +#include "suricata-common.h" +#include "suricata.h" +#include "detect.h" +#include "detect-parse.h" +#include "detect-content.h" +#include "output-json.h" +#include "util-buffer.h" +#include "util-print.h" + +#ifdef PROFILING +static void DumpFp(const SigMatch *sm, char *pat_orig, uint32_t pat_orig_sz, char *pat_chop, uint32_t pat_chop_sz) +{ + int fast_pattern_chop_set = 0; + const DetectContentData *cd = (DetectContentData *)sm->ctx; + + if (cd->flags & DETECT_CONTENT_FAST_PATTERN) { + if (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) { + fast_pattern_chop_set = 1; + } + } + + uint32_t off = 0; + PrintRawUriBuf(pat_orig, &off, pat_orig_sz, cd->content, cd->content_len); + + if (fast_pattern_chop_set) { + off = 0; + PrintRawUriBuf(pat_chop, &off, pat_chop_sz, cd->content + cd->fp_chop_offset, cd->fp_chop_len); + } +} + +SCMutex g_rule_dump_write_m = SCMUTEX_INITIALIZER; +void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const Packet *p) +{ + json_t *js = CreateJSONHeader(p, 0, "inspectedrules"); + if (js == NULL) + return; + json_t *ir = json_object(); + if (ir == NULL) + return; + + json_object_set_new(ir, "rule_group_id", json_integer(det_ctx->sgh->id)); + json_object_set_new(ir, "rule_cnt", json_integer(det_ctx->match_array_cnt)); + + json_t *js_array = json_array(); + uint32_t x; + for (x = 0; x < det_ctx->match_array_cnt; x++) + { + const Signature *s = det_ctx->match_array[x]; + if (s == NULL) + continue; + + json_t *js_sig = json_object(); + if (unlikely(js == NULL)) + continue; + json_object_set_new(js_sig, "sig_id", json_integer(s->id)); + + json_object_set_new(js_sig, "mpm", (s->mpm_sm != NULL) ? json_true() : json_false()); + + if (s->mpm_sm != NULL) { + char orig[256] = ""; + char chop[256] = ""; + + DumpFp(s->mpm_sm, orig, sizeof(orig), chop, sizeof(chop)); + + json_object_set_new(js_sig, "mpm_buffer", json_string(DetectListToHumanString(SigMatchListSMBelongsTo(s, s->mpm_sm)))); + json_object_set_new(js_sig, "mpm_pattern", json_string(orig)); + + if (strlen(chop) > 0) { + json_object_set_new(js_sig, "mpm_pattern_chop", json_string(chop)); + } + } + json_array_append_new(js_array, js_sig); + } + + json_object_set_new(ir, "rules", js_array); + json_object_set_new(js, "inspectedrules", ir); + + const char *filename = "packet_inspected_rules.json"; + const char *log_dir = ConfigGetLogDirectory(); + char log_path[PATH_MAX] = ""; + snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename); + + MemBuffer *mbuf = NULL; + mbuf = MemBufferCreateNew(4096); + BUG_ON(mbuf == NULL); + + OutputJSONMemBufferWrapper wrapper = { + .buffer = &mbuf, + .expand_by = 4096, + }; + + int r = json_dump_callback(js, OutputJSONMemBufferCallback, &wrapper, + JSON_PRESERVE_ORDER|JSON_COMPACT|JSON_ENSURE_ASCII| + JSON_ESCAPE_SLASH); + if (r != 0) { + SCLogWarning(SC_ERR_SOCKET, "unable to serialize JSON object"); + } else { + MemBufferWriteString(mbuf, "\n"); + SCMutexLock(&g_rule_dump_write_m); + FILE *fp = fopen(log_path, "a"); + if (fp != NULL) { + MemBufferPrintToFPAsString(mbuf, fp); + fclose(fp); + SCMutexUnlock(&g_rule_dump_write_m); + } + } + + MemBufferFree(mbuf); + json_object_clear(js); + json_decref(js); +} +#endif /* PROFILING */ diff --git a/src/detect-engine-profile.h b/src/detect-engine-profile.h new file mode 100644 index 0000000000..d7ed1c93ea --- /dev/null +++ b/src/detect-engine-profile.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef _DETECT_ENGINE_PROFILE_H +#define _DETECT_ENGINE_PROFILE_H + +void RulesDumpMatchArray(const DetectEngineThreadCtx *det_ctx, const Packet *p); + +#endif /* _DETECT_ENGINE_PROFILE_H */ diff --git a/src/detect-engine.c b/src/detect-engine.c index c293a2ea81..e9e9ecec5a 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -881,6 +881,11 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix) #ifdef PROFILING SCProfilingKeywordInitCounters(de_ctx); + de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled + + intmax_t v = 0; + if (ConfGetInt("detect.profiling.inspect-logging-threshold", &v) == 1) + de_ctx->profile_match_logging_threshold = (uint32_t)v; #endif SCClassConfLoadClassficationConfigFile(de_ctx, NULL); diff --git a/src/detect.c b/src/detect.c index 49bd118c37..bdf85559a1 100644 --- a/src/detect.c +++ b/src/detect.c @@ -33,6 +33,7 @@ #include "detect-parse.h" #include "detect-engine.h" +#include "detect-engine-profile.h" #include "detect-engine-alert.h" #include "detect-engine-siggroup.h" @@ -201,7 +202,6 @@ #include "util-optimize.h" #include "util-path.h" #include "util-mpm-ac.h" - #include "runmodes.h" #include @@ -1540,6 +1540,11 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh SGH_PROFILING_RECORD(det_ctx, det_ctx->sgh); +#ifdef PROFILING + if (match_cnt >= de_ctx->profile_match_logging_threshold) + RulesDumpMatchArray(det_ctx, p); +#endif + uint32_t sflags, next_sflags = 0; if (match_cnt) { next_s = *match_array++; diff --git a/src/detect.h b/src/detect.h index f70493f3cb..beb4aca097 100644 --- a/src/detect.h +++ b/src/detect.h @@ -661,6 +661,7 @@ typedef struct DetectEngineCtx_ { struct SCProfileKeywordDetectCtx_ *profile_keyword_ctx; struct SCProfileKeywordDetectCtx_ *profile_keyword_ctx_per_list[DETECT_SM_LIST_MAX]; struct SCProfileSghDetectCtx_ *profile_sgh_ctx; + uint32_t profile_match_logging_threshold; #endif char config_prefix[64]; diff --git a/suricata.yaml.in b/suricata.yaml.in index 368ee3408c..59c4d8e161 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -608,6 +608,13 @@ detect-engine: # is started. This will limit the downtime in IPS mode. #- delayed-detect: yes + profiling: + # Log the rules that made it past the prefilter stage, per packet + # default is off. The threshold setting determines how many rules + # must have made it past pre-filter for that rule to trigger the + # logging. + #inspect-logging-threshold: 200 + # Suricata is multi-threaded. Here the threading can be influenced. threading: # On some cpu's/architectures it is beneficial to tie individual threads