From a8938f449d71554a31b1c585e3c8e75ab1b7e09c Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Tue, 2 Apr 2019 16:14:36 -0700 Subject: [PATCH] logging: Anomaly logging This changeset adds anomaly logging to suricata for issue 2282. Anomaly logging is controlled via the `anomaly` section within eve-log. There is a single option -- `packethdr` -- for including the packet header in the anomaly. --- src/Makefile.am | 1 + src/output-json-anomaly.c | 348 ++++++++++++++++++++++++++++++++++++++ src/output-json-anomaly.h | 37 ++++ src/output.c | 2 + src/suricata-common.h | 1 + src/util-profiling.c | 1 + suricata.yaml.in | 2 + 7 files changed, 392 insertions(+) create mode 100644 src/output-json-anomaly.c create mode 100644 src/output-json-anomaly.h diff --git a/src/Makefile.am b/src/Makefile.am index a762c590f5..fa5c8b9ff1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -299,6 +299,7 @@ output-filedata.c output-filedata.h \ output-filestore.c output-filestore.h \ output-flow.c output-flow.h \ output-json-alert.c output-json-alert.h \ +output-json-anomaly.c output-json-anomaly.h \ output-json-dns.c output-json-dns.h \ output-json-dnp3.c output-json-dnp3.h \ output-json-dnp3-objects.c output-json-dnp3-objects.h \ diff --git a/src/output-json-anomaly.c b/src/output-json-anomaly.c new file mode 100644 index 0000000000..abe444538e --- /dev/null +++ b/src/output-json-anomaly.c @@ -0,0 +1,348 @@ +/* Copyright (C) 2019 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Jeff Lucovsky + * + * Logs anomalies in JSON format. + * + */ + +#include "suricata-common.h" +#include "debug.h" +#include "detect.h" +#include "flow.h" +#include "conf.h" + +#include "threads.h" +#include "tm-threads.h" +#include "threadvars.h" +#include "util-debug.h" + +#include "util-misc.h" + +#include "detect-parse.h" +#include "detect-engine.h" +#include "util-logopenfile.h" + +#include "output.h" +#include "output-json.h" +#include "output-json-anomaly.h" + +#include "util-byte.h" +#include "util-byte.h" +#include "util-privs.h" +#include "util-print.h" +#include "util-proto-name.h" +#include "util-optimize.h" +#include "util-buffer.h" +#include "util-validate.h" + +#define MODULE_NAME "JsonAnomalyLog" + +#ifdef HAVE_LIBJANSSON + +#define LOG_JSON_PACKET BIT_U16(0) +#define JSON_STREAM_BUFFER_SIZE 4096 + +typedef struct AnomalyJsonOutputCtx_ { + LogFileCtx* file_ctx; + uint16_t flags; + OutputJsonCommonSettings cfg; +} AnomalyJsonOutputCtx; + +typedef struct JsonAnomalyLogThread_ { + /** LogFileCtx has the pointer to the file and a mutex to allow multithreading */ + LogFileCtx* file_ctx; + MemBuffer *json_buffer; + AnomalyJsonOutputCtx* json_output_ctx; +} JsonAnomalyLogThread; + +static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet *p) +{ + bool is_IP_pkt = PKT_IS_IPV4(p) || PKT_IS_IPV6(p); + + char timebuf[64]; + CreateIsoTimeString(&p->ts, timebuf, sizeof(timebuf)); + + for (int i = 0; i < p->events.cnt; i++) { + MemBufferReset(aft->json_buffer); + + json_t *js; + if (is_IP_pkt) { + js = CreateJSONHeader(p, LOG_DIR_PACKET, "anomaly"); + } else { + js = json_object(); + } + + if (unlikely(js == NULL)) { + return TM_ECODE_OK; + } + + json_t *ajs = json_object(); + if (unlikely(ajs == NULL)) { + json_decref(js); + return TM_ECODE_OK; + } + + if (!is_IP_pkt) { + json_object_set_new(js, "timestamp", json_string(timebuf)); + } else { + JsonFiveTuple((const Packet *)p, LOG_DIR_PACKET, js); + JsonAddCommonOptions(&aft->json_output_ctx->cfg, p, p->flow, js); + } + + if (aft->json_output_ctx->flags & LOG_JSON_PACKET) { + char buf[(32 * 3) + 1]; + PrintRawLineHexBuf(buf, sizeof(buf), GET_PKT_DATA(p), + GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32); + json_object_set_new(js, "packethdr", json_string((char *)buf)); + + json_object_set_new(js, "linktype", json_integer(p->datalink)); + } + + uint8_t event_code = p->events.events[i]; + if (EVENT_IS_DECODER_PACKET_ERROR(event_code)) { + const char *event = DEvents[event_code].event_name; + json_object_set_new(ajs, "event", json_string(event)); + } else { + /* include event code with unrecognized events */ + uint32_t offset = 0; + char unknown_event_buf[32]; + PrintBufferData(unknown_event_buf, &offset, 32, "%s(%d)", "Unknown", event_code); + json_object_set_new(ajs, "event", json_string(unknown_event_buf)); + } + + /* anomaly */ + json_object_set_new(js, "anomaly", ajs); + OutputJSONBuffer(js, aft->file_ctx, &aft->json_buffer); + + json_object_clear(js); + json_decref(js); + } + + return TM_ECODE_OK; +} + + +static int JsonAnomalyLogger(ThreadVars *tv, void *thread_data, const Packet *p) +{ + JsonAnomalyLogThread *aft = thread_data; + return AnomalyJson(tv, aft, p); +} + +static int JsonAnomalyLogCondition(ThreadVars *tv, const Packet *p) +{ + return p->events.cnt > 0 ? TRUE : FALSE; +} + +#define OUTPUT_BUFFER_SIZE 65535 +static TmEcode JsonAnomalyLogThreadInit(ThreadVars *t, const void *initdata, void **data) +{ + JsonAnomalyLogThread *aft = SCCalloc(1, sizeof(JsonAnomalyLogThread)); + if (unlikely(aft == NULL)) { + return TM_ECODE_FAILED; + } + + if(initdata == NULL) { + SCLogDebug("Error getting context for EveLogAnomaly. \"initdata\" argument NULL"); + SCFree(aft); + return TM_ECODE_FAILED; + } + + aft->json_buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); + if (aft->json_buffer == NULL) { + SCFree(aft); + return TM_ECODE_FAILED; + } + + /** Use the Output Context (file pointer and mutex) */ + AnomalyJsonOutputCtx *json_output_ctx = ((OutputCtx *)initdata)->data; + aft->file_ctx = json_output_ctx->file_ctx; + aft->json_output_ctx = json_output_ctx; + + *data = (void *)aft; + return TM_ECODE_OK; +} + +static TmEcode JsonAnomalyLogThreadDeinit(ThreadVars *t, void *data) +{ + JsonAnomalyLogThread *aft = (JsonAnomalyLogThread *)data; + if (aft == NULL) { + return TM_ECODE_OK; + } + + MemBufferFree(aft->json_buffer); + + /* clear memory */ + memset(aft, 0, sizeof(JsonAnomalyLogThread)); + + SCFree(aft); + return TM_ECODE_OK; +} + +static void JsonAnomalyLogDeInitCtx(OutputCtx *output_ctx) +{ + AnomalyJsonOutputCtx *json_output_ctx = (AnomalyJsonOutputCtx *) output_ctx->data; + if (json_output_ctx != NULL) { + LogFileFreeCtx(json_output_ctx->file_ctx); + SCFree(json_output_ctx); + } + SCFree(output_ctx); +} + +static void JsonAnomalyLogDeInitCtxSub(OutputCtx *output_ctx) +{ + SCLogDebug("cleaning up sub output_ctx %p", output_ctx); + + AnomalyJsonOutputCtx *json_output_ctx = (AnomalyJsonOutputCtx *) output_ctx->data; + + if (json_output_ctx != NULL) { + SCFree(json_output_ctx); + } + SCFree(output_ctx); +} + +#define DEFAULT_LOG_FILENAME "anomaly.json" +static void SetFlag(const ConfNode *conf, const char *name, uint16_t flag, uint16_t *out_flags) +{ + DEBUG_VALIDATE_BUG_ON(conf == NULL); + const char *setting = ConfNodeLookupChildValue(conf, name); + if (setting != NULL) { + if (ConfValIsTrue(setting)) { + *out_flags |= flag; + } else { + *out_flags &= ~flag; + } + } +} + +static void JsonAnomalyLogConf(AnomalyJsonOutputCtx *json_output_ctx, + ConfNode *conf) +{ + uint16_t flags = 0; + if (conf != NULL) { + SetFlag(conf, "packethdr", LOG_JSON_PACKET, &flags); + } + json_output_ctx->flags |= flags; +} + +/** + * \brief Create a new LogFileCtx for "fast" output style. + * \param conf The configuration node for this output. + * \return A LogFileCtx pointer on success, NULL on failure. + */ +static OutputInitResult JsonAnomalyLogInitCtx(ConfNode *conf) +{ + OutputInitResult result = { NULL, false }; + AnomalyJsonOutputCtx *json_output_ctx = NULL; + LogFileCtx *logfile_ctx = LogFileNewCtx(); + if (logfile_ctx == NULL) { + SCLogDebug("JsonAnomalyLogInitCtx: Could not create new LogFileCtx"); + return result; + } + + if (SCConfLogOpenGeneric(conf, logfile_ctx, DEFAULT_LOG_FILENAME, 1) < 0) { + LogFileFreeCtx(logfile_ctx); + return result; + } + + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); + if (unlikely(output_ctx == NULL)) { + LogFileFreeCtx(logfile_ctx); + return result; + } + + json_output_ctx = SCCalloc(1, sizeof(AnomalyJsonOutputCtx)); + if (unlikely(json_output_ctx == NULL)) { + LogFileFreeCtx(logfile_ctx); + SCFree(output_ctx); + return result; + } + + json_output_ctx->file_ctx = logfile_ctx; + JsonAnomalyLogConf(json_output_ctx, conf); + + output_ctx->data = json_output_ctx; + output_ctx->DeInit = JsonAnomalyLogDeInitCtx; + + result.ctx = output_ctx; + result.ok = true; + return result; +} + +/** + * \brief Create a new LogFileCtx for "fast" output style. + * \param conf The configuration node for this output. + * \return A LogFileCtx pointer on success, NULL on failure. + */ +static OutputInitResult JsonAnomalyLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + OutputInitResult result = { NULL, false }; + OutputJsonCtx *ajt = parent_ctx->data; + AnomalyJsonOutputCtx *json_output_ctx = NULL; + + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); + if (unlikely(output_ctx == NULL)) + return result; + + json_output_ctx = SCCalloc(1, sizeof(AnomalyJsonOutputCtx)); + if (unlikely(json_output_ctx == NULL)) { + goto error; + } + + json_output_ctx->file_ctx = ajt->file_ctx; + JsonAnomalyLogConf(json_output_ctx, conf); + json_output_ctx->cfg = ajt->cfg; + + output_ctx->data = json_output_ctx; + output_ctx->DeInit = JsonAnomalyLogDeInitCtxSub; + + result.ctx = output_ctx; + result.ok = true; + return result; + +error: + if (json_output_ctx != NULL) { + SCFree(json_output_ctx); + } + + SCFree(output_ctx); + + return result; +} + +void JsonAnomalyLogRegister (void) +{ + OutputRegisterPacketModule(LOGGER_JSON_ANOMALY, MODULE_NAME, "anomaly-json-log", + JsonAnomalyLogInitCtx, JsonAnomalyLogger, JsonAnomalyLogCondition, + JsonAnomalyLogThreadInit, JsonAnomalyLogThreadDeinit, NULL); + OutputRegisterPacketSubModule(LOGGER_JSON_ANOMALY, "eve-log", MODULE_NAME, + "eve-log.anomaly", JsonAnomalyLogInitCtxSub, JsonAnomalyLogger, + JsonAnomalyLogCondition, JsonAnomalyLogThreadInit, JsonAnomalyLogThreadDeinit, + NULL); +} + +#else + +void JsonAnomalyLogRegister (void) +{ +} + +#endif diff --git a/src/output-json-anomaly.h b/src/output-json-anomaly.h new file mode 100644 index 0000000000..2e58551b5d --- /dev/null +++ b/src/output-json-anomaly.h @@ -0,0 +1,37 @@ +/* Copyright (C) 2019 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Jeff Lucovsky + * + * Logs anomalies in JSON format. + * + */ + +#ifndef __OUTPUT_JSON_ANOMALY_H__ +#define __OUTPUT_JSON_ANOMALY_H__ + +void JsonAnomalyLogRegister(void); +#ifdef HAVE_LIBJANSSON +void AnomalyJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, json_t *js, + uint16_t flags); +#endif /* HAVE_LIBJANSSON */ + +#endif /* __OUTPUT_JSON_ALERT_H__ */ + diff --git a/src/output.c b/src/output.c index f2bdedcf9c..576183b35c 100644 --- a/src/output.c +++ b/src/output.c @@ -46,6 +46,7 @@ #include "alert-prelude.h" #include "alert-syslog.h" #include "output-json-alert.h" +#include "output-json-anomaly.h" #include "output-json-flow.h" #include "output-json-netflow.h" #include "log-cf-common.h" @@ -1079,6 +1080,7 @@ void OutputRegisterLoggers(void) LogStatsLogRegister(); JsonAlertLogRegister(); + JsonAnomalyLogRegister(); /* flow/netflow */ JsonFlowLogRegister(); JsonNetFlowLogRegister(); diff --git a/src/suricata-common.h b/src/suricata-common.h index 859f312530..788e6037c2 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -447,6 +447,7 @@ typedef enum { LOGGER_ALERT_SYSLOG, LOGGER_DROP, LOGGER_JSON_ALERT, + LOGGER_JSON_ANOMALY, LOGGER_JSON_DROP, LOGGER_FILE_STORE, LOGGER_JSON_FILE, diff --git a/src/util-profiling.c b/src/util-profiling.c index 0aa9683778..91a6e6bd11 100644 --- a/src/util-profiling.c +++ b/src/util-profiling.c @@ -1297,6 +1297,7 @@ const char * PacketProfileLoggertIdToString(LoggerId id) CASE_CODE (LOGGER_ALERT_SYSLOG); CASE_CODE (LOGGER_DROP); CASE_CODE (LOGGER_JSON_ALERT); + CASE_CODE (LOGGER_JSON_ANOMALY); CASE_CODE (LOGGER_JSON_DROP); CASE_CODE (LOGGER_JSON_SSH); CASE_CODE (LOGGER_DNS_TS); diff --git a/suricata.yaml.in b/suricata.yaml.in index abe169687e..091cbd96c1 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -154,6 +154,8 @@ outputs: # Enable the logging of tagged packets for rules using the # "tag" keyword. tagged-packets: yes + - anomaly: + # packethdr: no # enable dumping of packet header - http: extended: yes # enable this for extended logging information # custom allows additional http fields to be included in eve-log