From 3fc63d365611d47cabcdbb070925b6f48facfc04 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 30 Jan 2014 12:26:01 +0100 Subject: [PATCH] jansson file log: make file log module Turn the libjansson based file logger into a file module, as a child of eve-log. --- src/output-json-file.c | 171 ++++++++++++++++++++-------------------- src/output-json-file.h | 1 + src/output-json.c | 13 --- src/suricata.c | 2 + src/tm-threads-common.h | 1 + 5 files changed, 90 insertions(+), 98 deletions(-) diff --git a/src/output-json-file.c b/src/output-json-file.c index 6eb33c46f6..218a0f89d1 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -64,10 +64,16 @@ #include typedef struct OutputFileCtx_ { + LogFileCtx *file_ctx; uint32_t file_cnt; } OutputFileCtx; -static json_t *LogFileMetaGetUri(Packet *p, File *ff) { +typedef struct JsonFileLogThread_ { + OutputFileCtx *filelog_ctx; + MemBuffer *buffer; +} JsonFileLogThread; + +static json_t *LogFileMetaGetUri(const Packet *p, const File *ff) { HtpState *htp_state = (HtpState *)p->flow->alstate; json_t *js = NULL; if (htp_state != NULL) { @@ -88,7 +94,7 @@ static json_t *LogFileMetaGetUri(Packet *p, File *ff) { return json_string(""); } -static json_t *LogFileMetaGetHost(Packet *p, File *ff) { +static json_t *LogFileMetaGetHost(const Packet *p, const File *ff) { HtpState *htp_state = (HtpState *)p->flow->alstate; json_t *js = NULL; if (htp_state != NULL) { @@ -106,7 +112,7 @@ static json_t *LogFileMetaGetHost(Packet *p, File *ff) { return json_string(""); } -static json_t *LogFileMetaGetReferer(Packet *p, File *ff) { +static json_t *LogFileMetaGetReferer(const Packet *p, const File *ff) { HtpState *htp_state = (HtpState *)p->flow->alstate; json_t *js = NULL; if (htp_state != NULL) { @@ -129,7 +135,7 @@ static json_t *LogFileMetaGetReferer(Packet *p, File *ff) { return json_string(""); } -static json_t *LogFileMetaGetUserAgent(Packet *p, File *ff) { +static json_t *LogFileMetaGetUserAgent(const Packet *p, const File *ff) { HtpState *htp_state = (HtpState *)p->flow->alstate; json_t *js = NULL; if (htp_state != NULL) { @@ -156,9 +162,9 @@ static json_t *LogFileMetaGetUserAgent(Packet *p, File *ff) { * \internal * \brief Write meta data on a single line json record */ -static void LogFileWriteJsonRecord(AlertJsonThread /*LogFileLogThread*/ *aft, Packet *p, File *ff, int ipver) { +static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff) { MemBuffer *buffer = (MemBuffer *)aft->buffer; - json_t *js = CreateJSONHeader(p, 0); + json_t *js = CreateJSONHeader((Packet *)p, 0); //TODO const if (unlikely(js == NULL)) return; @@ -216,122 +222,91 @@ static void LogFileWriteJsonRecord(AlertJsonThread /*LogFileLogThread*/ *aft, Pa json_object_set_new(fjs, "size", json_integer(ff->size)); json_object_set_new(js, "file", fjs); - OutputJSON(js, aft, &aft->files_cnt); + OutputJSONBuffer(js, aft->filelog_ctx->file_ctx, buffer); json_object_del(js, "file"); json_object_clear(js); json_decref(js); } -static TmEcode OutputFileLogWrap(ThreadVars *tv, Packet *p, void *data, int ipver) +static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff) { SCEnter(); - AlertJsonThread *aft = (AlertJsonThread *)data; - uint8_t flags = 0; - uint8_t direction = 0; - - /* no flow, no htp state */ - if (p->flow == NULL) { - SCReturnInt(TM_ECODE_OK); - } - - if (p->flowflags & FLOW_PKT_TOCLIENT) - flags |= (direction = STREAM_TOCLIENT); - else - flags |= (direction = STREAM_TOSERVER); - - int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0; - int file_trunc = 0; - - FLOWLOCK_WRLOCK(p->flow); - file_trunc = StreamTcpReassembleDepthReached(p); - - Flow *f = p->flow; - FileContainer *ffc = AppLayerParserGetFiles(f->proto, f->alproto, f->alstate, direction); - SCLogDebug("ffc %p", ffc); - if (ffc != NULL) { - File *ff; - for (ff = ffc->head; ff != NULL; ff = ff->next) { - if (ff->flags & FILE_LOGGED) - continue; - - if (FileForceMagic() && ff->magic == NULL) { - FilemagicGlobalLookup(ff); - } + JsonFileLogThread *aft = (JsonFileLogThread *)thread_data; - SCLogDebug("ff %p", ff); + BUG_ON(ff->flags & FILE_LOGGED); - if (file_trunc && ff->state < FILE_STATE_CLOSED) - ff->state = FILE_STATE_TRUNCATED; + SCLogDebug("ff %p", ff); - if (ff->state == FILE_STATE_CLOSED || - ff->state == FILE_STATE_TRUNCATED || ff->state == FILE_STATE_ERROR || - (file_close == 1 && ff->state < FILE_STATE_CLOSED)) - { - LogFileWriteJsonRecord(aft, p, ff, ipver); + FileWriteJsonRecord(aft, p, ff); + return 0; +} - ff->flags |= FILE_LOGGED; - } - } - FilePrune(ffc); +#define OUTPUT_BUFFER_SIZE 65535 +static TmEcode JsonFileLogThreadInit(ThreadVars *t, void *initdata, void **data) +{ + JsonFileLogThread *aft = SCMalloc(sizeof(JsonFileLogThread)); + if (unlikely(aft == NULL)) + return TM_ECODE_FAILED; + memset(aft, 0, sizeof(JsonFileLogThread)); + + if(initdata == NULL) + { + SCLogDebug("Error getting context for HTTPLog. \"initdata\" argument NULL"); + SCFree(aft); + return TM_ECODE_FAILED; } - FLOWLOCK_UNLOCK(p->flow); - SCReturnInt(TM_ECODE_OK); -} + /* Use the Ouptut Context (file pointer and mutex) */ + aft->filelog_ctx = ((OutputCtx *)initdata)->data; -TmEcode OutputFileLogIPv4(ThreadVars *tv, Packet *p, void *data) -{ - return OutputFileLogWrap(tv, p, data, AF_INET); -} + aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); + if (aft->buffer == NULL) { + SCFree(aft); + return TM_ECODE_FAILED; + } -TmEcode OutputFileLogIPv6(ThreadVars *tv, Packet *p, void *data) -{ - return OutputFileLogWrap(tv, p, data, AF_INET6); + *data = (void *)aft; + return TM_ECODE_OK; } -TmEcode OutputFileLog (ThreadVars *tv, Packet *p, void *data) +static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data) { - SCEnter(); - int r = TM_ECODE_OK; - - /* no flow, no htp state */ - if (p->flow == NULL) { - SCReturnInt(TM_ECODE_OK); - } - - if (!(PKT_IS_TCP(p))) { - SCReturnInt(TM_ECODE_OK); + JsonFileLogThread *aft = (JsonFileLogThread *)data; + if (aft == NULL) { + return TM_ECODE_OK; } - SCLogDebug("p->pcap_cnt %"PRIu64, p->pcap_cnt); - - if (PKT_IS_IPV4(p)) { - r = OutputFileLogIPv4(tv, p, data); - } else if (PKT_IS_IPV6(p)) { - r = OutputFileLogIPv6(tv, p, data); - } + MemBufferFree(aft->buffer); + /* clear memory */ + memset(aft, 0, sizeof(JsonFileLogThread)); - SCReturnInt(r); + SCFree(aft); + return TM_ECODE_OK; } + /** \brief Create a new http log LogFileCtx. * \param conf Pointer to ConfNode containing this loggers configuration. * \return NULL if failure, LogFileCtx* to the file_ctx if succesful * */ -OutputCtx *OutputFileLogInit(ConfNode *conf) +OutputCtx *OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) { - OutputFileCtx *file_ctx = SCMalloc(sizeof(OutputFileCtx)); - if (unlikely(file_ctx == NULL)) + AlertJsonThread *ajt = parent_ctx->data; + + OutputFileCtx *output_file_ctx = SCMalloc(sizeof(OutputFileCtx)); + if (unlikely(output_file_ctx == NULL)) return NULL; OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); if (unlikely(output_ctx == NULL)) { - SCFree(file_ctx); + SCFree(output_file_ctx); return NULL; } + output_file_ctx->file_ctx = ajt->file_ctx; + if (conf) { const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic"); if (force_magic != NULL && ConfValIsTrue(force_magic)) { @@ -350,8 +325,34 @@ OutputCtx *OutputFileLogInit(ConfNode *conf) } } + output_ctx->data = output_file_ctx; + FileForceTrackingEnable(); return output_ctx; } +void TmModuleJsonFileLogRegister (void) { + tmm_modules[TMM_JSONFILELOG].name = "JsonFileLog"; + tmm_modules[TMM_JSONFILELOG].ThreadInit = JsonFileLogThreadInit; + tmm_modules[TMM_JSONFILELOG].ThreadDeinit = JsonFileLogThreadDeinit; + + /* register as child of eve-log */ + OutputRegisterFileSubModule("eve-log", "JsonFileLog", "eve-log.files", + OutputFileLogInitSub, JsonFileLogger); +} + +#else + +static TmEcode OutputJsonThreadInit(ThreadVars *t, void *initdata, void **data) +{ + SCLogInfo("Can't init JSON output - JSON support was disabled during build."); + return TM_ECODE_FAILED; +} + +void TmModuleJsonFileLogRegister (void) +{ + tmm_modules[TMM_JSONFILELOG].name = "JsonFileLog"; + tmm_modules[TMM_JSONFILELOG].ThreadInit = OutputJsonThreadInit; +} + #endif diff --git a/src/output-json-file.h b/src/output-json-file.h index a97c0d54ae..cb39618157 100644 --- a/src/output-json-file.h +++ b/src/output-json-file.h @@ -26,5 +26,6 @@ TmEcode OutputFileLog (ThreadVars *tv, Packet *p, void *data); OutputCtx *OutputFileLogInit(ConfNode *); +void TmModuleJsonFileLogRegister (void); #endif /* __OUTPUT_FILELOG_H__ */ diff --git a/src/output-json.c b/src/output-json.c index 5a88a7c924..226cc4d35c 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -163,8 +163,6 @@ static enum JsonOutput json_out = ALERT_FILE; #define OUTPUT_HTTP (1<<4) #define OUTPUT_TLS (1<<5) -static uint32_t output_flags = 0; - enum JsonFormat { COMPACT, INDENT }; static enum JsonFormat format = COMPACT; @@ -357,10 +355,6 @@ TmEcode OutputJSON(json_t *js, void *data, uint64_t *count) TmEcode OutputJson (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { - if (output_flags & OUTPUT_FILES) { - OutputFileLog(tv, p, data); - } - return TM_ECODE_OK; } @@ -527,13 +521,6 @@ OutputCtx *OutputJsonInitCtx(ConfNode *conf) * registration capability */ TAILQ_FOREACH(output, &outputs->head, next) { - if (strcmp(output->val, "files") == 0) { - SCLogDebug("Enabling files output"); - ConfNode *child = ConfNodeLookupChild(output, "files"); - json_ctx->files_ctx = OutputFileLogInit(child); - output_flags |= OUTPUT_FILES; - continue; - } } } } diff --git a/src/suricata.c b/src/suricata.c index 1a20728dc2..2b2c88a133 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -90,6 +90,7 @@ #include "output-tlslog.h" #include "log-pcap.h" #include "log-file.h" +#include "output-json-file.h" #include "log-filestore.h" #include "output-json.h" @@ -811,6 +812,7 @@ void RegisterAllModules() TmModulePcapLogRegister(); /* file log */ TmModuleLogFileLogRegister(); + TmModuleJsonFileLogRegister(); TmModuleLogFilestoreRegister(); /* dns log */ TmModuleLogDnsLogRegister(); diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h index 09b8e6b25f..1336d77c0b 100644 --- a/src/tm-threads-common.h +++ b/src/tm-threads-common.h @@ -89,6 +89,7 @@ typedef enum { TMM_JSONHTTPLOG, TMM_JSONDNSLOG, TMM_JSONTLSLOG, + TMM_JSONFILELOG, TMM_SIZE, } TmmId;