From 9ff6608668d31787e59fd7fc997c460bfb7b38e2 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 16 Jan 2014 15:20:09 +0100 Subject: [PATCH] Introduce Filedata Logger API A new logger API for registering file storage handlers. Where the FileLog handler is called once per file, this handler will be called for each data chunk so that storing the entire file is possible. The logger call in the API is as follows: typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, const File *, const FileData *, uint8_t flags); All data is const, thus should be read only. The final flags field is used to indicate to the caller that the file is new, or if it's being closed. Files use an internal unique id 'file_id' which can be used by the loggers to create unique file names. This id can use the 'waldo' feature of the log-filestore module. This patch moves that waldo loading and storing logic to this API's implementation. A new configuration directive 'file-store-waldo: ' is added, but the existing waldo settings will also continue to work. --- src/Makefile.am | 1 + src/output-filedata.c | 440 ++++++++++++++++++++++++++++++++++++++++ src/output-filedata.h | 48 +++++ src/output.c | 38 ++++ src/output.h | 4 + src/runmodes.c | 22 ++ src/suricata.c | 2 + src/tm-modules.c | 1 + src/tm-threads-common.h | 1 + 9 files changed, 557 insertions(+) create mode 100644 src/output-filedata.c create mode 100644 src/output-filedata.h diff --git a/src/Makefile.am b/src/Makefile.am index 86b7c123a6..ff778d813d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -215,6 +215,7 @@ log-pcap.c log-pcap.h \ log-tlslog.c log-tlslog.h \ output.c output.h \ output-file.c output-file.h \ +output-filedata.c output-filedata.h \ output-packet.c output-packet.h \ output-tx.c output-tx.h \ packet-queue.c packet-queue.h \ diff --git a/src/output-filedata.c b/src/output-filedata.c new file mode 100644 index 0000000000..967dcffd70 --- /dev/null +++ b/src/output-filedata.c @@ -0,0 +1,440 @@ +/* Copyright (C) 2007-2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * AppLayer Filedata Logger Output registration functions + */ + +#include "suricata-common.h" +#include "tm-modules.h" +#include "output-filedata.h" +#include "app-layer.h" +#include "app-layer-parser.h" +#include "detect-filemagic.h" +#include "conf.h" + +typedef struct OutputLoggerThreadStore_ { + void *thread_data; + struct OutputLoggerThreadStore_ *next; +} OutputLoggerThreadStore; + +/** per thread data for this module, contains a list of per thread + * data for the packet loggers. */ +typedef struct OutputLoggerThreadData_ { + OutputLoggerThreadStore *store; +} OutputLoggerThreadData; + +/* logger instance, a module + a output ctx, + * it's perfectly valid that have multiple instances of the same + * log module (e.g. http.log) with different output ctx'. */ +typedef struct OutputFiledataLogger_ { + FiledataLogger LogFunc; + OutputCtx *output_ctx; + struct OutputFiledataLogger_ *next; + const char *name; +} OutputFiledataLogger; + +static OutputFiledataLogger *list = NULL; +static char g_waldo[PATH_MAX] = ""; +static SCMutex g_waldo_mutex = SCMUTEX_INITIALIZER; +static int g_waldo_init = 0; +static int g_waldo_deinit = 0; + +int OutputRegisterFiledataLogger(char *name, FiledataLogger LogFunc, OutputCtx *output_ctx) { + OutputFiledataLogger *op = SCMalloc(sizeof(*op)); + if (op == NULL) + return -1; + memset(op, 0x00, sizeof(*op)); + + op->LogFunc = LogFunc; + op->output_ctx = output_ctx; + op->name = SCStrdup(name); + if (op->name == NULL) { + SCFree(op); + return -1; + } + + if (list == NULL) + list = op; + else { + OutputFiledataLogger *t = list; + while (t->next) + t = t->next; + t->next = op; + } + + SCLogDebug("OutputRegisterTxLogger happy"); + return 0; +} + +SC_ATOMIC_DECLARE(unsigned int, file_id); + +static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq) { + BUG_ON(thread_data == NULL); + BUG_ON(list == NULL); + + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputFiledataLogger *logger = list; + OutputLoggerThreadStore *store = op_thread_data->store; + + BUG_ON(logger == NULL && store != NULL); + BUG_ON(logger != NULL && store == NULL); + BUG_ON(logger == NULL && store == NULL); + + uint8_t flags = 0; + Flow * const f = p->flow; + + /* no flow, no files */ + if (f == NULL) { + SCReturnInt(TM_ECODE_OK); + } + + if (p->flowflags & FLOW_PKT_TOCLIENT) + flags |= STREAM_TOCLIENT; + else + flags |= STREAM_TOSERVER; + + int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0; + int file_trunc = 0; + + FLOWLOCK_WRLOCK(f); // < need write lock for FiledataPrune below + file_trunc = StreamTcpReassembleDepthReached(p); + + FileContainer *ffc = AppLayerParserGetFiles(p->proto, f->alproto, + f->alstate, flags); + SCLogDebug("ffc %p", ffc); + if (ffc != NULL) { + File *ff; + for (ff = ffc->head; ff != NULL; ff = ff->next) { + if (FileForceMagic() && ff->magic == NULL) { + FilemagicGlobalLookup(ff); + } + + SCLogDebug("ff %p", ff); + if (ff->flags & FILE_STORED) { + SCLogDebug("stored flag set"); + continue; + } + + if (!(ff->flags & FILE_STORE)) { + SCLogDebug("ff FILE_STORE not set"); + continue; + } + + FileData *ffd; + for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) { + uint8_t flags = 0; + int file_logged = 0; + FileData *write_ffd = ffd; + + SCLogDebug("ffd %p", ffd); + + /* special case: on stream end we may inform the + * loggers that the file is truncated. In this + * case we already logged the current ffd, which + * is the last in our list. */ + if (ffd->stored == 1) { + if (!(file_close == 1 || ffd->next == NULL)) { + continue; + } + + // call writer with NULL ffd, so it can 'close' + // so really a 'close' msg + write_ffd = NULL; + flags |= OUTPUT_FILEDATA_FLAG_CLOSE; + } + + /* store */ + + /* if file_id == 0, this is the first store of this file */ + if (ff->file_id == 0) { + /* new file */ + ff->file_id = SC_ATOMIC_ADD(file_id, 1); + flags |= OUTPUT_FILEDATA_FLAG_OPEN; + } else { + /* existing file */ + } + + /* if file needs to be closed or truncated, inform + * loggers */ + if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) { + ff->state = FILE_STATE_TRUNCATED; + } + + /* for the last data chunk we have, also tell the logger + * we're closing up */ + if (ffd->next == NULL && ff->state >= FILE_STATE_CLOSED) + flags |= OUTPUT_FILEDATA_FLAG_CLOSE; + + logger = list; + store = op_thread_data->store; + while (logger && store) { + BUG_ON(logger->LogFunc == NULL); + + SCLogDebug("logger %p", logger); + logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, + (const FileData *)write_ffd, flags); + file_logged = 1; + + logger = logger->next; + store = store->next; + + BUG_ON(logger == NULL && store != NULL); + BUG_ON(logger != NULL && store == NULL); + } + + if (file_logged) { + ffd->stored = 1; + + /* all done */ + if (flags & OUTPUT_FILEDATA_FLAG_CLOSE) { + ff->flags |= FILE_STORED; + break; + } + } + } + } + + FilePrune(ffc); + } + + FLOWLOCK_UNLOCK(f); + return TM_ECODE_OK; +} + +/** + * \internal + * + * \brief Open the waldo file (if available) and load the file_id + * + * \param path full path for the waldo file + */ +static void LogFiledataLogLoadWaldo(const char *path) { + char line[16] = ""; + unsigned int id = 0; + + FILE *fp = fopen(path, "r"); + if (fp == NULL) { + SCLogInfo("couldn't open waldo: %s", strerror(errno)); + SCReturn; + } + + if (fgets(line, (int)sizeof(line), fp) != NULL) { + if (sscanf(line, "%10u", &id) == 1) { + SCLogInfo("id %u", id); + (void) SC_ATOMIC_CAS(&file_id, 0, id); + } + } + fclose(fp); +} + +/** + * \internal + * + * \brief Store the waldo file based on the file_id + * + * \param path full path for the waldo file + */ +static void LogFiledataLogStoreWaldo(const char *path) { + char line[16] = ""; + + if (SC_ATOMIC_GET(file_id) == 0) { + SCReturn; + } + + FILE *fp = fopen(path, "w"); + if (fp == NULL) { + SCLogInfo("couldn't open waldo: %s", strerror(errno)); + SCReturn; + } + + snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(file_id)); + if (fwrite(line, strlen(line), 1, fp) != 1) { + SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno)); + } + fclose(fp); +} + +/** \brief thread init for the tx logger + * This will run the thread init functions for the individual registered + * loggers */ +static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, void *initdata, void **data) { + OutputLoggerThreadData *td = SCMalloc(sizeof(*td)); + if (td == NULL) + return TM_ECODE_FAILED; + memset(td, 0x00, sizeof(*td)); + + *data = (void *)td; + + SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data); + + OutputFiledataLogger *logger = list; + while (logger) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadInit) { + void *retptr = NULL; + if (tm_module->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) { + OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts)); +/* todo */ BUG_ON(ts == NULL); + memset(ts, 0x00, sizeof(*ts)); + + /* store thread handle */ + ts->thread_data = retptr; + + if (td->store == NULL) { + td->store = ts; + } else { + OutputLoggerThreadStore *tmp = td->store; + while (tmp->next != NULL) + tmp = tmp->next; + tmp->next = ts; + } + + SCLogDebug("%s is now set up", logger->name); + } + } + + logger = logger->next; + } + + SCMutexLock(&g_waldo_mutex); + if (g_waldo_init == 0) { + ConfNode *node = ConfGetNode("file-store-waldo"); + if (node == NULL) { + ConfNode *outputs = ConfGetNode("outputs"); + if (outputs) { + ConfNode *output; + TAILQ_FOREACH(output, &outputs->head, next) { + /* we only care about file and file-store */ + if (!(strcmp(output->val, "file") == 0 || strcmp(output->val, "file-store") == 0)) + continue; + + ConfNode *file = ConfNodeLookupChild(output, output->val); + BUG_ON(file == NULL); + if (file == NULL) { + SCLogDebug("file-store failed, lets try 'file'"); + file = ConfNodeLookupChild(outputs, "file"); + if (file == NULL) + SCLogDebug("file failed as well, giving up"); + } + + if (file != NULL) { + node = ConfNodeLookupChild(file, "waldo"); + if (node == NULL) + SCLogDebug("no waldo node"); + } + } + } + } + if (node != NULL) { + char *s_default_log_dir = NULL; + s_default_log_dir = ConfigGetLogDirectory(); + + const char *waldo = node->val; + SCLogDebug("loading waldo %s", waldo); + if (waldo != NULL && strlen(waldo) > 0) { + if (PathIsAbsolute(waldo)) { + snprintf(g_waldo, sizeof(g_waldo), "%s", waldo); + } else { + snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo); + } + + SCLogDebug("loading waldo file %s", g_waldo); + LogFiledataLogLoadWaldo(g_waldo); + } + } + g_waldo_init = 1; + } + SCMutexUnlock(&g_waldo_mutex); + return TM_ECODE_OK; +} + +static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data) { + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputLoggerThreadStore *store = op_thread_data->store; + OutputFiledataLogger *logger = list; + + while (logger && store) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadDeinit) { + tm_module->ThreadDeinit(tv, store->thread_data); + } + + logger = logger->next; + store = store->next; + } + + SCMutexLock(&g_waldo_mutex); + if (g_waldo_deinit == 0) { + if (strlen(g_waldo) > 0) { + SCLogDebug("we have a waldo at %s", g_waldo); + LogFiledataLogStoreWaldo(g_waldo); + } + g_waldo_deinit = 1; + } + SCMutexUnlock(&g_waldo_mutex); + return TM_ECODE_OK; +} + +static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data) { + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputLoggerThreadStore *store = op_thread_data->store; + OutputFiledataLogger *logger = list; + + while (logger && store) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadExitPrintStats) { + tm_module->ThreadExitPrintStats(tv, store->thread_data); + } + + logger = logger->next; + store = store->next; + } +} + +void TmModuleFiledataLoggerRegister (void) { + tmm_modules[TMM_FILEDATALOGGER].name = "__filedata_logger__"; + tmm_modules[TMM_FILEDATALOGGER].ThreadInit = OutputFiledataLogThreadInit; + tmm_modules[TMM_FILEDATALOGGER].Func = OutputFiledataLog; + tmm_modules[TMM_FILEDATALOGGER].ThreadExitPrintStats = OutputFiledataLogExitPrintStats; + tmm_modules[TMM_FILEDATALOGGER].ThreadDeinit = OutputFiledataLogThreadDeinit; + tmm_modules[TMM_FILEDATALOGGER].cap_flags = 0; + + SC_ATOMIC_INIT(file_id); +} diff --git a/src/output-filedata.h b/src/output-filedata.h new file mode 100644 index 0000000000..c35fd21ab3 --- /dev/null +++ b/src/output-filedata.h @@ -0,0 +1,48 @@ +/* Copyright (C) 2007-2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * AppLayer Filedata Logger Output registration functions + */ + +#ifndef __OUTPUT_FILEDATA_H__ +#define __OUTPUT_FILEDATA_H__ + +#include "decode.h" +#include "util-file.h" + +#define OUTPUT_FILEDATA_FLAG_OPEN 0x01 +#define OUTPUT_FILEDATA_FLAG_CLOSE 0x02 + +/** filedata logger function pointer type */ +typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, + const File *, const FileData *, uint8_t); + +/** packet logger condition function pointer type, + * must return true for packets that should be logged + */ +//typedef int (*TxLogCondition)(ThreadVars *, const Packet *); + +int OutputRegisterFiledataLogger(char *name, FiledataLogger LogFunc, OutputCtx *); + +void TmModuleFiledataLoggerRegister (void); + +#endif /* __OUTPUT_FILE_H__ */ diff --git a/src/output.c b/src/output.c index cf921e4a5b..286310ac92 100644 --- a/src/output.c +++ b/src/output.c @@ -186,6 +186,44 @@ error: exit(EXIT_FAILURE); } +/** + * \brief Register a file data output module. + * + * This function will register an output module so it can be + * configured with the configuration file. + * + * \retval Returns 0 on success, -1 on failure. + */ +void +OutputRegisterFiledataModule(char *name, char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), FiledataLogger FiledataLogFunc) +{ + if (unlikely(FiledataLogFunc == NULL)) { + goto error; + } + + OutputModule *module = SCCalloc(1, sizeof(*module)); + if (unlikely(module == NULL)) { + goto error; + } + + module->name = SCStrdup(name); + if (unlikely(module->name == NULL)) + goto error; + module->conf_name = SCStrdup(conf_name); + if (unlikely(module->conf_name == NULL)) + goto error; + module->InitFunc = InitFunc; + module->FiledataLogFunc = FiledataLogFunc; + TAILQ_INSERT_TAIL(&output_modules, module, entries); + + SCLogDebug("Filedata logger \"%s\" registered.", name); + return; +error: + SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting..."); + exit(EXIT_FAILURE); +} + /** * \brief Get an output module by name. * diff --git a/src/output.h b/src/output.h index c4c843256f..78b16df8f0 100644 --- a/src/output.h +++ b/src/output.h @@ -33,6 +33,7 @@ #include "output-packet.h" #include "output-tx.h" #include "output-file.h" +#include "output-filedata.h" typedef struct OutputModule_ { char *name; @@ -43,6 +44,7 @@ typedef struct OutputModule_ { PacketLogCondition PacketConditionFunc; TxLogger TxLogFunc; FileLogger FileLogFunc; + FiledataLogger FiledataLogFunc; uint16_t alproto; TAILQ_ENTRY(OutputModule_) entries; @@ -58,6 +60,8 @@ void OutputRegisterTxModule(char *name, char *conf_name, TxLogger TxLogFunc); void OutputRegisterFileModule(char *name, char *conf_name, OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc); +void OutputRegisterFiledataModule(char *name, char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), FiledataLogger FiledataLogFunc); OutputModule *OutputGetModuleByConfName(char *name); void OutputDeregisterAll(void); diff --git a/src/runmodes.c b/src/runmodes.c index 766c0320c6..e478943568 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -426,6 +426,7 @@ void RunModeInitializeOutputs(void) TmModule *pkt_logger_module = NULL; TmModule *tx_logger_module = NULL; TmModule *file_logger_module = NULL; + TmModule *filedata_logger_module = NULL; const char *enabled; TAILQ_FOREACH(output, &outputs->head, next) { @@ -552,6 +553,27 @@ void RunModeInitializeOutputs(void) TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); SCLogDebug("__file_logger__ added"); } + } else if (module->FiledataLogFunc) { + SCLogDebug("%s is a filedata logger", module->name); + OutputRegisterFiledataLogger(module->name, module->FiledataLogFunc, output_ctx); + + /* need one instance of the tx logger module */ + if (filedata_logger_module == NULL) { + filedata_logger_module = TmModuleGetByName("__filedata_logger__"); + if (filedata_logger_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for __filedata_logger__ failed"); + exit(EXIT_FAILURE); + } + + RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); + if (unlikely(runmode_output == NULL)) + return; + runmode_output->tm_module = filedata_logger_module; + runmode_output->output_ctx = NULL; + TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); + SCLogDebug("__filedata_logger__ added"); + } } else { SCLogDebug("%s is a regular logger", module->name); diff --git a/src/suricata.c b/src/suricata.c index 230a1190bc..bca1e201ba 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -154,6 +154,7 @@ #include "output-packet.h" #include "output-tx.h" #include "output-file.h" +#include "output-filedata.h" #include "util-privs.h" @@ -803,6 +804,7 @@ void RegisterAllModules() TmModulePacketLoggerRegister(); TmModuleTxLoggerRegister(); TmModuleFileLoggerRegister(); + TmModuleFiledataLoggerRegister(); TmModuleDebugList(); } diff --git a/src/tm-modules.c b/src/tm-modules.c index acc68cfe3c..d0a8a7f400 100644 --- a/src/tm-modules.c +++ b/src/tm-modules.c @@ -276,6 +276,7 @@ const char * TmModuleTmmIdToString(TmmId id) CASE_CODE (TMM_PACKETLOGGER); CASE_CODE (TMM_TXLOGGER); CASE_CODE (TMM_FILELOGGER); + CASE_CODE (TMM_FILEDATALOGGER); default: return "UNKNOWN"; diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h index 6ee213649d..ed595609a3 100644 --- a/src/tm-threads-common.h +++ b/src/tm-threads-common.h @@ -82,6 +82,7 @@ typedef enum { TMM_PACKETLOGGER, TMM_TXLOGGER, TMM_FILELOGGER, + TMM_FILEDATALOGGER, TMM_SIZE, } TmmId;