From f830cb80268962300312826ee7bca3d34bb506ba Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Wed, 29 Jan 2014 17:38:04 +0100 Subject: [PATCH] output: introduce concept of sub-modules To support the 'eve-log' idea, we need to be able to force all log modules to be enabled by the master eve-log module, and need to be able to make all logs go into a single file. This didn't fit the API so far, so added the sub-module concept. A sub-module is a regular module, that registers itself as a sub- module of another module: OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http", OutputHttpLogInitSub, ALPROTO_HTTP, JsonHttpLogger); The first argument is the name of the parent. The 4th argument is the OutputCtx init function. It differs slightly from the non-sub one. The different is that in addition to it's ConfNode, it gets the OutputCtx from the parent. This way it can set the parents LogFileCtx in it's own OutputCtx. The runmode setup code will take care of all the extra setup. It's possible to register a module both as a normal module and as a sub- module, which can operate at the same time. Only the TxLogger API is handled in this patch, the rest will be updated later. --- src/output-dnslog.c | 31 +++++ src/output-httplog.c | 35 ++++++ src/output.c | 35 ++++++ src/output.h | 5 + src/runmodes.c | 262 ++++++++++++++++++++++++++----------------- 5 files changed, 263 insertions(+), 105 deletions(-) diff --git a/src/output-dnslog.c b/src/output-dnslog.c index b9bab57050..677edaba6f 100644 --- a/src/output-dnslog.c +++ b/src/output-dnslog.c @@ -299,6 +299,35 @@ static void LogDnsLogDeInitCtx(OutputCtx *output_ctx) SCFree(output_ctx); } +static OutputCtx *JsonDnsLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AlertJsonThread *ajt = parent_ctx->data; + + LogDnsFileCtx *dnslog_ctx = SCMalloc(sizeof(LogDnsFileCtx)); + if (unlikely(dnslog_ctx == NULL)) { + return NULL; + } + memset(dnslog_ctx, 0x00, sizeof(LogDnsFileCtx)); + + dnslog_ctx->file_ctx = ajt->file_ctx; + + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); + if (unlikely(output_ctx == NULL)) { + SCFree(dnslog_ctx); + return NULL; + } + + output_ctx->data = dnslog_ctx; + output_ctx->DeInit = LogDnsLogDeInitCtx; + + SCLogDebug("DNS log sub-module initialized"); + + AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_DNS); + AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_DNS); + + return output_ctx; +} + #define DEFAULT_LOG_FILENAME "dns.json" /** \brief Create a new dns log LogFileCtx. * \param conf Pointer to ConfNode containing this loggers configuration. @@ -356,6 +385,8 @@ void TmModuleJsonDnsLogRegister (void) { OutputRegisterTxModule(MODULE_NAME, "dns-json-log", JsonDnsLogInitCtx, ALPROTO_DNS, JsonDnsLogger); + OutputRegisterTxSubModule("eve-log", MODULE_NAME, "dns", JsonDnsLogInitCtxSub, + ALPROTO_DNS, JsonDnsLogger); } #else diff --git a/src/output-httplog.c b/src/output-httplog.c index 607052ba3a..492eda53f9 100644 --- a/src/output-httplog.c +++ b/src/output-httplog.c @@ -274,6 +274,36 @@ OutputCtx *OutputHttpLogInit(ConfNode *conf) return output_ctx; } +OutputCtx *OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) +{ + AlertJsonThread *ajt = parent_ctx->data; + + LogHttpFileCtx *http_ctx = SCMalloc(sizeof(LogHttpFileCtx)); + if (unlikely(http_ctx == NULL)) + return NULL; + + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); + if (unlikely(output_ctx == NULL)) + return NULL; + + http_ctx->file_ctx = ajt->file_ctx; + http_ctx->flags = LOG_HTTP_DEFAULT; + + if (conf) { + const char *extended = ConfNodeLookupChildValue(conf, "extended"); + + if (extended != NULL) { + if (ConfValIsTrue(extended)) { + http_ctx->flags = LOG_HTTP_EXTENDED; + } + } + } + output_ctx->data = http_ctx; + output_ctx->DeInit = NULL; + + return output_ctx; +} + #define OUTPUT_BUFFER_SIZE 65535 static TmEcode JsonHttpLogThreadInit(ThreadVars *t, void *initdata, void **data) { @@ -324,8 +354,13 @@ void TmModuleJsonHttpLogRegister (void) { tmm_modules[TMM_JSONHTTPLOG].RegisterTests = NULL; tmm_modules[TMM_JSONHTTPLOG].cap_flags = 0; + /* register as separate module */ OutputRegisterTxModule("JsonHttpLog", "http-json-log", OutputHttpLogInit, ALPROTO_HTTP, JsonHttpLogger); + + /* also register as child of eve-log */ + OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http", OutputHttpLogInitSub, + ALPROTO_HTTP, JsonHttpLogger); } #else diff --git a/src/output.c b/src/output.c index 286310ac92..b56eb316e1 100644 --- a/src/output.c +++ b/src/output.c @@ -148,6 +148,41 @@ error: exit(EXIT_FAILURE); } +void +OutputRegisterTxSubModule(const char *parent_name, char *name, char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), uint16_t alproto, + TxLogger TxLogFunc) +{ + if (unlikely(TxLogFunc == NULL)) { + goto error; + } + + OutputModule *module = SCCalloc(1, sizeof(*module)); + if (unlikely(module == NULL)) { + goto error; + } + + module->name = SCStrdup(name); + if (unlikely(module->name == NULL)) + goto error; + module->conf_name = SCStrdup(conf_name); + if (unlikely(module->conf_name == NULL)) + goto error; + module->parent_name = SCStrdup(parent_name); + if (unlikely(module->parent_name == NULL)) + goto error; + module->InitSubFunc = InitFunc; + module->TxLogFunc = TxLogFunc; + module->alproto = alproto; + TAILQ_INSERT_TAIL(&output_modules, module, entries); + + SCLogDebug("Tx logger \"%s\" registered.", name); + return; +error: + SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting..."); + exit(EXIT_FAILURE); +} + /** * \brief Register a file output module. * diff --git a/src/output.h b/src/output.h index 78b16df8f0..6d668238a9 100644 --- a/src/output.h +++ b/src/output.h @@ -38,7 +38,9 @@ typedef struct OutputModule_ { char *name; char *conf_name; + char *parent_name; OutputCtx *(*InitFunc)(ConfNode *); + OutputCtx *(*InitSubFunc)(ConfNode *, OutputCtx *parent_ctx); PacketLogger PacketLogFunc; PacketLogCondition PacketConditionFunc; @@ -58,6 +60,9 @@ void OutputRegisterPacketModule(char *name, char *conf_name, void OutputRegisterTxModule(char *name, char *conf_name, OutputCtx *(*InitFunc)(ConfNode *), uint16_t alproto, TxLogger TxLogFunc); +void OutputRegisterTxSubModule(const char *parent_name, char *name, char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), uint16_t alproto, + TxLogger TxLogFunc); void OutputRegisterFileModule(char *name, char *conf_name, OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc); void OutputRegisterFiledataModule(char *name, char *conf_name, diff --git a/src/runmodes.c b/src/runmodes.c index e478943568..2916deff7b 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -410,6 +410,121 @@ void RunModeShutDown(void) } } +static TmModule *pkt_logger_module = NULL; +static TmModule *tx_logger_module = NULL; +static TmModule *file_logger_module = NULL; +static TmModule *filedata_logger_module = NULL; + +/** \brief Turn output into thread module */ +static void SetupOutput(const char *name, OutputModule *module, OutputCtx *output_ctx) +{ + TmModule *tm_module = TmModuleGetByName(module->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", module->name); + exit(EXIT_FAILURE); + } + if (strcmp(tmm_modules[TMM_ALERTDEBUGLOG].name, tm_module->name) == 0) + debuglog_enabled = 1; + + if (module->PacketLogFunc) { + SCLogDebug("%s is a packet logger", module->name); + OutputRegisterPacketLogger(module->name, module->PacketLogFunc, + module->PacketConditionFunc, output_ctx); + + /* need one instance of the packet logger module */ + if (pkt_logger_module == NULL) { + pkt_logger_module = TmModuleGetByName("__packet_logger__"); + if (pkt_logger_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for __packet_logger__ failed"); + exit(EXIT_FAILURE); + } + + RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); + if (unlikely(runmode_output == NULL)) + return; + runmode_output->tm_module = pkt_logger_module; + runmode_output->output_ctx = NULL; + TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); + SCLogDebug("__packet_logger__ added"); + } + } else if (module->TxLogFunc) { + SCLogDebug("%s is a tx logger", module->name); + OutputRegisterTxLogger(module->name, module->alproto, + module->TxLogFunc, output_ctx); + + /* need one instance of the tx logger module */ + if (tx_logger_module == NULL) { + tx_logger_module = TmModuleGetByName("__tx_logger__"); + if (tx_logger_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for __tx_logger__ failed"); + exit(EXIT_FAILURE); + } + + RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); + if (unlikely(runmode_output == NULL)) + return; + runmode_output->tm_module = tx_logger_module; + runmode_output->output_ctx = NULL; + TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); + SCLogDebug("__tx_logger__ added"); + } + } else if (module->FileLogFunc) { + SCLogDebug("%s is a file logger", module->name); + OutputRegisterFileLogger(module->name, module->FileLogFunc, output_ctx); + + /* need one instance of the tx logger module */ + if (file_logger_module == NULL) { + file_logger_module = TmModuleGetByName("__file_logger__"); + if (file_logger_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for __file_logger__ failed"); + exit(EXIT_FAILURE); + } + + RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); + if (unlikely(runmode_output == NULL)) + return; + runmode_output->tm_module = file_logger_module; + runmode_output->output_ctx = NULL; + TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); + SCLogDebug("__file_logger__ added"); + } + } else if (module->FiledataLogFunc) { + SCLogDebug("%s is a filedata logger", module->name); + OutputRegisterFiledataLogger(module->name, module->FiledataLogFunc, output_ctx); + + /* need one instance of the tx logger module */ + if (filedata_logger_module == NULL) { + filedata_logger_module = TmModuleGetByName("__filedata_logger__"); + if (filedata_logger_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for __filedata_logger__ failed"); + exit(EXIT_FAILURE); + } + + RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); + if (unlikely(runmode_output == NULL)) + return; + runmode_output->tm_module = filedata_logger_module; + runmode_output->output_ctx = NULL; + TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); + SCLogDebug("__filedata_logger__ added"); + } + } else { + SCLogDebug("%s is a regular logger", module->name); + + RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); + if (unlikely(runmode_output == NULL)) + return; + runmode_output->tm_module = tm_module; + runmode_output->output_ctx = output_ctx; + TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); + } +} + /** * Initialize the output modules. */ @@ -422,11 +537,6 @@ void RunModeInitializeOutputs(void) } ConfNode *output, *output_config; - TmModule *tm_module; - TmModule *pkt_logger_module = NULL; - TmModule *tx_logger_module = NULL; - TmModule *file_logger_module = NULL; - TmModule *filedata_logger_module = NULL; const char *enabled; TAILQ_FOREACH(output, &outputs->head, next) { @@ -470,6 +580,7 @@ void RunModeInitializeOutputs(void) "No output module named %s, ignoring", output->val); continue; } + OutputCtx *output_ctx = NULL; if (module->InitFunc != NULL) { output_ctx = module->InitFunc(output_config); @@ -478,111 +589,52 @@ void RunModeInitializeOutputs(void) * error. Maybe we should exit on init errors? */ continue; } + } else if (module->InitSubFunc != NULL) { + SCLogInfo("skipping submodule"); + continue; } - tm_module = TmModuleGetByName(module->name); - if (tm_module == NULL) { - SCLogError(SC_ERR_INVALID_ARGUMENT, - "TmModuleGetByName for %s failed", module->name); - exit(EXIT_FAILURE); - } - if (strcmp(tmm_modules[TMM_ALERTDEBUGLOG].name, tm_module->name) == 0) - debuglog_enabled = 1; - - if (module->PacketLogFunc) { - SCLogDebug("%s is a packet logger", module->name); - OutputRegisterPacketLogger(module->name, module->PacketLogFunc, - module->PacketConditionFunc, output_ctx); - - /* need one instance of the packet logger module */ - if (pkt_logger_module == NULL) { - pkt_logger_module = TmModuleGetByName("__packet_logger__"); - if (pkt_logger_module == NULL) { - SCLogError(SC_ERR_INVALID_ARGUMENT, - "TmModuleGetByName for __packet_logger__ failed"); - exit(EXIT_FAILURE); - } - RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); - if (unlikely(runmode_output == NULL)) - return; - runmode_output->tm_module = pkt_logger_module; - runmode_output->output_ctx = NULL; - TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); - SCLogDebug("__packet_logger__ added"); - } - } else if (module->TxLogFunc) { - SCLogDebug("%s is a tx logger", module->name); - OutputRegisterTxLogger(module->name, module->alproto, - module->TxLogFunc, output_ctx); - - /* need one instance of the tx logger module */ - if (tx_logger_module == NULL) { - tx_logger_module = TmModuleGetByName("__tx_logger__"); - if (tx_logger_module == NULL) { - SCLogError(SC_ERR_INVALID_ARGUMENT, - "TmModuleGetByName for __tx_logger__ failed"); - exit(EXIT_FAILURE); - } - - RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); - if (unlikely(runmode_output == NULL)) - return; - runmode_output->tm_module = tx_logger_module; - runmode_output->output_ctx = NULL; - TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); - SCLogDebug("__tx_logger__ added"); - } - } else if (module->FileLogFunc) { - SCLogDebug("%s is a file logger", module->name); - OutputRegisterFileLogger(module->name, module->FileLogFunc, output_ctx); - - /* need one instance of the tx logger module */ - if (file_logger_module == NULL) { - file_logger_module = TmModuleGetByName("__file_logger__"); - if (file_logger_module == NULL) { - SCLogError(SC_ERR_INVALID_ARGUMENT, - "TmModuleGetByName for __file_logger__ failed"); - exit(EXIT_FAILURE); + // TODO if module == parent, find it's children + if (strcmp(output->val, "eve-log") == 0) { + ConfNode *types = ConfNodeLookupChild(output_config, "types"); + SCLogInfo("types %p", types); + if (types != NULL) { + ConfNode *type = NULL; + TAILQ_FOREACH(type, &types->head, next) { + SCLogInfo("type %s", type->val); + + OutputModule *sub_module = OutputGetModuleByConfName(type->val); + if (sub_module == NULL) { + SCLogWarning(SC_ERR_INVALID_ARGUMENT, + "No output module named %s, ignoring", type->val); + continue; + } + if (sub_module->parent_name == NULL || + strcmp(sub_module->parent_name,output->val) != 0) { + SCLogWarning(SC_ERR_INVALID_ARGUMENT, + "bad parent for %s, ignoring", type->val); + continue; + } + if (sub_module->InitSubFunc == NULL) { + SCLogWarning(SC_ERR_INVALID_ARGUMENT, + "bad sub-module for %s, ignoring", type->val); + continue; + } + ConfNode *sub_output_config = ConfNodeLookupChild(type, type->val); + // sub_output_config may be NULL if no config + + /* pass on parent output_ctx */ + OutputCtx *sub_output_ctx = + sub_module->InitSubFunc(sub_output_config, output_ctx); + if (sub_output_ctx == NULL) { + continue; + } + + SetupOutput(sub_module->name, sub_module, sub_output_ctx); } - - RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); - if (unlikely(runmode_output == NULL)) - return; - runmode_output->tm_module = file_logger_module; - runmode_output->output_ctx = NULL; - TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); - SCLogDebug("__file_logger__ added"); - } - } else if (module->FiledataLogFunc) { - SCLogDebug("%s is a filedata logger", module->name); - OutputRegisterFiledataLogger(module->name, module->FiledataLogFunc, output_ctx); - - /* need one instance of the tx logger module */ - if (filedata_logger_module == NULL) { - filedata_logger_module = TmModuleGetByName("__filedata_logger__"); - if (filedata_logger_module == NULL) { - SCLogError(SC_ERR_INVALID_ARGUMENT, - "TmModuleGetByName for __filedata_logger__ failed"); - exit(EXIT_FAILURE); - } - - RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); - if (unlikely(runmode_output == NULL)) - return; - runmode_output->tm_module = filedata_logger_module; - runmode_output->output_ctx = NULL; - TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); - SCLogDebug("__filedata_logger__ added"); } } else { - SCLogDebug("%s is a regular logger", module->name); - - RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput)); - if (unlikely(runmode_output == NULL)) - return; - runmode_output->tm_module = tm_module; - runmode_output->output_ctx = output_ctx; - TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries); + SetupOutput(module->name, module, output_ctx); } } }