output: introduce concept of sub-modules

To support the 'eve-log' idea, we need to be able to force all log
modules to be enabled by the master eve-log module, and need to be
able to make all logs go into a single file. This didn't fit the
API so far, so added the sub-module concept.

A sub-module is a regular module, that registers itself as a sub-
module of another module:

    OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http",
            OutputHttpLogInitSub, ALPROTO_HTTP, JsonHttpLogger);

The first argument is the name of the parent. The 4th argument is
the OutputCtx init function. It differs slightly from the non-sub
one. The different is that in addition to it's ConfNode, it gets
the OutputCtx from the parent. This way it can set the parents
LogFileCtx in it's own OutputCtx.

The runmode setup code will take care of all the extra setup. It's
possible to register a module both as a normal module and as a sub-
module, which can operate at the same time.

Only the TxLogger API is handled in this patch, the rest will be
updated later.

src/output-dnslog.c

@@ -299,6 +299,35 @@ static void LogDnsLogDeInitCtx(OutputCtx *output_ctx)
     SCFree(output_ctx);
 }
 
+static OutputCtx *JsonDnsLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
+{
+    AlertJsonThread *ajt = parent_ctx->data;
+
+    LogDnsFileCtx *dnslog_ctx = SCMalloc(sizeof(LogDnsFileCtx));
+    if (unlikely(dnslog_ctx == NULL)) {
+        return NULL;
+    }
+    memset(dnslog_ctx, 0x00, sizeof(LogDnsFileCtx));
+
+    dnslog_ctx->file_ctx = ajt->file_ctx;
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL)) {
+        SCFree(dnslog_ctx);
+        return NULL;
+    }
+
+    output_ctx->data = dnslog_ctx;
+    output_ctx->DeInit = LogDnsLogDeInitCtx;
+
+    SCLogDebug("DNS log sub-module initialized");
+
+    AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_DNS);
+    AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_DNS);
+
+    return output_ctx;
+}
+
 #define DEFAULT_LOG_FILENAME "dns.json"
 /** \brief Create a new dns log LogFileCtx.
  *  \param conf Pointer to ConfNode containing this loggers configuration.
@@ -356,6 +385,8 @@ void TmModuleJsonDnsLogRegister (void) {
 
     OutputRegisterTxModule(MODULE_NAME, "dns-json-log", JsonDnsLogInitCtx,
             ALPROTO_DNS, JsonDnsLogger);
+    OutputRegisterTxSubModule("eve-log", MODULE_NAME, "dns", JsonDnsLogInitCtxSub,
+            ALPROTO_DNS, JsonDnsLogger);
 }
 
 #else

src/output-httplog.c

@@ -274,6 +274,36 @@ OutputCtx *OutputHttpLogInit(ConfNode *conf)
     return output_ctx;
 }
 
+OutputCtx *OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
+{
+    AlertJsonThread *ajt = parent_ctx->data;
+
+    LogHttpFileCtx *http_ctx = SCMalloc(sizeof(LogHttpFileCtx));
+    if (unlikely(http_ctx == NULL))
+        return NULL;
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL))
+        return NULL;
+
+    http_ctx->file_ctx = ajt->file_ctx;
+    http_ctx->flags = LOG_HTTP_DEFAULT;
+
+    if (conf) {
+        const char *extended = ConfNodeLookupChildValue(conf, "extended");
+
+        if (extended != NULL) {
+            if (ConfValIsTrue(extended)) {
+                http_ctx->flags = LOG_HTTP_EXTENDED;
+            }
+        }
+    }
+    output_ctx->data = http_ctx;
+    output_ctx->DeInit = NULL;
+
+    return output_ctx;
+}
+
 #define OUTPUT_BUFFER_SIZE 65535
 static TmEcode JsonHttpLogThreadInit(ThreadVars *t, void *initdata, void **data)
 {
@@ -324,8 +354,13 @@ void TmModuleJsonHttpLogRegister (void) {
     tmm_modules[TMM_JSONHTTPLOG].RegisterTests = NULL;
     tmm_modules[TMM_JSONHTTPLOG].cap_flags = 0;
 
+    /* register as separate module */
     OutputRegisterTxModule("JsonHttpLog", "http-json-log", OutputHttpLogInit,
             ALPROTO_HTTP, JsonHttpLogger);
+
+    /* also register as child of eve-log */
+    OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http", OutputHttpLogInitSub,
+            ALPROTO_HTTP, JsonHttpLogger);
 }
 
 #else

src/output.c

@@ -148,6 +148,41 @@ error:
     exit(EXIT_FAILURE);
 }
 
+void
+OutputRegisterTxSubModule(const char *parent_name, char *name, char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), uint16_t alproto,
+    TxLogger TxLogFunc)
+{
+    if (unlikely(TxLogFunc == NULL)) {
+        goto error;
+    }
+
+    OutputModule *module = SCCalloc(1, sizeof(*module));
+    if (unlikely(module == NULL)) {
+        goto error;
+    }
+
+    module->name = SCStrdup(name);
+    if (unlikely(module->name == NULL))
+        goto error;
+    module->conf_name = SCStrdup(conf_name);
+    if (unlikely(module->conf_name == NULL))
+        goto error;
+    module->parent_name = SCStrdup(parent_name);
+    if (unlikely(module->parent_name == NULL))
+        goto error;
+    module->InitSubFunc = InitFunc;
+    module->TxLogFunc = TxLogFunc;
+    module->alproto = alproto;
+    TAILQ_INSERT_TAIL(&output_modules, module, entries);
+
+    SCLogDebug("Tx logger \"%s\" registered.", name);
+    return;
+error:
+    SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting...");
+    exit(EXIT_FAILURE);
+}
+
 /**
  * \brief Register a file output module.
  *

src/output.h

@@ -38,7 +38,9 @@
 typedef struct OutputModule_ {
     char *name;
     char *conf_name;
+    char *parent_name;
     OutputCtx *(*InitFunc)(ConfNode *);
+    OutputCtx *(*InitSubFunc)(ConfNode *, OutputCtx *parent_ctx);
 
     PacketLogger PacketLogFunc;
     PacketLogCondition PacketConditionFunc;
@@ -58,6 +60,9 @@ void OutputRegisterPacketModule(char *name, char *conf_name,
 void OutputRegisterTxModule(char *name, char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *), uint16_t alproto,
     TxLogger TxLogFunc);
+void OutputRegisterTxSubModule(const char *parent_name, char *name, char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), uint16_t alproto,
+    TxLogger TxLogFunc);
 void OutputRegisterFileModule(char *name, char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc);
 void OutputRegisterFiledataModule(char *name, char *conf_name,

src/runmodes.c

@@ -410,76 +410,15 @@ void RunModeShutDown(void)
     }
 }
 
-/**
+static TmModule *pkt_logger_module = NULL;
- * Initialize the output modules.
+static TmModule *tx_logger_module = NULL;
- */
+static TmModule *file_logger_module = NULL;
-void RunModeInitializeOutputs(void)
+static TmModule *filedata_logger_module = NULL;
-{
-    ConfNode *outputs = ConfGetNode("outputs");
-    if (outputs == NULL) {
-        /* No "outputs" section in the configuration. */
-        return;
-    }
-
-    ConfNode *output, *output_config;
-    TmModule *tm_module;
-    TmModule *pkt_logger_module = NULL;
-    TmModule *tx_logger_module = NULL;
-    TmModule *file_logger_module = NULL;
-    TmModule *filedata_logger_module = NULL;
-    const char *enabled;
-
-    TAILQ_FOREACH(output, &outputs->head, next) {
-
-        if (strcmp(output->val, "stats") == 0)
-            continue;
-
-        output_config = ConfNodeLookupChild(output, output->val);
-        if (output_config == NULL) {
-            /* Shouldn't happen. */
-            SCLogError(SC_ERR_INVALID_ARGUMENT,
-                "Failed to lookup configuration child node: fast");
-            exit(1);
-        }
-
-        enabled = ConfNodeLookupChildValue(output_config, "enabled");
-        if (enabled == NULL || !ConfValIsTrue(enabled)) {
-            continue;
-        }
-
-        if (strncmp(output->val, "unified-", sizeof("unified-") - 1) == 0) {
-            SCLogWarning(SC_ERR_INVALID_ARGUMENT,
-                    "Unified1 is no longer supported,"
-                    " use Unified2 instead "
-                    "(see https://redmine.openinfosecfoundation.org/issues/353"
-                    " for an explanation)");
-            continue;
-        } else if (strcmp(output->val, "alert-prelude") == 0) {
-#ifndef PRELUDE
-            SCLogWarning(SC_ERR_INVALID_ARGUMENT,
-                    "Prelude support not compiled in. Reconfigure/"
-                    "recompile with --enable-prelude to add Prelude "
-                    "support.");
-            continue;
-#endif
-        }
 
-        OutputModule *module = OutputGetModuleByConfName(output->val);
+/** \brief Turn output into thread module */
-        if (module == NULL) {
+static void SetupOutput(const char *name, OutputModule *module, OutputCtx *output_ctx)
-            SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+{
-                "No output module named %s, ignoring", output->val);
+    TmModule *tm_module = TmModuleGetByName(module->name);
-            continue;
-        }
-        OutputCtx *output_ctx = NULL;
-        if (module->InitFunc != NULL) {
-            output_ctx = module->InitFunc(output_config);
-            if (output_ctx == NULL) {
-                /* In most cases the init function will have logged the
-                 * error. Maybe we should exit on init errors? */
-                continue;
-            }
-        }
-        tm_module = TmModuleGetByName(module->name);
     if (tm_module == NULL) {
         SCLogError(SC_ERR_INVALID_ARGUMENT,
                 "TmModuleGetByName for %s failed", module->name);
@@ -585,6 +524,119 @@ void RunModeInitializeOutputs(void)
         TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
     }
 }
+
+/**
+ * Initialize the output modules.
+ */
+void RunModeInitializeOutputs(void)
+{
+    ConfNode *outputs = ConfGetNode("outputs");
+    if (outputs == NULL) {
+        /* No "outputs" section in the configuration. */
+        return;
+    }
+
+    ConfNode *output, *output_config;
+    const char *enabled;
+
+    TAILQ_FOREACH(output, &outputs->head, next) {
+
+        if (strcmp(output->val, "stats") == 0)
+            continue;
+
+        output_config = ConfNodeLookupChild(output, output->val);
+        if (output_config == NULL) {
+            /* Shouldn't happen. */
+            SCLogError(SC_ERR_INVALID_ARGUMENT,
+                "Failed to lookup configuration child node: fast");
+            exit(1);
+        }
+
+        enabled = ConfNodeLookupChildValue(output_config, "enabled");
+        if (enabled == NULL || !ConfValIsTrue(enabled)) {
+            continue;
+        }
+
+        if (strncmp(output->val, "unified-", sizeof("unified-") - 1) == 0) {
+            SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                    "Unified1 is no longer supported,"
+                    " use Unified2 instead "
+                    "(see https://redmine.openinfosecfoundation.org/issues/353"
+                    " for an explanation)");
+            continue;
+        } else if (strcmp(output->val, "alert-prelude") == 0) {
+#ifndef PRELUDE
+            SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                    "Prelude support not compiled in. Reconfigure/"
+                    "recompile with --enable-prelude to add Prelude "
+                    "support.");
+            continue;
+#endif
+        }
+
+        OutputModule *module = OutputGetModuleByConfName(output->val);
+        if (module == NULL) {
+            SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                "No output module named %s, ignoring", output->val);
+            continue;
+        }
+
+        OutputCtx *output_ctx = NULL;
+        if (module->InitFunc != NULL) {
+            output_ctx = module->InitFunc(output_config);
+            if (output_ctx == NULL) {
+                /* In most cases the init function will have logged the
+                 * error. Maybe we should exit on init errors? */
+                continue;
+            }
+        } else if (module->InitSubFunc != NULL) {
+            SCLogInfo("skipping submodule");
+            continue;
+        }
+
+        // TODO if module == parent, find it's children
+        if (strcmp(output->val, "eve-log") == 0) {
+            ConfNode *types = ConfNodeLookupChild(output_config, "types");
+            SCLogInfo("types %p", types);
+            if (types != NULL) {
+                ConfNode *type = NULL;
+                TAILQ_FOREACH(type, &types->head, next) {
+                    SCLogInfo("type %s", type->val);
+
+                    OutputModule *sub_module = OutputGetModuleByConfName(type->val);
+                    if (sub_module == NULL) {
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                                "No output module named %s, ignoring", type->val);
+                        continue;
+                    }
+                    if (sub_module->parent_name == NULL ||
+                            strcmp(sub_module->parent_name,output->val) != 0) {
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                                "bad parent for %s, ignoring", type->val);
+                        continue;
+                    }
+                    if (sub_module->InitSubFunc == NULL) {
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
+                                "bad sub-module for %s, ignoring", type->val);
+                        continue;
+                    }
+                    ConfNode *sub_output_config = ConfNodeLookupChild(type, type->val);
+                    // sub_output_config may be NULL if no config
+
+                    /* pass on parent output_ctx */
+                    OutputCtx *sub_output_ctx =
+                        sub_module->InitSubFunc(sub_output_config, output_ctx);
+                    if (sub_output_ctx == NULL) {
+                        continue;
+                    }
+
+                    SetupOutput(sub_module->name, sub_module, sub_output_ctx);
+                }
+            }
+        } else {
+            SetupOutput(module->name, module, output_ctx);
+        }
+    }
 }
 
 /**