output: introduce concept of sub-modules

To support the 'eve-log' idea, we need to be able to force all log
modules to be enabled by the master eve-log module, and need to be
able to make all logs go into a single file. This didn't fit the
API so far, so added the sub-module concept.

A sub-module is a regular module, that registers itself as a sub-
module of another module:

    OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http",
            OutputHttpLogInitSub, ALPROTO_HTTP, JsonHttpLogger);

The first argument is the name of the parent. The 4th argument is
the OutputCtx init function. It differs slightly from the non-sub
one. The different is that in addition to it's ConfNode, it gets
the OutputCtx from the parent. This way it can set the parents
LogFileCtx in it's own OutputCtx.

The runmode setup code will take care of all the extra setup. It's
possible to register a module both as a normal module and as a sub-
module, which can operate at the same time.

Only the TxLogger API is handled in this patch, the rest will be
updated later.

src/output-dnslog.c

@@ -299,6 +299,35 @@ static void LogDnsLogDeInitCtx(OutputCtx *output_ctx)
     SCFree(output_ctx);
 }
 
+static OutputCtx *JsonDnsLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
+{
+    AlertJsonThread *ajt = parent_ctx->data;
+
+    LogDnsFileCtx *dnslog_ctx = SCMalloc(sizeof(LogDnsFileCtx));
+    if (unlikely(dnslog_ctx == NULL)) {
+        return NULL;
+    }
+    memset(dnslog_ctx, 0x00, sizeof(LogDnsFileCtx));
+
+    dnslog_ctx->file_ctx = ajt->file_ctx;
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL)) {
+        SCFree(dnslog_ctx);
+        return NULL;
+    }
+
+    output_ctx->data = dnslog_ctx;
+    output_ctx->DeInit = LogDnsLogDeInitCtx;
+
+    SCLogDebug("DNS log sub-module initialized");
+
+    AppLayerParserRegisterLogger(IPPROTO_UDP, ALPROTO_DNS);
+    AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_DNS);
+
+    return output_ctx;
+}
+
 #define DEFAULT_LOG_FILENAME "dns.json"
 /** \brief Create a new dns log LogFileCtx.
  *  \param conf Pointer to ConfNode containing this loggers configuration.
@@ -356,6 +385,8 @@ void TmModuleJsonDnsLogRegister (void) {
 
     OutputRegisterTxModule(MODULE_NAME, "dns-json-log", JsonDnsLogInitCtx,
             ALPROTO_DNS, JsonDnsLogger);
+    OutputRegisterTxSubModule("eve-log", MODULE_NAME, "dns", JsonDnsLogInitCtxSub,
+            ALPROTO_DNS, JsonDnsLogger);
 }
 
 #else

src/output-httplog.c

@@ -274,6 +274,36 @@ OutputCtx *OutputHttpLogInit(ConfNode *conf)
     return output_ctx;
 }
 
+OutputCtx *OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx)
+{
+    AlertJsonThread *ajt = parent_ctx->data;
+
+    LogHttpFileCtx *http_ctx = SCMalloc(sizeof(LogHttpFileCtx));
+    if (unlikely(http_ctx == NULL))
+        return NULL;
+
+    OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
+    if (unlikely(output_ctx == NULL))
+        return NULL;
+
+    http_ctx->file_ctx = ajt->file_ctx;
+    http_ctx->flags = LOG_HTTP_DEFAULT;
+
+    if (conf) {
+        const char *extended = ConfNodeLookupChildValue(conf, "extended");
+
+        if (extended != NULL) {
+            if (ConfValIsTrue(extended)) {
+                http_ctx->flags = LOG_HTTP_EXTENDED;
+            }
+        }
+    }
+    output_ctx->data = http_ctx;
+    output_ctx->DeInit = NULL;
+
+    return output_ctx;
+}
+
 #define OUTPUT_BUFFER_SIZE 65535
 static TmEcode JsonHttpLogThreadInit(ThreadVars *t, void *initdata, void **data)
 {
@@ -324,8 +354,13 @@ void TmModuleJsonHttpLogRegister (void) {
     tmm_modules[TMM_JSONHTTPLOG].RegisterTests = NULL;
     tmm_modules[TMM_JSONHTTPLOG].cap_flags = 0;
 
+    /* register as separate module */
     OutputRegisterTxModule("JsonHttpLog", "http-json-log", OutputHttpLogInit,
             ALPROTO_HTTP, JsonHttpLogger);
+
+    /* also register as child of eve-log */
+    OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http", OutputHttpLogInitSub,
+            ALPROTO_HTTP, JsonHttpLogger);
 }
 
 #else

src/output.c

@@ -148,6 +148,41 @@ error:
     exit(EXIT_FAILURE);
 }
 
+void
+OutputRegisterTxSubModule(const char *parent_name, char *name, char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), uint16_t alproto,
+    TxLogger TxLogFunc)
+{
+    if (unlikely(TxLogFunc == NULL)) {
+        goto error;
+    }
+
+    OutputModule *module = SCCalloc(1, sizeof(*module));
+    if (unlikely(module == NULL)) {
+        goto error;
+    }
+
+    module->name = SCStrdup(name);
+    if (unlikely(module->name == NULL))
+        goto error;
+    module->conf_name = SCStrdup(conf_name);
+    if (unlikely(module->conf_name == NULL))
+        goto error;
+    module->parent_name = SCStrdup(parent_name);
+    if (unlikely(module->parent_name == NULL))
+        goto error;
+    module->InitSubFunc = InitFunc;
+    module->TxLogFunc = TxLogFunc;
+    module->alproto = alproto;
+    TAILQ_INSERT_TAIL(&output_modules, module, entries);
+
+    SCLogDebug("Tx logger \"%s\" registered.", name);
+    return;
+error:
+    SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting...");
+    exit(EXIT_FAILURE);
+}
+
 /**
  * \brief Register a file output module.
  *

src/output.h

@@ -38,7 +38,9 @@
 typedef struct OutputModule_ {
     char *name;
     char *conf_name;
+    char *parent_name;
     OutputCtx *(*InitFunc)(ConfNode *);
+    OutputCtx *(*InitSubFunc)(ConfNode *, OutputCtx *parent_ctx);
 
     PacketLogger PacketLogFunc;
     PacketLogCondition PacketConditionFunc;
@@ -58,6 +60,9 @@ void OutputRegisterPacketModule(char *name, char *conf_name,
 void OutputRegisterTxModule(char *name, char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *), uint16_t alproto,
     TxLogger TxLogFunc);
+void OutputRegisterTxSubModule(const char *parent_name, char *name, char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), uint16_t alproto,
+    TxLogger TxLogFunc);
 void OutputRegisterFileModule(char *name, char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc);
 void OutputRegisterFiledataModule(char *name, char *conf_name,

src/runmodes.c

@@ -410,6 +410,121 @@ void RunModeShutDown(void)
     }
 }
 
+static TmModule *pkt_logger_module = NULL;
+static TmModule *tx_logger_module = NULL;
+static TmModule *file_logger_module = NULL;
+static TmModule *filedata_logger_module = NULL;
+
+/** \brief Turn output into thread module */
+static void SetupOutput(const char *name, OutputModule *module, OutputCtx *output_ctx)
+{
+    TmModule *tm_module = TmModuleGetByName(module->name);
+    if (tm_module == NULL) {
+        SCLogError(SC_ERR_INVALID_ARGUMENT,
+                "TmModuleGetByName for %s failed", module->name);
+        exit(EXIT_FAILURE);
+    }
+    if (strcmp(tmm_modules[TMM_ALERTDEBUGLOG].name, tm_module->name) == 0)
+        debuglog_enabled = 1;
+
+    if (module->PacketLogFunc) {
+        SCLogDebug("%s is a packet logger", module->name);
+        OutputRegisterPacketLogger(module->name, module->PacketLogFunc,
+                module->PacketConditionFunc, output_ctx);
+
+        /* need one instance of the packet logger module */
+        if (pkt_logger_module == NULL) {
+            pkt_logger_module = TmModuleGetByName("__packet_logger__");
+            if (pkt_logger_module == NULL) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "TmModuleGetByName for __packet_logger__ failed");
+                exit(EXIT_FAILURE);
+            }
+
+            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+            if (unlikely(runmode_output == NULL))
+                return;
+            runmode_output->tm_module = pkt_logger_module;
+            runmode_output->output_ctx = NULL;
+            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SCLogDebug("__packet_logger__ added");
+        }
+    } else if (module->TxLogFunc) {
+        SCLogDebug("%s is a tx logger", module->name);
+        OutputRegisterTxLogger(module->name, module->alproto,
+                module->TxLogFunc, output_ctx);
+
+        /* need one instance of the tx logger module */
+        if (tx_logger_module == NULL) {
+            tx_logger_module = TmModuleGetByName("__tx_logger__");
+            if (tx_logger_module == NULL) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "TmModuleGetByName for __tx_logger__ failed");
+                exit(EXIT_FAILURE);
+            }
+
+            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+            if (unlikely(runmode_output == NULL))
+                return;
+            runmode_output->tm_module = tx_logger_module;
+            runmode_output->output_ctx = NULL;
+            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SCLogDebug("__tx_logger__ added");
+        }
+    } else if (module->FileLogFunc) {
+        SCLogDebug("%s is a file logger", module->name);
+        OutputRegisterFileLogger(module->name, module->FileLogFunc, output_ctx);
+
+        /* need one instance of the tx logger module */
+        if (file_logger_module == NULL) {
+            file_logger_module = TmModuleGetByName("__file_logger__");
+            if (file_logger_module == NULL) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "TmModuleGetByName for __file_logger__ failed");
+                exit(EXIT_FAILURE);
+            }
+
+            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+            if (unlikely(runmode_output == NULL))
+                return;
+            runmode_output->tm_module = file_logger_module;
+            runmode_output->output_ctx = NULL;
+            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SCLogDebug("__file_logger__ added");
+        }
+    } else if (module->FiledataLogFunc) {
+        SCLogDebug("%s is a filedata logger", module->name);
+        OutputRegisterFiledataLogger(module->name, module->FiledataLogFunc, output_ctx);
+
+        /* need one instance of the tx logger module */
+        if (filedata_logger_module == NULL) {
+            filedata_logger_module = TmModuleGetByName("__filedata_logger__");
+            if (filedata_logger_module == NULL) {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        "TmModuleGetByName for __filedata_logger__ failed");
+                exit(EXIT_FAILURE);
+            }
+
+            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+            if (unlikely(runmode_output == NULL))
+                return;
+            runmode_output->tm_module = filedata_logger_module;
+            runmode_output->output_ctx = NULL;
+            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+            SCLogDebug("__filedata_logger__ added");
+        }
+    } else {
+        SCLogDebug("%s is a regular logger", module->name);
+
+        RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+        if (unlikely(runmode_output == NULL))
+            return;
+        runmode_output->tm_module = tm_module;
+        runmode_output->output_ctx = output_ctx;
+        TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+    }
+}
+
 /**
  * Initialize the output modules.
  */
@@ -422,11 +537,6 @@ void RunModeInitializeOutputs(void)
     }
 
     ConfNode *output, *output_config;
-    TmModule *tm_module;
-    TmModule *pkt_logger_module = NULL;
-    TmModule *tx_logger_module = NULL;
-    TmModule *file_logger_module = NULL;
-    TmModule *filedata_logger_module = NULL;
     const char *enabled;
 
     TAILQ_FOREACH(output, &outputs->head, next) {
@@ -470,6 +580,7 @@ void RunModeInitializeOutputs(void)
                 "No output module named %s, ignoring", output->val);
             continue;
         }
+
         OutputCtx *output_ctx = NULL;
         if (module->InitFunc != NULL) {
             output_ctx = module->InitFunc(output_config);
@@ -478,111 +589,52 @@ void RunModeInitializeOutputs(void)
                  * error. Maybe we should exit on init errors? */
                 continue;
             }
+        } else if (module->InitSubFunc != NULL) {
+            SCLogInfo("skipping submodule");
+            continue;
         }
-        tm_module = TmModuleGetByName(module->name);
-        if (tm_module == NULL) {
-            SCLogError(SC_ERR_INVALID_ARGUMENT,
-                    "TmModuleGetByName for %s failed", module->name);
-            exit(EXIT_FAILURE);
-        }
-        if (strcmp(tmm_modules[TMM_ALERTDEBUGLOG].name, tm_module->name) == 0)
-            debuglog_enabled = 1;
-
-        if (module->PacketLogFunc) {
-            SCLogDebug("%s is a packet logger", module->name);
-            OutputRegisterPacketLogger(module->name, module->PacketLogFunc,
-                    module->PacketConditionFunc, output_ctx);
-
-            /* need one instance of the packet logger module */
-            if (pkt_logger_module == NULL) {
-                pkt_logger_module = TmModuleGetByName("__packet_logger__");
-                if (pkt_logger_module == NULL) {
-                    SCLogError(SC_ERR_INVALID_ARGUMENT,
-                            "TmModuleGetByName for __packet_logger__ failed");
-                    exit(EXIT_FAILURE);
-                }
 
-                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+        // TODO if module == parent, find it's children
-                if (unlikely(runmode_output == NULL))
+        if (strcmp(output->val, "eve-log") == 0) {
-                    return;
+            ConfNode *types = ConfNodeLookupChild(output_config, "types");
-                runmode_output->tm_module = pkt_logger_module;
+            SCLogInfo("types %p", types);
-                runmode_output->output_ctx = NULL;
+            if (types != NULL) {
-                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+                ConfNode *type = NULL;
-                SCLogDebug("__packet_logger__ added");
+                TAILQ_FOREACH(type, &types->head, next) {
-            }
+                    SCLogInfo("type %s", type->val);
-        } else if (module->TxLogFunc) {
+
-            SCLogDebug("%s is a tx logger", module->name);
+                    OutputModule *sub_module = OutputGetModuleByConfName(type->val);
-            OutputRegisterTxLogger(module->name, module->alproto,
+                    if (sub_module == NULL) {
-                    module->TxLogFunc, output_ctx);
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
-
+                                "No output module named %s, ignoring", type->val);
-            /* need one instance of the tx logger module */
+                        continue;
-            if (tx_logger_module == NULL) {
+                    }
-                tx_logger_module = TmModuleGetByName("__tx_logger__");
+                    if (sub_module->parent_name == NULL ||
-                if (tx_logger_module == NULL) {
+                            strcmp(sub_module->parent_name,output->val) != 0) {
-                    SCLogError(SC_ERR_INVALID_ARGUMENT,
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
-                            "TmModuleGetByName for __tx_logger__ failed");
+                                "bad parent for %s, ignoring", type->val);
-                    exit(EXIT_FAILURE);
+                        continue;
-                }
+                    }
-
+                    if (sub_module->InitSubFunc == NULL) {
-                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
+                        SCLogWarning(SC_ERR_INVALID_ARGUMENT,
-                if (unlikely(runmode_output == NULL))
+                                "bad sub-module for %s, ignoring", type->val);
-                    return;
+                        continue;
-                runmode_output->tm_module = tx_logger_module;
+                    }
-                runmode_output->output_ctx = NULL;
+                    ConfNode *sub_output_config = ConfNodeLookupChild(type, type->val);
-                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
+                    // sub_output_config may be NULL if no config
-                SCLogDebug("__tx_logger__ added");
+
-            }
+                    /* pass on parent output_ctx */
-        } else if (module->FileLogFunc) {
+                    OutputCtx *sub_output_ctx =
-            SCLogDebug("%s is a file logger", module->name);
+                        sub_module->InitSubFunc(sub_output_config, output_ctx);
-            OutputRegisterFileLogger(module->name, module->FileLogFunc, output_ctx);
+                    if (sub_output_ctx == NULL) {
-
+                        continue;
-            /* need one instance of the tx logger module */
+                    }
-            if (file_logger_module == NULL) {
+
-                file_logger_module = TmModuleGetByName("__file_logger__");
+                    SetupOutput(sub_module->name, sub_module, sub_output_ctx);
-                if (file_logger_module == NULL) {
-                    SCLogError(SC_ERR_INVALID_ARGUMENT,
-                            "TmModuleGetByName for __file_logger__ failed");
-                    exit(EXIT_FAILURE);
                 }
-
-                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-                if (unlikely(runmode_output == NULL))
-                    return;
-                runmode_output->tm_module = file_logger_module;
-                runmode_output->output_ctx = NULL;
-                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
-                SCLogDebug("__file_logger__ added");
-            }
-        } else if (module->FiledataLogFunc) {
-            SCLogDebug("%s is a filedata logger", module->name);
-            OutputRegisterFiledataLogger(module->name, module->FiledataLogFunc, output_ctx);
-
-            /* need one instance of the tx logger module */
-            if (filedata_logger_module == NULL) {
-                filedata_logger_module = TmModuleGetByName("__filedata_logger__");
-                if (filedata_logger_module == NULL) {
-                    SCLogError(SC_ERR_INVALID_ARGUMENT,
-                            "TmModuleGetByName for __filedata_logger__ failed");
-                    exit(EXIT_FAILURE);
-                }
-
-                RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-                if (unlikely(runmode_output == NULL))
-                    return;
-                runmode_output->tm_module = filedata_logger_module;
-                runmode_output->output_ctx = NULL;
-                TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
-                SCLogDebug("__filedata_logger__ added");
             }
         } else {
-            SCLogDebug("%s is a regular logger", module->name);
+            SetupOutput(module->name, module, output_ctx);
-
-            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-            if (unlikely(runmode_output == NULL))
-                return;
-            runmode_output->tm_module = tm_module;
-            runmode_output->output_ctx = output_ctx;
-            TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
         }
     }
 }