logging: use a single entry point for all loggers

Introduces a new thread module, TMM_LOGGER, which is the
root most logger.

Only handles loggers in the packet path, stats and flow
logging are not included.

The loggers are made up of a hierarchy of loggers. At the top we
have the root logger which is the main entry point to
logging. Under the root there exists parent loggers that are the
entry point for specific types of loggers such as packet logger,
transaction loggers, etc. Each parent logger may have 0 or more
loggers that actual handle the job of producing output to something
like a file.

src/output-file.c

@@ -25,6 +25,7 @@
 
 #include "suricata-common.h"
 #include "tm-modules.h"
+#include "output.h"
 #include "output-file.h"
 #include "app-layer.h"
 #include "app-layer-parser.h"
@@ -92,7 +93,11 @@ int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc,
 static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq)
 {
     BUG_ON(thread_data == NULL);
-    BUG_ON(list == NULL);
+
+    if (list == NULL) {
+        /* No child loggers. */
+        return TM_ECODE_OK;
+    }
 
     OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
     OutputFileLogger *logger = list;
@@ -263,12 +268,8 @@ static void OutputFileLogExitPrintStats(ThreadVars *tv, void *thread_data)
 
 void TmModuleFileLoggerRegister (void)
 {
-    tmm_modules[TMM_FILELOGGER].name = "__file_logger__";
+    OutputRegisterRootLogger(OutputFileLogThreadInit,
-    tmm_modules[TMM_FILELOGGER].ThreadInit = OutputFileLogThreadInit;
+        OutputFileLogThreadDeinit, OutputFileLogExitPrintStats, OutputFileLog);
-    tmm_modules[TMM_FILELOGGER].Func = OutputFileLog;
-    tmm_modules[TMM_FILELOGGER].ThreadExitPrintStats = OutputFileLogExitPrintStats;
-    tmm_modules[TMM_FILELOGGER].ThreadDeinit = OutputFileLogThreadDeinit;
-    tmm_modules[TMM_FILELOGGER].cap_flags = 0;
 }
 
 void OutputFileShutdown(void)

src/output-filedata.c

@@ -25,6 +25,7 @@
 
 #include "suricata-common.h"
 #include "tm-modules.h"
+#include "output.h"
 #include "output-filedata.h"
 #include "app-layer.h"
 #include "app-layer-parser.h"
@@ -126,7 +127,11 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
 static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq)
 {
     BUG_ON(thread_data == NULL);
-    BUG_ON(list == NULL);
+
+    if (list == NULL) {
+        /* No child loggers. */
+        return TM_ECODE_OK;
+    }
 
     OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
     OutputFiledataLogger *logger = list;
@@ -431,13 +436,9 @@ static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
 
 void TmModuleFiledataLoggerRegister (void)
 {
-    tmm_modules[TMM_FILEDATALOGGER].name = "__filedata_logger__";
+    OutputRegisterRootLogger(OutputFiledataLogThreadInit,
-    tmm_modules[TMM_FILEDATALOGGER].ThreadInit = OutputFiledataLogThreadInit;
+        OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
-    tmm_modules[TMM_FILEDATALOGGER].Func = OutputFiledataLog;
+        OutputFiledataLog);
-    tmm_modules[TMM_FILEDATALOGGER].ThreadExitPrintStats = OutputFiledataLogExitPrintStats;
-    tmm_modules[TMM_FILEDATALOGGER].ThreadDeinit = OutputFiledataLogThreadDeinit;
-    tmm_modules[TMM_FILEDATALOGGER].cap_flags = 0;
-
     SC_ATOMIC_INIT(file_id);
 }
 

src/output-flow.c

@@ -93,7 +93,6 @@ TmEcode OutputFlowLog(ThreadVars *tv, void *thread_data, Flow *f)
 
     if (list == NULL)
         return TM_ECODE_OK;
-    //BUG_ON(list == NULL);
 
     OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
     OutputFlowLogger *logger = list;

src/output-packet.c

@@ -93,7 +93,11 @@ int OutputRegisterPacketLogger(LoggerId logger_id, const char *name,
 static TmEcode OutputPacketLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq)
 {
     BUG_ON(thread_data == NULL);
-    BUG_ON(list == NULL);
+
+    if (list == NULL) {
+        /* No child loggers. */
+        return TM_ECODE_OK;
+    }
 
     OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
     OutputPacketLogger *logger = list;
@@ -207,12 +211,9 @@ static void OutputPacketLogExitPrintStats(ThreadVars *tv, void *thread_data)
 
 void TmModulePacketLoggerRegister (void)
 {
-    tmm_modules[TMM_PACKETLOGGER].name = "__packet_logger__";
+    OutputRegisterRootLogger(OutputPacketLogThreadInit,
-    tmm_modules[TMM_PACKETLOGGER].ThreadInit = OutputPacketLogThreadInit;
+        OutputPacketLogThreadDeinit, OutputPacketLogExitPrintStats,
-    tmm_modules[TMM_PACKETLOGGER].Func = OutputPacketLog;
+        OutputPacketLog);
-    tmm_modules[TMM_PACKETLOGGER].ThreadExitPrintStats = OutputPacketLogExitPrintStats;
-    tmm_modules[TMM_PACKETLOGGER].ThreadDeinit = OutputPacketLogThreadDeinit;
-    tmm_modules[TMM_PACKETLOGGER].cap_flags = 0;
 }
 
 void OutputPacketShutdown(void)

src/output-stats.c

@@ -168,12 +168,7 @@ static TmEcode OutputStatsLogThreadDeinit(ThreadVars *tv, void *thread_data)
     while (logger && store) {
         if (logger->ThreadDeinit) {
             logger->ThreadDeinit(tv, store->thread_data);
-        } else {
-            /* XXX Temporary. */
-            SCLogNotice("Logger {%s} does not have ThreadDeinit.",
-                logger->name);
         }
-
         OutputLoggerThreadStore *next_store = store->next;
         SCFree(store);
         store = next_store;
@@ -204,7 +199,6 @@ void TmModuleStatsLoggerRegister (void)
 {
     tmm_modules[TMM_STATSLOGGER].name = "__stats_logger__";
     tmm_modules[TMM_STATSLOGGER].ThreadInit = OutputStatsLogThreadInit;
-    //tmm_modules[TMM_STATSLOGGER].Func = OutputStatsLog;
     tmm_modules[TMM_STATSLOGGER].ThreadExitPrintStats = OutputStatsLogExitPrintStats;
     tmm_modules[TMM_STATSLOGGER].ThreadDeinit = OutputStatsLogThreadDeinit;
     tmm_modules[TMM_STATSLOGGER].cap_flags = 0;

src/output-streaming.c

@@ -25,6 +25,7 @@
 
 #include "suricata-common.h"
 #include "tm-modules.h"
+#include "output.h"
 #include "output-streaming.h"
 #include "app-layer.h"
 #include "app-layer-parser.h"
@@ -300,7 +301,11 @@ int StreamIterator(Flow *f, TcpStream *stream, int close, void *cbdata, uint8_t
 static TmEcode OutputStreamingLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq)
 {
     BUG_ON(thread_data == NULL);
-    BUG_ON(list == NULL);
+
+    if (list == NULL) {
+        /* No child loggers. */
+        return TM_ECODE_OK;
+    }
 
     OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
     OutputStreamingLogger *logger = list;
@@ -437,12 +442,9 @@ static void OutputStreamingLogExitPrintStats(ThreadVars *tv, void *thread_data)
 }
 
 void TmModuleStreamingLoggerRegister (void) {
-    tmm_modules[TMM_STREAMINGLOGGER].name = "__streaming_logger__";
+    OutputRegisterRootLogger(OutputStreamingLogThreadInit,
-    tmm_modules[TMM_STREAMINGLOGGER].ThreadInit = OutputStreamingLogThreadInit;
+        OutputStreamingLogThreadDeinit, OutputStreamingLogExitPrintStats,
-    tmm_modules[TMM_STREAMINGLOGGER].Func = OutputStreamingLog;
+        OutputStreamingLog);
-    tmm_modules[TMM_STREAMINGLOGGER].ThreadExitPrintStats = OutputStreamingLogExitPrintStats;
-    tmm_modules[TMM_STREAMINGLOGGER].ThreadDeinit = OutputStreamingLogThreadDeinit;
-    tmm_modules[TMM_STREAMINGLOGGER].cap_flags = 0;
 }
 
 void OutputStreamingShutdown(void)

src/output-tx.c

@@ -25,6 +25,7 @@
 
 #include "suricata-common.h"
 #include "tm-modules.h"
+#include "output.h"
 #include "output-tx.h"
 #include "app-layer.h"
 #include "app-layer-parser.h"
@@ -70,12 +71,6 @@ int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto,
                            ThreadDeinitFunc ThreadDeinit,
                            void (*ThreadExitPrintStats)(ThreadVars *, void *))
 {
-#if 0
-    int module_id = TmModuleGetIdByName(name);
-    if (module_id < 0)
-        return -1;
-#endif
-
     if (!(AppLayerParserIsTxAware(alproto))) {
         SCLogNotice("%s logger not enabled: protocol %s is disabled",
             name, AppProtoToString(alproto));
@@ -267,7 +262,6 @@ static TmEcode OutputTxLogThreadInit(ThreadVars *tv, void *initdata, void **data
     memset(td, 0x00, sizeof(*td));
 
     *data = (void *)td;
-
     SCLogDebug("OutputTxLogThreadInit happy (*data %p)", *data);
 
     OutputTxLogger *logger = list;
@@ -340,12 +334,8 @@ static void OutputTxLogExitPrintStats(ThreadVars *tv, void *thread_data)
 
 void TmModuleTxLoggerRegister (void)
 {
-    tmm_modules[TMM_TXLOGGER].name = "__tx_logger__";
+    OutputRegisterRootLogger(OutputTxLogThreadInit, OutputTxLogThreadDeinit,
-    tmm_modules[TMM_TXLOGGER].ThreadInit = OutputTxLogThreadInit;
+        OutputTxLogExitPrintStats, OutputTxLog);
-    tmm_modules[TMM_TXLOGGER].Func = OutputTxLog;
-    tmm_modules[TMM_TXLOGGER].ThreadExitPrintStats = OutputTxLogExitPrintStats;
-    tmm_modules[TMM_TXLOGGER].ThreadDeinit = OutputTxLogThreadDeinit;
-    tmm_modules[TMM_TXLOGGER].cap_flags = 0;
 }
 
 void OutputTxShutdown(void)

src/output.c

@@ -18,9 +18,18 @@
 /**
  * \file
  *
+ * \author OISF, Jason Ish <jason.ish@oisf.net>
  * \author Endace Technology Limited, Jason Ish <jason.ish@endace.com>
  *
- * Output registration functions
+ * The root logging output for all non-application logging.
+ *
+ * The loggers are made up of a hierarchy of loggers. At the top we
+ * have the root logger which is the main entry point to
+ * logging. Under the root there exists parent loggers that are the
+ * entry point for specific types of loggers such as packet logger,
+ * transaction loggers, etc. Each parent logger may have 0 or more
+ * loggers that actual handle the job of producing output to something
+ * like a file.
  */
 
 #include "suricata-common.h"
@@ -31,6 +40,25 @@
 #include "util-debug.h"
 #include "output.h"
 
+typedef struct RootLogger_ {
+    ThreadInitFunc ThreadInit;
+    ThreadDeinitFunc ThreadDeinit;
+    ThreadExitPrintStatsFunc ThreadExitPrintStats;
+    OutputLogFunc LogFunc;
+
+    TAILQ_ENTRY(RootLogger_) entries;
+} RootLogger;
+
+static TAILQ_HEAD(, RootLogger_) RootLoggers =
+    TAILQ_HEAD_INITIALIZER(RootLoggers);
+
+typedef struct LoggerThreadStoreNode_ {
+    void *thread_data;
+    TAILQ_ENTRY(LoggerThreadStoreNode_) entries;
+} LoggerThreadStoreNode;
+
+typedef TAILQ_HEAD(LoggerThreadStore_, LoggerThreadStoreNode_) LoggerThreadStore;
+
 OutputModuleList output_modules = TAILQ_HEAD_INITIALIZER(output_modules);
 
 /**
@@ -851,3 +879,114 @@ void OutputNotifyFileRotation(void) {
         *(flag->flag) = 1;
     }
 }
+
+static TmEcode OutputLoggerLog(ThreadVars *tv, Packet *p, void *thread_data,
+    PacketQueue *pq, PacketQueue *postpq)
+{
+    LoggerThreadStore *thread_store = (LoggerThreadStore *)thread_data;
+    RootLogger *logger = TAILQ_FIRST(&RootLoggers);
+    LoggerThreadStoreNode *thread_store_node = TAILQ_FIRST(thread_store);
+    while (logger && thread_store_node) {
+        if (logger->LogFunc != NULL) {
+            logger->LogFunc(tv, p, thread_store_node->thread_data, pq, postpq);
+        }
+        logger = TAILQ_NEXT(logger, entries);
+        thread_store_node = TAILQ_NEXT(thread_store_node, entries);
+    }
+
+    return TM_ECODE_OK;
+}
+
+static TmEcode OutputLoggerThreadInit(ThreadVars *tv, void *initdata,
+    void **data)
+{
+    LoggerThreadStore *thread_store = SCCalloc(1, sizeof(*thread_store));
+    if (thread_store == NULL) {
+        return TM_ECODE_FAILED;
+    }
+    TAILQ_INIT(thread_store);
+    *data = (void *)thread_store;
+
+    RootLogger *logger;
+    TAILQ_FOREACH(logger, &RootLoggers, entries) {
+
+        void *child_thread_data = NULL;
+        if (logger->ThreadInit != NULL) {
+            if (logger->ThreadInit(tv, initdata, &child_thread_data) == TM_ECODE_OK) {
+                LoggerThreadStoreNode *thread_store_node =
+                    SCCalloc(1, sizeof(*thread_store_node));
+                BUG_ON(thread_store_node == NULL);
+                thread_store_node->thread_data = child_thread_data;
+                TAILQ_INSERT_TAIL(thread_store, thread_store_node, entries);
+            }
+        }
+    }
+    return TM_ECODE_OK;
+}
+
+static TmEcode OutputLoggerThreadDeinit(ThreadVars *tv, void *thread_data)
+{
+    LoggerThreadStore *thread_store = (LoggerThreadStore *)thread_data;
+    RootLogger *logger = TAILQ_FIRST(&RootLoggers);
+    LoggerThreadStoreNode *thread_store_node = TAILQ_FIRST(thread_store);
+    while (logger && thread_store_node) {
+        if (logger->ThreadDeinit != NULL) {
+            logger->ThreadDeinit(tv, thread_store_node->thread_data);
+        }
+        logger = TAILQ_NEXT(logger, entries);
+        thread_store_node = TAILQ_NEXT(thread_store_node, entries);
+    }
+
+    /* Free the thread store. */
+    while ((thread_store_node = TAILQ_FIRST(thread_store)) != NULL) {
+        TAILQ_REMOVE(thread_store, thread_store_node, entries);
+        SCFree(thread_store_node);
+    }
+    SCFree(thread_store);
+
+    return TM_ECODE_OK;
+}
+
+static void OutputLoggerExitPrintStats(ThreadVars *tv, void *thread_data)
+{
+    LoggerThreadStore *thread_store = (LoggerThreadStore *)thread_data;
+    RootLogger *logger = TAILQ_FIRST(&RootLoggers);
+    LoggerThreadStoreNode *thread_store_node = TAILQ_FIRST(thread_store);
+    while (logger && thread_store_node) {
+        if (logger->ThreadExitPrintStats != NULL) {
+            logger->ThreadExitPrintStats(tv, thread_store_node->thread_data);
+        }
+        logger = TAILQ_NEXT(logger, entries);
+        thread_store_node = TAILQ_NEXT(thread_store_node, entries);
+    }
+}
+
+void OutputRegisterRootLogger(ThreadInitFunc ThreadInit,
+    ThreadDeinitFunc ThreadDeinit,
+    ThreadExitPrintStatsFunc ThreadExitPrintStats,
+    OutputLogFunc LogFunc)
+{
+    RootLogger *logger = SCCalloc(1, sizeof(*logger));
+    if (logger == NULL) {
+        return;
+    }
+    logger->ThreadInit = ThreadInit;
+    logger->ThreadDeinit = ThreadDeinit;
+    logger->ThreadExitPrintStats = ThreadExitPrintStats;
+    logger->LogFunc = LogFunc;
+    TAILQ_INSERT_TAIL(&RootLoggers, logger, entries);
+}
+
+void TmModuleLoggerRegister(void)
+{
+    tmm_modules[TMM_LOGGER].name = "__root_logger__";
+    tmm_modules[TMM_LOGGER].ThreadInit = OutputLoggerThreadInit;
+    tmm_modules[TMM_LOGGER].ThreadDeinit = OutputLoggerThreadDeinit;
+    tmm_modules[TMM_LOGGER].ThreadExitPrintStats = OutputLoggerExitPrintStats;
+    tmm_modules[TMM_LOGGER].Func = OutputLoggerLog;;
+}
+
+void SetupOutputs(ThreadVars *tv)
+{
+    TmSlotSetFuncAppend(tv, &tmm_modules[TMM_LOGGER], NULL);
+}

src/output.h

@@ -40,6 +40,8 @@
 
 typedef OutputCtx *(*OutputInitFunc)(ConfNode *);
 typedef OutputCtx *(*OutputInitSubFunc)(ConfNode *, OutputCtx *);
+typedef TmEcode (*OutputLogFunc)(ThreadVars *, Packet *, void *, PacketQueue *,
+    PacketQueue *);
 
 typedef struct OutputModule_ {
     LoggerId logger_id;
@@ -190,4 +192,11 @@ void OutputRegisterFileRotationFlag(int *flag);
 void OutputUnregisterFileRotationFlag(int *flag);
 void OutputNotifyFileRotation(void);
 
+void OutputRegisterRootLogger(ThreadInitFunc ThreadInit,
+    ThreadDeinitFunc ThreadDeinit,
+    ThreadExitPrintStatsFunc ThreadExitPrintStats,
+    OutputLogFunc LogFunc);
+void TmModuleLoggerRegister(void);
+void SetupOutputs(ThreadVars *);
+
 #endif /* ! __OUTPUT_H__ */

src/runmode-erf-file.c

@@ -231,7 +231,7 @@ int RunModeErfFileAutoFp(void)
 
         TmThreadSetGroupName(tv_detect_ncpu, "Detect");
 
-        /* add outputs as well */
+        /* Add logger. */
         SetupOutputs(tv_detect_ncpu);
 
         if (TmThreadSpawn(tv_detect_ncpu) != TM_ECODE_OK) {

src/runmode-pcap-file.c

@@ -250,7 +250,7 @@ int RunModeFilePcapAutoFp(void)
 
         TmThreadSetGroupName(tv_detect_ncpu, "Detect");
 
-        /* add outputs as well */
+        /* Add logger. */
         SetupOutputs(tv_detect_ncpu);
 
         TmThreadSetCPU(tv_detect_ncpu, WORKER_CPU_SET);

src/runmodes.c

@@ -87,19 +87,6 @@ typedef struct RunModes_ {
     RunMode *runmodes;
 } RunModes;
 
-/**
- * A list of output modules that will be active for the run mode.
- */
-typedef struct RunModeOutput_ {
-    const char *name;
-    TmModule *tm_module;
-    OutputCtx *output_ctx;
-
-    TAILQ_ENTRY(RunModeOutput_) entries;
-} RunModeOutput;
-TAILQ_HEAD(, RunModeOutput_) RunModeOutputs =
-    TAILQ_HEAD_INITIALIZER(RunModeOutputs);
-
 static RunModes runmodes[RUNMODE_USER_MAX];
 
 static char *active_runmode;
@@ -111,7 +98,7 @@ typedef struct OutputFreeList_ {
 
     TAILQ_ENTRY(OutputFreeList_) entries;
 } OutputFreeList;
-TAILQ_HEAD(, OutputFreeList_) output_free_list =
+static TAILQ_HEAD(, OutputFreeList_) output_free_list =
     TAILQ_HEAD_INITIALIZER(output_free_list);
 
 /**
@@ -462,7 +449,7 @@ void RunModeRegisterNewRunMode(int runmode, const char *name,
  * \param tv The ThreadVars for the thread the outputs will be
  * appended to.
  */
-void RunOutputFreeList(void)
+static void RunOutputFreeList(void)
 {
     OutputFreeList *output;
     while ((output = TAILQ_FIRST(&output_free_list))) {
@@ -477,20 +464,17 @@ void RunOutputFreeList(void)
     }
 }
 
-static TmModule *pkt_logger_module = NULL;
+static int file_logger_count = 0;
-static TmModule *tx_logger_module = NULL;
+static int filedata_logger_count = 0;
-static TmModule *file_logger_module = NULL;
-static TmModule *filedata_logger_module = NULL;
-static TmModule *streaming_logger_module = NULL;
 
 int RunModeOutputFileEnabled(void)
 {
-    return (file_logger_module != NULL);
+    return file_logger_count > 0;
 }
 
 int RunModeOutputFiledataEnabled(void)
 {
-    return (filedata_logger_module != NULL);
+    return filedata_logger_count > 0;
 }
 
 /**
@@ -508,20 +492,9 @@ void RunModeShutDown(void)
     OutputStatsShutdown();
     OutputFlowShutdown();
 
-    /* Close any log files. */
+    /* Reset logger counts. */
-    RunModeOutput *output;
+    file_logger_count = 0;
-    while ((output = TAILQ_FIRST(&RunModeOutputs))) {
+    filedata_logger_count = 0;
-        SCLogDebug("Shutting down output %s.", output->tm_module->name);
-        TAILQ_REMOVE(&RunModeOutputs, output, entries);
-        SCFree(output);
-    }
-
-    /* reset logger pointers */
-    pkt_logger_module = NULL;
-    tx_logger_module = NULL;
-    file_logger_module = NULL;
-    filedata_logger_module = NULL;
-    streaming_logger_module = NULL;
 }
 
 /** \internal
@@ -537,11 +510,6 @@ static void AddOutputToFreeList(OutputModule *module, OutputCtx *output_ctx)
     TAILQ_INSERT_TAIL(&output_free_list, fl_output, entries);
 }
 
-static void InsertInRunModeOutputs(RunModeOutput *runmode_output)
-{
-    TAILQ_INSERT_TAIL(&RunModeOutputs, runmode_output, entries);
-}
-
 /** \brief Turn output into thread module */
 static void SetupOutput(const char *name, OutputModule *module, OutputCtx *output_ctx)
 {
@@ -565,30 +533,11 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu
     }
 
     if (module->PacketLogFunc) {
-        SCLogNotice("%s is a packet logger", module->name);
+        SCLogDebug("%s is a packet logger", module->name);
         OutputRegisterPacketLogger(module->logger_id, module->name,
             module->PacketLogFunc, module->PacketConditionFunc, output_ctx,
             module->ThreadInit, module->ThreadDeinit,
             module->ThreadExitPrintStats);
-
-        /* need one instance of the packet logger module */
-        if (pkt_logger_module == NULL) {
-            pkt_logger_module = TmModuleGetByName("__packet_logger__");
-            if (pkt_logger_module == NULL) {
-                SCLogError(SC_ERR_INVALID_ARGUMENT,
-                        "TmModuleGetByName for __packet_logger__ failed");
-                exit(EXIT_FAILURE);
-            }
-
-            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-            if (unlikely(runmode_output == NULL))
-                return;
-            runmode_output->name = module->name;
-            runmode_output->tm_module = pkt_logger_module;
-            runmode_output->output_ctx = NULL;
-            InsertInRunModeOutputs(runmode_output);
-            SCLogDebug("__packet_logger__ added");
-        }
     } else if (module->TxLogFunc) {
         SCLogDebug("%s is a tx logger", module->name);
         OutputRegisterTxLogger(module->logger_id, module->name, module->alproto,
@@ -596,107 +545,27 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu
                 module->ts_log_progress, module->TxLogCondition,
                 module->ThreadInit, module->ThreadDeinit,
                 module->ThreadExitPrintStats);
-
-        /* need one instance of the tx logger module */
-        if (tx_logger_module == NULL) {
-            tx_logger_module = TmModuleGetByName("__tx_logger__");
-            if (tx_logger_module == NULL) {
-                SCLogError(SC_ERR_INVALID_ARGUMENT,
-                        "TmModuleGetByName for __tx_logger__ failed");
-                exit(EXIT_FAILURE);
-            }
-
-            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-            if (unlikely(runmode_output == NULL))
-                return;
-            runmode_output->name = module->name;
-            runmode_output->tm_module = tx_logger_module;
-            runmode_output->output_ctx = NULL;
-            InsertInRunModeOutputs(runmode_output);
-            SCLogDebug("__tx_logger__ added");
-        }
     } else if (module->FiledataLogFunc) {
         SCLogDebug("%s is a filedata logger", module->name);
         OutputRegisterFiledataLogger(module->logger_id, module->name,
             module->FiledataLogFunc, output_ctx, module->ThreadInit,
             module->ThreadDeinit, module->ThreadExitPrintStats);
-
+        filedata_logger_count++;
-        /* need one instance of the tx logger module */
-        if (filedata_logger_module == NULL) {
-            filedata_logger_module = TmModuleGetByName("__filedata_logger__");
-            if (filedata_logger_module == NULL) {
-                SCLogError(SC_ERR_INVALID_ARGUMENT,
-                        "TmModuleGetByName for __filedata_logger__ failed");
-                exit(EXIT_FAILURE);
-            }
-
-            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-            if (unlikely(runmode_output == NULL))
-                return;
-            runmode_output->name = module->name;
-            runmode_output->tm_module = filedata_logger_module;
-            runmode_output->output_ctx = NULL;
-            InsertInRunModeOutputs(runmode_output);
-            SCLogDebug("__filedata_logger__ added");
-        }
     } else if (module->FileLogFunc) {
         SCLogDebug("%s is a file logger", module->name);
         OutputRegisterFileLogger(module->logger_id, module->name,
             module->FileLogFunc, output_ctx, module->ThreadInit,
             module->ThreadDeinit, module->ThreadExitPrintStats);
-
+        file_logger_count++;
-        /* need one instance of the tx logger module */
-        if (file_logger_module == NULL) {
-            file_logger_module = TmModuleGetByName("__file_logger__");
-            if (file_logger_module == NULL) {
-                SCLogError(SC_ERR_INVALID_ARGUMENT,
-                        "TmModuleGetByName for __file_logger__ failed");
-                exit(EXIT_FAILURE);
-            }
-
-            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-            if (unlikely(runmode_output == NULL))
-                return;
-            runmode_output->name = module->name;
-            runmode_output->tm_module = file_logger_module;
-            runmode_output->output_ctx = NULL;
-            InsertInRunModeOutputs(runmode_output);
-            SCLogDebug("__file_logger__ added");
-        }
     } else if (module->StreamingLogFunc) {
         SCLogDebug("%s is a streaming logger", module->name);
         OutputRegisterStreamingLogger(module->logger_id, module->name,
             module->StreamingLogFunc, output_ctx, module->stream_type,
             module->ThreadInit, module->ThreadDeinit,
             module->ThreadExitPrintStats);
-
-        /* need one instance of the streaming logger module */
-        if (streaming_logger_module == NULL) {
-            streaming_logger_module = TmModuleGetByName("__streaming_logger__");
-            if (streaming_logger_module == NULL) {
-                SCLogError(SC_ERR_INVALID_ARGUMENT,
-                        "TmModuleGetByName for __streaming_logger__ failed");
-                exit(EXIT_FAILURE);
-            }
-
-            RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-            if (unlikely(runmode_output == NULL))
-                return;
-            runmode_output->name = module->name;
-            runmode_output->tm_module = streaming_logger_module;
-            runmode_output->output_ctx = NULL;
-            InsertInRunModeOutputs(runmode_output);
-            SCLogDebug("__streaming_logger__ added");
-        }
     } else {
-        SCLogDebug("%s is a regular logger", module->name);
+        SCLogError(SC_ERR_INVALID_ARGUMENT, "Unknown logger type: name=%s",
-
+            module->name);
-        RunModeOutput *runmode_output = SCCalloc(1, sizeof(RunModeOutput));
-        if (unlikely(runmode_output == NULL))
-            return;
-        runmode_output->name = module->name;
-        runmode_output->output_ctx = output_ctx;
-        InsertInRunModeOutputs(runmode_output);
     }
 }
 
@@ -945,21 +814,6 @@ void RunModeInitializeOutputs(void)
 
 }
 
-/**
- * Setup the outputs for this run mode.
- *
- * \param tv The ThreadVars for the thread the outputs will be
- * appended to.
- */
-void SetupOutputs(ThreadVars *tv)
-{
-    RunModeOutput *output;
-    TAILQ_FOREACH(output, &RunModeOutputs, entries) {
-        tv->cap_flags |= output->tm_module->cap_flags;
-        TmSlotSetFuncAppend(tv, output->tm_module, output->output_ctx);
-    }
-}
-
 float threading_detect_ratio = 1;
 
 /**

src/runmodes.h

@@ -82,7 +82,6 @@ void RunModeRegisterNewRunMode(int, const char *, const char *,
                                int (*RunModeFunc)(void));
 void RunModeInitialize(void);
 void RunModeInitializeOutputs(void);
-void SetupOutputs(ThreadVars *);
 void RunModeShutDown(void);
 
 /* bool indicating if file logger is enabled */

src/suricata.c

@@ -919,6 +919,7 @@ void RegisterAllModules()
     TmModuleJsonTemplateLogRegister();
 
     /* log api */
+    TmModuleLoggerRegister();
     TmModulePacketLoggerRegister();
     TmModuleTxLoggerRegister();
     TmModuleFileLoggerRegister();

src/tm-modules.c

@@ -227,18 +227,14 @@ const char * TmModuleTmmIdToString(TmmId id)
         CASE_CODE (TMM_RECEIVEAFP);
         CASE_CODE (TMM_ALERTPCAPINFO);
         CASE_CODE (TMM_DECODEAFP);
-        CASE_CODE (TMM_PACKETLOGGER);
-        CASE_CODE (TMM_TXLOGGER);
         CASE_CODE (TMM_STATSLOGGER);
-        CASE_CODE (TMM_FILELOGGER);
-        CASE_CODE (TMM_FILEDATALOGGER);
-        CASE_CODE (TMM_STREAMINGLOGGER);
         CASE_CODE (TMM_FLOWMANAGER);
         CASE_CODE (TMM_FLOWRECYCLER);
         CASE_CODE (TMM_UNIXMANAGER);
         CASE_CODE (TMM_DETECTLOADER);
         CASE_CODE (TMM_RECEIVENETMAP);
         CASE_CODE (TMM_DECODENETMAP);
+        CASE_CODE (TMM_LOGGER);
 
         CASE_CODE (TMM_SIZE);
     }

src/tm-threads-common.h

@@ -59,12 +59,7 @@ typedef enum {
     TMM_DECODEMPIPE,
     TMM_RECEIVENAPATECH,
     TMM_DECODENAPATECH,
-    TMM_PACKETLOGGER,
-    TMM_TXLOGGER,
     TMM_STATSLOGGER,
-    TMM_FILELOGGER,
-    TMM_FILEDATALOGGER,
-    TMM_STREAMINGLOGGER,
     TMM_RECEIVENFLOG,
     TMM_DECODENFLOG,
 
@@ -74,6 +69,8 @@ typedef enum {
 
     TMM_UNIXMANAGER,
 
+    TMM_LOGGER,
+
     TMM_SIZE,
 } TmmId;
 

src/util-runmodes.c

@@ -257,7 +257,7 @@ int RunModeSetLiveCaptureAutoFp(ConfigIfaceParserFunc ConfigParser,
         }
         TmSlotSetFuncAppend(tv_detect_ncpu, tm_module, NULL);
 
-        /* add outputs as well */
+        /* Add logger. */
         SetupOutputs(tv_detect_ncpu);
 
         if (TmThreadSpawn(tv_detect_ncpu) != TM_ECODE_OK) {