output: add new logger to log at specified state

Sometimes we want to log when we reach a specified state instead of
waiting for the session to end. E.g for TLS we want to log as soon
as the handshake is done.

To do this, a new logger is added, where it is possible to specify
a custom "ProgressCompletionStatus".

src/output-tx.c

@@ -51,11 +51,16 @@ typedef struct OutputTxLogger_ {
     struct OutputTxLogger_ *next;
     const char *name;
     TmmId module_id;
+    uint32_t id;
+    int tc_log_progress;
+    int ts_log_progress;
 } OutputTxLogger;
 
 static OutputTxLogger *list = NULL;
 
-int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc, OutputCtx *output_ctx)
+int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc,
+                           OutputCtx *output_ctx, int tc_log_progress,
+                           int ts_log_progress)
 {
     int module_id = TmModuleGetIdByName(name);
     if (module_id < 0)
@@ -72,12 +77,34 @@ int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc,
     op->name = name;
     op->module_id = (TmmId) module_id;
 
-    if (list == NULL)
+    if (tc_log_progress) {
+        op->tc_log_progress = tc_log_progress;
+    } else {
+        op->tc_log_progress =
+            AppLayerParserGetStateProgressCompletionStatus(alproto,
+                                                           STREAM_TOCLIENT);
+    }
+
+    if (ts_log_progress) {
+        op->ts_log_progress = ts_log_progress;
+    } else {
+        op->ts_log_progress =
+            AppLayerParserGetStateProgressCompletionStatus(alproto,
+                                                           STREAM_TOSERVER);
+    }
+
+    if (list == NULL) {
+        op->id = 1;
         list = op;
-    else {
+    } else {
         OutputTxLogger *t = list;
         while (t->next)
             t = t->next;
+        if (t->id * 2 > UINT32_MAX) {
+            SCLogError(SC_ERR_FATAL, "Too many loggers registered.");
+            exit(EXIT_FAILURE);
+        }
+        op->id = t->id * 2;
         t->next = op;
     }
 
@@ -119,15 +146,10 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQ
 
     uint64_t total_txs = AppLayerParserGetTxCnt(p->proto, alproto, alstate);
     uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
-    int tx_progress_done_value_ts =
+
-        AppLayerParserGetStateProgressCompletionStatus(alproto,
-                                                       STREAM_TOSERVER);
-    int tx_progress_done_value_tc =
-        AppLayerParserGetStateProgressCompletionStatus(alproto,
-                                                       STREAM_TOCLIENT);
     for (; tx_id < total_txs; tx_id++)
     {
-        int proto_logged = 0;
+        int logger_not_logged = 0;
 
         void *tx = AppLayerParserGetTx(p->proto, alproto, alstate, tx_id);
         if (tx == NULL) {
@@ -135,22 +157,11 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQ
             continue;
         }
 
-        if (!(AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF)))
+        int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto,
-        {
+                tx, FlowGetDisruptionFlags(f, STREAM_TOSERVER));
-            int tx_progress = AppLayerParserGetStateProgress(p->proto, alproto,
-                                                             tx, FlowGetDisruptionFlags(f, STREAM_TOSERVER));
-            if (tx_progress < tx_progress_done_value_ts) {
-                SCLogDebug("progress not far enough, not logging");
-                break;
-            }
 
-            tx_progress = AppLayerParserGetStateProgress(p->proto, alproto,
+        int tx_progress_tc = AppLayerParserGetStateProgress(p->proto, alproto,
-                                                         tx, FlowGetDisruptionFlags(f, STREAM_TOCLIENT));
+                tx, FlowGetDisruptionFlags(f, STREAM_TOCLIENT));
-            if (tx_progress < tx_progress_done_value_tc) {
-                SCLogDebug("progress not far enough, not logging");
-                break;
-            }
-        }
 
         // call each logger here (pseudo code)
         logger = list;
@@ -161,12 +172,38 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQ
             SCLogDebug("logger %p", logger);
             if (logger->alproto == alproto) {
                 SCLogDebug("alproto match, logging tx_id %ju", tx_id);
+
+                if (AppLayerParserGetTxLogged(p->proto, alproto, alstate, tx,
+                        logger->id)) {
+                    SCLogDebug("logger has already logged this transaction");
+
+                    goto next;
+                }
+
+                if (!(AppLayerParserStateIssetFlag(f->alparser,
+                                                   APP_LAYER_PARSER_EOF))) {
+                    if (tx_progress_tc < logger->tc_log_progress) {
+                        SCLogDebug("progress not far enough, not logging");
+                        logger_not_logged = 1;
+                        goto next;
+                    }
+
+                    if (tx_progress_ts < logger->ts_log_progress) {
+                        SCLogDebug("progress not far enough, not logging");
+                        logger_not_logged = 1;
+                        goto next;
+                    }
+                }
+
                 PACKET_PROFILING_TMM_START(p, logger->module_id);
                 logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
                 PACKET_PROFILING_TMM_END(p, logger->module_id);
-                proto_logged = 1;
+
+                AppLayerParserSetTxLogged(p->proto, alproto, alstate, tx,
+                                          logger->id);
             }
 
+next:
             logger = logger->next;
             store = store->next;
 
@@ -174,7 +211,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQ
             BUG_ON(logger != NULL && store == NULL);
         }
 
-        if (proto_logged) {
+        if (!logger_not_logged) {
             SCLogDebug("updating log tx_id %ju", tx_id);
             AppLayerParserSetTransactionLogId(f->alparser);
         }

src/output-tx.h

@@ -36,7 +36,8 @@ typedef int (*TxLogger)(ThreadVars *, void *thread_data, const Packet *, Flow *f
  */
 //typedef int (*TxLogCondition)(ThreadVars *, const Packet *);
 
-int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc, OutputCtx *);
+int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc,
+        OutputCtx *, int tc_log_progress, int ts_log_progress);
 
 void TmModuleTxLoggerRegister (void);
 

src/output.c

@@ -150,17 +150,16 @@ error:
 }
 
 /**
- * \brief Register a tx output module.
+ * \brief Register a tx output module with progress.
  *
  * This function will register an output module so it can be
  * configured with the configuration file.
  *
  * \retval Returns 0 on success, -1 on failure.
  */
-void
+void OutputRegisterTxModuleWithProgress(const char *name, const char *conf_name,
-OutputRegisterTxModule(const char *name, const char *conf_name,
+        OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto,
-    OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto,
+        TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress)
-    TxLogger TxLogFunc)
 {
     if (unlikely(TxLogFunc == NULL)) {
         goto error;
@@ -176,6 +175,8 @@ OutputRegisterTxModule(const char *name, const char *conf_name,
     module->InitFunc = InitFunc;
     module->TxLogFunc = TxLogFunc;
     module->alproto = alproto;
+    module->tc_log_progress = tc_log_progress;
+    module->ts_log_progress = ts_log_progress;
     TAILQ_INSERT_TAIL(&output_modules, module, entries);
 
     SCLogDebug("Tx logger \"%s\" registered.", name);
@@ -185,10 +186,10 @@ error:
     exit(EXIT_FAILURE);
 }
 
-void
+void OutputRegisterTxSubModuleWithProgress(const char *parent_name,
-OutputRegisterTxSubModule(const char *parent_name, const char *name,
+        const char *name, const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *,
-    const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx),
+        OutputCtx *parent_ctx), AppProto alproto, TxLogger TxLogFunc,
-    AppProto alproto, TxLogger TxLogFunc)
+        int tc_log_progress, int ts_log_progress)
 {
     if (unlikely(TxLogFunc == NULL)) {
         goto error;
@@ -205,6 +206,8 @@ OutputRegisterTxSubModule(const char *parent_name, const char *name,
     module->InitSubFunc = InitFunc;
     module->TxLogFunc = TxLogFunc;
     module->alproto = alproto;
+    module->tc_log_progress = tc_log_progress;
+    module->ts_log_progress = ts_log_progress;
     TAILQ_INSERT_TAIL(&output_modules, module, entries);
 
     SCLogDebug("Tx logger \"%s\" registered.", name);
@@ -214,6 +217,34 @@ error:
     exit(EXIT_FAILURE);
 }
 
+/**
+ * \brief Register a tx output module.
+ *
+ * This function will register an output module so it can be
+ * configured with the configuration file.
+ *
+ * \retval Returns 0 on success, -1 on failure.
+ */
+void
+OutputRegisterTxModule(const char *name, const char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto,
+    TxLogger TxLogFunc)
+{
+    /* wrapper function */
+    OutputRegisterTxModuleWithProgress(name, conf_name, InitFunc, alproto,
+                                       TxLogFunc, 0, 0);
+}
+
+void
+OutputRegisterTxSubModule(const char *parent_name, const char *name,
+    const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx),
+    AppProto alproto, TxLogger TxLogFunc)
+{
+    /* wrapper function */
+    OutputRegisterTxSubModuleWithProgress(parent_name, name, conf_name,
+                                          InitFunc, alproto, TxLogFunc, 0, 0);
+}
+
 /**
  * \brief Register a file output module.
  *

src/output.h

@@ -55,6 +55,8 @@ typedef struct OutputModule_ {
     StatsLogger StatsLogFunc;
     AppProto alproto;
     enum OutputStreamingType stream_type;
+    int tc_log_progress;
+    int ts_log_progress;
 
     TAILQ_ENTRY(OutputModule_) entries;
 } OutputModule;
@@ -75,6 +77,13 @@ void OutputRegisterTxSubModule(const char *parent_name, const char *name,
     const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx),
     AppProto alproto, TxLogger TxLogFunc);
 
+void OutputRegisterTxModuleWithProgress(const char *name, const char *conf_name,
+    OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto,
+    TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress);
+void OutputRegisterTxSubModuleWithProgress(const char *parent_name, const char *name,
+    const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx),
+    AppProto alproto, TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress);
+
 void OutputRegisterFileModule(const char *name, const char *conf_name,
     OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc);
 void OutputRegisterFileSubModule(const char *parent_name, const char *name,

src/runmodes.c

@@ -617,7 +617,8 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu
     } else if (module->TxLogFunc) {
         SCLogDebug("%s is a tx logger", module->name);
         OutputRegisterTxLogger(module->name, module->alproto,
-                module->TxLogFunc, output_ctx);
+                module->TxLogFunc, output_ctx, module->tc_log_progress,
+                module->ts_log_progress);
 
         /* need one instance of the tx logger module */
         if (tx_logger_module == NULL) {