From fa6257b66ab4415ad51a0ca09a24cb6c65bd36ef Mon Sep 17 00:00:00 2001 From: Mats Klepsland Date: Wed, 27 Apr 2016 14:13:44 +0200 Subject: [PATCH] output: add new logger to log at specified state Sometimes we want to log when we reach a specified state instead of waiting for the session to end. E.g for TLS we want to log as soon as the handshake is done. To do this, a new logger is added, where it is possible to specify a custom "ProgressCompletionStatus". --- src/output-tx.c | 91 ++++++++++++++++++++++++++++++++++--------------- src/output-tx.h | 3 +- src/output.c | 49 +++++++++++++++++++++----- src/output.h | 9 +++++ src/runmodes.c | 3 +- 5 files changed, 117 insertions(+), 38 deletions(-) diff --git a/src/output-tx.c b/src/output-tx.c index 1e3396bd5f..79b8963259 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -51,11 +51,16 @@ typedef struct OutputTxLogger_ { struct OutputTxLogger_ *next; const char *name; TmmId module_id; + uint32_t id; + int tc_log_progress; + int ts_log_progress; } OutputTxLogger; static OutputTxLogger *list = NULL; -int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc, OutputCtx *output_ctx) +int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc, + OutputCtx *output_ctx, int tc_log_progress, + int ts_log_progress) { int module_id = TmModuleGetIdByName(name); if (module_id < 0) @@ -72,12 +77,34 @@ int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc, op->name = name; op->module_id = (TmmId) module_id; - if (list == NULL) + if (tc_log_progress) { + op->tc_log_progress = tc_log_progress; + } else { + op->tc_log_progress = + AppLayerParserGetStateProgressCompletionStatus(alproto, + STREAM_TOCLIENT); + } + + if (ts_log_progress) { + op->ts_log_progress = ts_log_progress; + } else { + op->ts_log_progress = + AppLayerParserGetStateProgressCompletionStatus(alproto, + STREAM_TOSERVER); + } + + if (list == NULL) { + op->id = 1; list = op; - else { + } else { OutputTxLogger *t = list; while (t->next) t = t->next; + if (t->id * 2 > UINT32_MAX) { + SCLogError(SC_ERR_FATAL, "Too many loggers registered."); + exit(EXIT_FAILURE); + } + op->id = t->id * 2; t->next = op; } @@ -119,15 +146,10 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQ uint64_t total_txs = AppLayerParserGetTxCnt(p->proto, alproto, alstate); uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser); - int tx_progress_done_value_ts = - AppLayerParserGetStateProgressCompletionStatus(alproto, - STREAM_TOSERVER); - int tx_progress_done_value_tc = - AppLayerParserGetStateProgressCompletionStatus(alproto, - STREAM_TOCLIENT); + for (; tx_id < total_txs; tx_id++) { - int proto_logged = 0; + int logger_not_logged = 0; void *tx = AppLayerParserGetTx(p->proto, alproto, alstate, tx_id); if (tx == NULL) { @@ -135,22 +157,11 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQ continue; } - if (!(AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF))) - { - int tx_progress = AppLayerParserGetStateProgress(p->proto, alproto, - tx, FlowGetDisruptionFlags(f, STREAM_TOSERVER)); - if (tx_progress < tx_progress_done_value_ts) { - SCLogDebug("progress not far enough, not logging"); - break; - } + int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto, + tx, FlowGetDisruptionFlags(f, STREAM_TOSERVER)); - tx_progress = AppLayerParserGetStateProgress(p->proto, alproto, - tx, FlowGetDisruptionFlags(f, STREAM_TOCLIENT)); - if (tx_progress < tx_progress_done_value_tc) { - SCLogDebug("progress not far enough, not logging"); - break; - } - } + int tx_progress_tc = AppLayerParserGetStateProgress(p->proto, alproto, + tx, FlowGetDisruptionFlags(f, STREAM_TOCLIENT)); // call each logger here (pseudo code) logger = list; @@ -161,12 +172,38 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQ SCLogDebug("logger %p", logger); if (logger->alproto == alproto) { SCLogDebug("alproto match, logging tx_id %ju", tx_id); + + if (AppLayerParserGetTxLogged(p->proto, alproto, alstate, tx, + logger->id)) { + SCLogDebug("logger has already logged this transaction"); + + goto next; + } + + if (!(AppLayerParserStateIssetFlag(f->alparser, + APP_LAYER_PARSER_EOF))) { + if (tx_progress_tc < logger->tc_log_progress) { + SCLogDebug("progress not far enough, not logging"); + logger_not_logged = 1; + goto next; + } + + if (tx_progress_ts < logger->ts_log_progress) { + SCLogDebug("progress not far enough, not logging"); + logger_not_logged = 1; + goto next; + } + } + PACKET_PROFILING_TMM_START(p, logger->module_id); logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id); PACKET_PROFILING_TMM_END(p, logger->module_id); - proto_logged = 1; + + AppLayerParserSetTxLogged(p->proto, alproto, alstate, tx, + logger->id); } +next: logger = logger->next; store = store->next; @@ -174,7 +211,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQ BUG_ON(logger != NULL && store == NULL); } - if (proto_logged) { + if (!logger_not_logged) { SCLogDebug("updating log tx_id %ju", tx_id); AppLayerParserSetTransactionLogId(f->alparser); } diff --git a/src/output-tx.h b/src/output-tx.h index 7ffb8a6d25..e8e8163d89 100644 --- a/src/output-tx.h +++ b/src/output-tx.h @@ -36,7 +36,8 @@ typedef int (*TxLogger)(ThreadVars *, void *thread_data, const Packet *, Flow *f */ //typedef int (*TxLogCondition)(ThreadVars *, const Packet *); -int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc, OutputCtx *); +int OutputRegisterTxLogger(const char *name, AppProto alproto, TxLogger LogFunc, + OutputCtx *, int tc_log_progress, int ts_log_progress); void TmModuleTxLoggerRegister (void); diff --git a/src/output.c b/src/output.c index 1146b7b226..d7c5ff910e 100644 --- a/src/output.c +++ b/src/output.c @@ -150,17 +150,16 @@ error: } /** - * \brief Register a tx output module. + * \brief Register a tx output module with progress. * * This function will register an output module so it can be * configured with the configuration file. * * \retval Returns 0 on success, -1 on failure. */ -void -OutputRegisterTxModule(const char *name, const char *conf_name, - OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto, - TxLogger TxLogFunc) +void OutputRegisterTxModuleWithProgress(const char *name, const char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto, + TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress) { if (unlikely(TxLogFunc == NULL)) { goto error; @@ -176,6 +175,8 @@ OutputRegisterTxModule(const char *name, const char *conf_name, module->InitFunc = InitFunc; module->TxLogFunc = TxLogFunc; module->alproto = alproto; + module->tc_log_progress = tc_log_progress; + module->ts_log_progress = ts_log_progress; TAILQ_INSERT_TAIL(&output_modules, module, entries); SCLogDebug("Tx logger \"%s\" registered.", name); @@ -185,10 +186,10 @@ error: exit(EXIT_FAILURE); } -void -OutputRegisterTxSubModule(const char *parent_name, const char *name, - const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), - AppProto alproto, TxLogger TxLogFunc) +void OutputRegisterTxSubModuleWithProgress(const char *parent_name, + const char *name, const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, + OutputCtx *parent_ctx), AppProto alproto, TxLogger TxLogFunc, + int tc_log_progress, int ts_log_progress) { if (unlikely(TxLogFunc == NULL)) { goto error; @@ -205,6 +206,8 @@ OutputRegisterTxSubModule(const char *parent_name, const char *name, module->InitSubFunc = InitFunc; module->TxLogFunc = TxLogFunc; module->alproto = alproto; + module->tc_log_progress = tc_log_progress; + module->ts_log_progress = ts_log_progress; TAILQ_INSERT_TAIL(&output_modules, module, entries); SCLogDebug("Tx logger \"%s\" registered.", name); @@ -214,6 +217,34 @@ error: exit(EXIT_FAILURE); } +/** + * \brief Register a tx output module. + * + * This function will register an output module so it can be + * configured with the configuration file. + * + * \retval Returns 0 on success, -1 on failure. + */ +void +OutputRegisterTxModule(const char *name, const char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto, + TxLogger TxLogFunc) +{ + /* wrapper function */ + OutputRegisterTxModuleWithProgress(name, conf_name, InitFunc, alproto, + TxLogFunc, 0, 0); +} + +void +OutputRegisterTxSubModule(const char *parent_name, const char *name, + const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), + AppProto alproto, TxLogger TxLogFunc) +{ + /* wrapper function */ + OutputRegisterTxSubModuleWithProgress(parent_name, name, conf_name, + InitFunc, alproto, TxLogFunc, 0, 0); +} + /** * \brief Register a file output module. * diff --git a/src/output.h b/src/output.h index 9a4add0efb..152262b72a 100644 --- a/src/output.h +++ b/src/output.h @@ -55,6 +55,8 @@ typedef struct OutputModule_ { StatsLogger StatsLogFunc; AppProto alproto; enum OutputStreamingType stream_type; + int tc_log_progress; + int ts_log_progress; TAILQ_ENTRY(OutputModule_) entries; } OutputModule; @@ -75,6 +77,13 @@ void OutputRegisterTxSubModule(const char *parent_name, const char *name, const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), AppProto alproto, TxLogger TxLogFunc); +void OutputRegisterTxModuleWithProgress(const char *name, const char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto, + TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress); +void OutputRegisterTxSubModuleWithProgress(const char *parent_name, const char *name, + const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx), + AppProto alproto, TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress); + void OutputRegisterFileModule(const char *name, const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc); void OutputRegisterFileSubModule(const char *parent_name, const char *name, diff --git a/src/runmodes.c b/src/runmodes.c index b4bf31b6c7..e0154a1f55 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -617,7 +617,8 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu } else if (module->TxLogFunc) { SCLogDebug("%s is a tx logger", module->name); OutputRegisterTxLogger(module->name, module->alproto, - module->TxLogFunc, output_ctx); + module->TxLogFunc, output_ctx, module->tc_log_progress, + module->ts_log_progress); /* need one instance of the tx logger module */ if (tx_logger_module == NULL) {