diff --git a/src/Makefile.am b/src/Makefile.am index d788afdc57..6fac36e750 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -215,6 +215,7 @@ log-file.c log-file.h \ log-filestore.c log-filestore.h \ log-httplog.c log-httplog.h \ log-pcap.c log-pcap.h \ +log-stats.c log-stats.h \ log-tcp-data.c log-tcp-data.h \ log-tlslog.c log-tlslog.h \ output.c output.h \ @@ -234,6 +235,7 @@ output-json-ssh.c output-json-ssh.h \ output-json-tls.c output-json-tls.h \ output-lua.c output-lua.h \ output-packet.c output-packet.h \ +output-stats.c output-stats.h \ output-streaming.c output-streaming.h \ output-tx.c output-tx.h \ output-json.c output-json.h \ diff --git a/src/counters.c b/src/counters.c index 9aae1fe52d..5ea17c55e2 100644 --- a/src/counters.c +++ b/src/counters.c @@ -19,6 +19,7 @@ * \file * * \author Anoop Saldanha + * \author Victor Julien * * Performance counters */ @@ -43,14 +44,33 @@ /* Used to parse the interval for Timebased counters */ #define SC_PERF_PCRE_TIMEBASED_INTERVAL "^(?:(\\d+)([shm]))(?:(\\d+)([shm]))?(?:(\\d+)([shm]))?$" +/* Time interval for syncing the local counters with the global ones */ +#define SC_PERF_WUT_TTS 3 + +/* Time interval at which the mgmt thread o/p the stats */ +#define SC_PERF_MGMTT_TTS 8 + +static void *stats_thread_data = NULL; static SCPerfOPIfaceContext *sc_perf_op_ctx = NULL; static time_t sc_start_time; /** refresh interval in seconds */ static uint32_t sc_counter_tts = SC_PERF_MGMTT_TTS; /** is the stats counter enabled? */ static char sc_counter_enabled = TRUE; -/** append or overwrite? 1: append, 0: overwrite */ -static char sc_counter_append = TRUE; + +static int SCPerfOutputCounterFileIface(ThreadVars *tv); + +/** stats table is filled each interval and passed to the + * loggers. Initialized at first use. */ +static StatsTable stats_table = { NULL, 0, 0, {0 , 0}}; + +/** + * \brief The output interface dispatcher for the counter api + */ +void SCPerfOutputCounters(ThreadVars *tv) +{ + SCPerfOutputCounterFileIface(tv); +} /** * \brief Adds a value of type uint64_t to the local counter. @@ -126,65 +146,6 @@ void SCPerfCounterSetUI64(uint16_t id, SCPerfCounterArray *pca, return; } -/** - * \brief Get the filename with path to the stats log file. - * - * This function returns a string containing the log filename. It uses - * allocated memory simply to drop into the existing code a little better - * where a SCStrdup was used. So as before, it is up to the caller to free - * the memory. - * - * \retval An allocated string containing the log filename on success or NULL on - * failure. - */ -static char *SCPerfGetLogFilename(ConfNode *stats) -{ - char *log_dir = NULL; - char *log_filename = NULL; - const char* filename = NULL; - - log_dir = ConfigGetLogDirectory(); - - if ( (log_filename = SCMalloc(PATH_MAX)) == NULL) { - return NULL; - } - - if (stats != NULL) { - filename = ConfNodeLookupChildValue(stats, "filename"); - if (filename == NULL) { - filename = SC_PERF_DEFAULT_LOG_FILENAME; - } - } else { - filename = SC_PERF_DEFAULT_LOG_FILENAME; - } - - if (snprintf(log_filename, PATH_MAX, "%s/%s", log_dir, - filename) < 0) { - SCLogError(SC_ERR_SPRINTF, "Sprintf Error"); - SCFree(log_filename); - return NULL; - } - - return log_filename; -} - -/** - * \brief Reopen the log file. - * - * \retval 1 if successful, otherwise 0. - */ -static int SCPerfFileReopen(SCPerfOPIfaceContext *sc_perf_op_ctx) -{ - fclose(sc_perf_op_ctx->fp); - if ((sc_perf_op_ctx->fp = fopen(sc_perf_op_ctx->file, "w+")) == NULL) { - SCLogError(SC_ERR_FOPEN, "Failed to reopen file \"%s\"." - "Stats logging will now be disabled.", - sc_perf_op_ctx->file); - return 0; - } - return 1; -} - /** * \brief Initializes the output interface context * @@ -193,73 +154,14 @@ static int SCPerfFileReopen(SCPerfOPIfaceContext *sc_perf_op_ctx) static void SCPerfInitOPCtx(void) { SCEnter(); - - ConfNode *root = ConfGetNode("outputs"); - ConfNode *node = NULL; - ConfNode *stats = NULL; - if (root != NULL) { - TAILQ_FOREACH(node, &root->head, next) { - if (strncmp(node->val, "stats", 5) == 0) { - stats = node->head.tqh_first; - } - } - } - /* Check if the stats module is enabled or not */ - if (stats != NULL) { - const char *enabled = ConfNodeLookupChildValue(stats, "enabled"); - if (enabled != NULL && ConfValIsFalse(enabled)) { - sc_counter_enabled = FALSE; - SCLogDebug("Stats module has been disabled"); - SCReturn; - } - const char *interval = ConfNodeLookupChildValue(stats, "interval"); - if (interval != NULL) - sc_counter_tts = (uint32_t) atoi(interval); - - const char *append = ConfNodeLookupChildValue(stats, "append"); - if (append != NULL) - sc_counter_append = ConfValIsTrue(append); - } - - /* Store the engine start time */ - time(&sc_start_time); - if ( (sc_perf_op_ctx = SCMalloc(sizeof(SCPerfOPIfaceContext))) == NULL) { SCLogError(SC_ERR_FATAL, "Fatal error encountered in SCPerfInitOPCtx. Exiting..."); exit(EXIT_FAILURE); } memset(sc_perf_op_ctx, 0, sizeof(SCPerfOPIfaceContext)); - sc_perf_op_ctx->iface = SC_PERF_IFACE_FILE; - - if ( (sc_perf_op_ctx->file = SCPerfGetLogFilename(stats)) == NULL) { - SCLogInfo("Error retrieving Perf Counter API output file path"); - } - - char *mode; - if (sc_counter_append) - mode = "a+"; - else - mode = "w+"; - - if ( (sc_perf_op_ctx->fp = fopen(sc_perf_op_ctx->file, mode)) == NULL) { - SCLogError(SC_ERR_FOPEN, "fopen error opening file \"%s\". Resorting " - "to using the standard output for output", - sc_perf_op_ctx->file); - - SCFree(sc_perf_op_ctx->file); - - /* Let us use the standard output for output */ - sc_perf_op_ctx->fp = stdout; - if ( (sc_perf_op_ctx->file = SCStrdup("stdout")) == NULL) { - SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory"); - exit(EXIT_FAILURE); - } - } - else { - /* File opened, register for rotation notification. */ - OutputRegisterFileRotationFlag(&sc_perf_op_ctx->rotation_flag); - } + /* Store the engine start time */ + time(&sc_start_time); /* init the lock used by SCPerfClubTMInst */ if (SCMutexInit(&sc_perf_op_ctx->pctmi_lock, NULL) != 0) { @@ -285,14 +187,6 @@ static void SCPerfReleaseOPCtx() SCPerfClubTMInst *temp = NULL; pctmi = sc_perf_op_ctx->pctmi; - OutputUnregisterFileRotationFlag(&sc_perf_op_ctx->rotation_flag); - - if (sc_perf_op_ctx->fp != NULL) - fclose(sc_perf_op_ctx->fp); - - if (sc_perf_op_ctx->file != NULL) - SCFree(sc_perf_op_ctx->file); - while (pctmi != NULL) { if (pctmi->tm_name != NULL) SCFree(pctmi->tm_name); @@ -308,6 +202,12 @@ static void SCPerfReleaseOPCtx() SCFree(sc_perf_op_ctx); sc_perf_op_ctx = NULL; + /* free stats table */ + if (stats_table.stats != NULL) { + SCFree(stats_table.stats); + memset(&stats_table, 0, sizeof(stats_table)); + } + return; } @@ -349,6 +249,17 @@ static void *SCPerfMgmtThread(void *arg) return NULL; } + TmModule *tm = &tmm_modules[TMM_STATSLOGGER]; + BUG_ON(tm->ThreadInit == NULL); + int r = tm->ThreadInit(tv_local, NULL, &stats_thread_data); + if (r != 0 || stats_thread_data == NULL) { + SCLogError(SC_ERR_THREAD_INIT, "Perf Counter API " + "ThreadInit failed"); + TmThreadsSetFlag(tv_local, THV_CLOSED | THV_RUNNING_DONE); + return NULL; + } + SCLogDebug("stats_thread_data %p", &stats_thread_data); + TmThreadsSetFlag(tv_local, THV_INIT_DONE); while (run) { if (TmThreadsCheckFlag(tv_local, THV_PAUSE)) { @@ -364,7 +275,7 @@ static void *SCPerfMgmtThread(void *arg) SCCtrlCondTimedwait(tv_local->ctrl_cond, tv_local->ctrl_mutex, &cond_time); SCCtrlMutexUnlock(tv_local->ctrl_mutex); - SCPerfOutputCounters(); + SCPerfOutputCounters(tv_local); if (TmThreadsCheckFlag(tv_local, THV_KILL)) { run = 0; @@ -633,68 +544,91 @@ static uint64_t SCPerfOutputCalculateCounterValue(SCPerfCounter *pc) return pc->value; } + /** * \brief The file output interface for the Perf Counter api */ -static int SCPerfOutputCounterFileIface() +static int SCPerfOutputCounterFileIface(ThreadVars *tv) { - SCPerfClubTMInst *pctmi = NULL; - SCPerfCounter *pc = NULL; + const SCPerfClubTMInst *pctmi = NULL; + const SCPerfCounter *pc = NULL; SCPerfCounter **pc_heads = NULL; uint64_t ui64_temp = 0; uint64_t ui64_result = 0; - struct timeval tval; - struct tm *tms; - uint32_t u = 0; int flag = 0; + void *td = stats_thread_data; - if (sc_perf_op_ctx->fp == NULL) { - SCLogDebug("perf_op_ctx->fp is NULL"); - return 0; - } + if (stats_table.nstats == 0) { + uint32_t nstats = 0; - if (sc_perf_op_ctx->rotation_flag) { - SCLogDebug("Rotating log file"); - sc_perf_op_ctx->rotation_flag = 0; - if (!SCPerfFileReopen(sc_perf_op_ctx)) { - /* Rotation failed, error already logged. */ - return 0; + pctmi = sc_perf_op_ctx->pctmi; + while (pctmi != NULL) { + if (pctmi->size == 0) { + pctmi = pctmi->next; + continue; + } + + if ((pc_heads = SCMalloc(pctmi->size * sizeof(SCPerfCounter *))) == NULL) + return 0; + memset(pc_heads, 0, pctmi->size * sizeof(SCPerfCounter *)); + + for (u = 0; u < pctmi->size; u++) { + pc_heads[u] = pctmi->head[u]->head; + SCMutexLock(&pctmi->head[u]->m); + } + + flag = 1; + while (flag) { + if (pc_heads[0] == NULL) + break; + + for (u = 0; u < pctmi->size; u++) { + if (pc_heads[u] != NULL) + pc_heads[u] = pc_heads[u]->next; + if (pc_heads[u] == NULL) + flag = 0; + } + + /* count */ + nstats++; + } + + for (u = 0; u < pctmi->size; u++) + SCMutexUnlock(&pctmi->head[u]->m); + + pctmi = pctmi->next; + SCFree(pc_heads); + + } + if (nstats == 0) { + SCLogError(SC_ERR_PERF_STATS_NOT_INIT, "no counters registered"); + return -1; + } + + stats_table.nstats = nstats; + stats_table.stats = SCCalloc(stats_table.nstats, sizeof(StatsRecord)); + if (stats_table.stats == NULL) { + stats_table.nstats = 0; + SCLogError(SC_ERR_MEM_ALLOC, "could not alloc memory for stats"); + return -1; } + + stats_table.start_time = sc_start_time; } + StatsRecord *table = stats_table.stats; - memset(&tval, 0, sizeof(struct timeval)); - - gettimeofday(&tval, NULL); - struct tm local_tm; - tms = SCLocalTime(tval.tv_sec, &local_tm); - - /* Calculate the Engine uptime */ - int up_time = (int)difftime(tval.tv_sec, sc_start_time); - int sec = up_time % 60; // Seconds in a minute - int in_min = up_time / 60; - int min = in_min % 60; // Minutes in a hour - int in_hours = in_min / 60; - int hours = in_hours % 24; // Hours in a day - int days = in_hours / 24; - - fprintf(sc_perf_op_ctx->fp, "----------------------------------------------" - "---------------------\n"); - fprintf(sc_perf_op_ctx->fp, "Date: %" PRId32 "/%" PRId32 "/%04d -- " - "%02d:%02d:%02d (uptime: %"PRId32"d, %02dh %02dm %02ds)\n", - tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900, tms->tm_hour, - tms->tm_min, tms->tm_sec, days, hours, min, sec); - fprintf(sc_perf_op_ctx->fp, "----------------------------------------------" - "---------------------\n"); - fprintf(sc_perf_op_ctx->fp, "%-25s | %-25s | %-s\n", "Counter", "TM Name", - "Value"); - fprintf(sc_perf_op_ctx->fp, "----------------------------------------------" - "---------------------\n"); + int table_i = 0; pctmi = sc_perf_op_ctx->pctmi; while (pctmi != NULL) { + if (pctmi->size == 0) { + pctmi = pctmi->next; + continue; + } + if ((pc_heads = SCMalloc(pctmi->size * sizeof(SCPerfCounter *))) == NULL) return 0; memset(pc_heads, 0, pctmi->size * sizeof(SCPerfCounter *)); @@ -722,20 +656,24 @@ static int SCPerfOutputCounterFileIface() flag = 0; } - fprintf(sc_perf_op_ctx->fp, "%-25s | %-25s | %-" PRIu64 "\n", - pc->cname, pctmi->tm_name, ui64_result); + /* store in the table */ + table[table_i].name = pc->cname; + table[table_i].tm_name = pctmi->tm_name; + table[table_i].pvalue = table[table_i].value; + table[table_i].value = ui64_result; + table_i++; } for (u = 0; u < pctmi->size; u++) SCMutexUnlock(&pctmi->head[u]->m); pctmi = pctmi->next; - SCFree(pc_heads); - fflush(sc_perf_op_ctx->fp); } + /* invoke logger(s) */ + OutputStatsLog(tv, td, &stats_table); return 1; } @@ -1294,29 +1232,6 @@ double SCPerfGetLocalCounterValue(uint16_t id, SCPerfCounterArray *pca) return pca->head[id].ui64_cnt; } -/** - * \brief The output interface dispatcher for the counter api - */ -void SCPerfOutputCounters() -{ - switch (sc_perf_op_ctx->iface) { - case SC_PERF_IFACE_FILE: - SCPerfOutputCounterFileIface(); - - break; - case SC_PERF_IFACE_CONSOLE: - /* yet to be implemented */ - - break; - case SC_PERF_IFACE_SYSLOG: - /* yet to be implemented */ - - break; - } - - return; -} - /** * \brief Releases the resources alloted by the Perf Counter API */ diff --git a/src/counters.h b/src/counters.h index 123a34f723..a1617d4037 100644 --- a/src/counters.h +++ b/src/counters.h @@ -27,12 +27,6 @@ /* forward declaration of the ThreadVars structure */ struct ThreadVars_; -/* Time interval for syncing the local counters with the global ones */ -#define SC_PERF_WUT_TTS 3 - -/* Time interval at which the mgmt thread o/p the stats */ -#define SC_PERF_MGMTT_TTS 8 - /** * \brief Data type for different kind of Perf counters that can be registered */ @@ -147,21 +141,8 @@ typedef struct SCPerfClubTMInst_ { * \brief Holds the output interface context for the counter api */ typedef struct SCPerfOPIfaceContext_ { - /* the iface to be used for output */ - uint32_t iface; - - /* the file to be used if the output interface used is SC_PERF_IFACE_FILE */ - char *file; - - /* more interfaces to be supported later. For now just a file */ - FILE *fp; - SCPerfClubTMInst *pctmi; SCMutex pctmi_lock; - - /* Flag set on file rotation notification. */ - int rotation_flag; - } SCPerfOPIfaceContext; /* the initialization functions */ @@ -186,8 +167,6 @@ SCPerfCounterArray * SCPerfGetAllCountersArray(SCPerfContext *); int SCPerfUpdateCounterArray(SCPerfCounterArray *, SCPerfContext *); double SCPerfGetLocalCounterValue(uint16_t, SCPerfCounterArray *); -void SCPerfOutputCounters(void); - /* functions used to free the resources alloted by the Perf counter API */ void SCPerfReleaseResources(void); void SCPerfReleasePerfCounterS(SCPerfCounter *); diff --git a/src/log-stats.c b/src/log-stats.c new file mode 100644 index 0000000000..fafd4c2852 --- /dev/null +++ b/src/log-stats.c @@ -0,0 +1,231 @@ +/* Copyright (C) 2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + */ + +#include "suricata-common.h" +#include "debug.h" +#include "detect.h" +#include "pkt-var.h" +#include "conf.h" + +#include "threads.h" +#include "threadvars.h" +#include "tm-threads.h" + +#include "util-print.h" +#include "util-unittest.h" + +#include "util-debug.h" + +#include "output.h" +#include "log-stats.h" +#include "util-privs.h" +#include "util-buffer.h" + +#include "util-logopenfile.h" +#include "util-time.h" + +#define DEFAULT_LOG_FILENAME "stats.log" +#define MODULE_NAME "LogStatsLog" +#define OUTPUT_BUFFER_SIZE 65535 + +TmEcode LogStatsLogThreadInit(ThreadVars *, void *, void **); +TmEcode LogStatsLogThreadDeinit(ThreadVars *, void *); +void LogStatsLogExitPrintStats(ThreadVars *, void *); +static void LogStatsLogDeInitCtx(OutputCtx *); + +typedef struct LogStatsFileCtx_ { + LogFileCtx *file_ctx; + uint32_t flags; /** Store mode */ +} LogStatsFileCtx; + +typedef struct LogStatsLogThread_ { + LogStatsFileCtx *statslog_ctx; + MemBuffer *buffer; +} LogStatsLogThread; + +int LogStatsLogger(ThreadVars *tv, void *thread_data, const StatsTable *st) +{ + SCEnter(); + LogStatsLogThread *aft = (LogStatsLogThread *)thread_data; + + struct timeval tval; + struct tm *tms; + + gettimeofday(&tval, NULL); + struct tm local_tm; + tms = SCLocalTime(tval.tv_sec, &local_tm); + + /* Calculate the Engine uptime */ + int up_time = (int)difftime(tval.tv_sec, st->start_time); + int sec = up_time % 60; // Seconds in a minute + int in_min = up_time / 60; + int min = in_min % 60; // Minutes in a hour + int in_hours = in_min / 60; + int hours = in_hours % 24; // Hours in a day + int days = in_hours / 24; + + MemBufferWriteString(aft->buffer, "----------------------------------------------" + "---------------------\n"); + MemBufferWriteString(aft->buffer, "Date: %" PRId32 "/%" PRId32 "/%04d -- " + "%02d:%02d:%02d (uptime: %"PRId32"d, %02dh %02dm %02ds)\n", + tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900, tms->tm_hour, + tms->tm_min, tms->tm_sec, days, hours, min, sec); + MemBufferWriteString(aft->buffer, "----------------------------------------------" + "---------------------\n"); + MemBufferWriteString(aft->buffer, "%-25s | %-25s | %-s\n", "Counter", "TM Name", + "Value"); + MemBufferWriteString(aft->buffer, "----------------------------------------------" + "---------------------\n"); + + uint32_t u = 0; + for (u = 0; u < st->nstats; u++) { + if (st->stats[u].name == NULL) + break; + MemBufferWriteString(aft->buffer, "%-25s | %-25s | %-" PRIu64 "\n", + st->stats[u].name, st->stats[u].tm_name, st->stats[u].value); + } + + SCMutexLock(&aft->statslog_ctx->file_ctx->fp_mutex); + aft->statslog_ctx->file_ctx->Write((const char *)MEMBUFFER_BUFFER(aft->buffer), + MEMBUFFER_OFFSET(aft->buffer), aft->statslog_ctx->file_ctx); + SCMutexUnlock(&aft->statslog_ctx->file_ctx->fp_mutex); + + MemBufferReset(aft->buffer); + + SCReturnInt(0); +} + +TmEcode LogStatsLogThreadInit(ThreadVars *t, void *initdata, void **data) +{ + LogStatsLogThread *aft = SCMalloc(sizeof(LogStatsLogThread)); + if (unlikely(aft == NULL)) + return TM_ECODE_FAILED; + memset(aft, 0, sizeof(LogStatsLogThread)); + + if(initdata == NULL) + { + SCLogDebug("Error getting context for HTTPLog. \"initdata\" argument NULL"); + SCFree(aft); + return TM_ECODE_FAILED; + } + + aft->buffer = MemBufferCreateNew(OUTPUT_BUFFER_SIZE); + if (aft->buffer == NULL) { + SCFree(aft); + return TM_ECODE_FAILED; + } + + /* Use the Ouptut Context (file pointer and mutex) */ + aft->statslog_ctx= ((OutputCtx *)initdata)->data; + + *data = (void *)aft; + return TM_ECODE_OK; +} + +TmEcode LogStatsLogThreadDeinit(ThreadVars *t, void *data) +{ + LogStatsLogThread *aft = (LogStatsLogThread *)data; + if (aft == NULL) { + return TM_ECODE_OK; + } + + MemBufferFree(aft->buffer); + /* clear memory */ + memset(aft, 0, sizeof(LogStatsLogThread)); + + SCFree(aft); + return TM_ECODE_OK; +} + +void LogStatsLogExitPrintStats(ThreadVars *tv, void *data) +{ + LogStatsLogThread *aft = (LogStatsLogThread *)data; + if (aft == NULL) { + return; + } +} + +/** \brief Create a new http log LogFileCtx. + * \param conf Pointer to ConfNode containing this loggers configuration. + * \return NULL if failure, LogFileCtx* to the file_ctx if succesful + * */ +OutputCtx *LogStatsLogInitCtx(ConfNode *conf) +{ + LogFileCtx *file_ctx = LogFileNewCtx(); + if (file_ctx == NULL) { + SCLogError(SC_ERR_HTTP_LOG_GENERIC, "couldn't create new file_ctx"); + return NULL; + } + + if (SCConfLogOpenGeneric(conf, file_ctx, DEFAULT_LOG_FILENAME) < 0) { + LogFileFreeCtx(file_ctx); + return NULL; + } + + LogStatsFileCtx *statslog_ctx = SCMalloc(sizeof(LogStatsFileCtx)); + if (unlikely(statslog_ctx == NULL)) { + LogFileFreeCtx(file_ctx); + return NULL; + } + memset(statslog_ctx, 0x00, sizeof(LogStatsFileCtx)); + + statslog_ctx->file_ctx = file_ctx; + + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); + if (unlikely(output_ctx == NULL)) { + LogFileFreeCtx(file_ctx); + SCFree(statslog_ctx); + return NULL; + } + + output_ctx->data = statslog_ctx; + output_ctx->DeInit = LogStatsLogDeInitCtx; + + SCLogDebug("STATS log output initialized"); + + OutputRegisterFileRotationFlag(&file_ctx->rotation_flag); + return output_ctx; +} + +static void LogStatsLogDeInitCtx(OutputCtx *output_ctx) +{ + LogStatsFileCtx *statslog_ctx = (LogStatsFileCtx *)output_ctx->data; + OutputUnregisterFileRotationFlag(&statslog_ctx->file_ctx->rotation_flag); + LogFileFreeCtx(statslog_ctx->file_ctx); + SCFree(statslog_ctx); + SCFree(output_ctx); +} + +void TmModuleLogStatsLogRegister (void) +{ + tmm_modules[TMM_LOGSTATSLOG].name = MODULE_NAME; + tmm_modules[TMM_LOGSTATSLOG].ThreadInit = LogStatsLogThreadInit; + tmm_modules[TMM_LOGSTATSLOG].ThreadExitPrintStats = LogStatsLogExitPrintStats; + tmm_modules[TMM_LOGSTATSLOG].ThreadDeinit = LogStatsLogThreadDeinit; + tmm_modules[TMM_LOGSTATSLOG].RegisterTests = NULL; + tmm_modules[TMM_LOGSTATSLOG].cap_flags = 0; + tmm_modules[TMM_LOGSTATSLOG].flags = TM_FLAG_LOGAPI_TM; + + OutputRegisterStatsModule(MODULE_NAME, "stats", LogStatsLogInitCtx, LogStatsLogger); +} diff --git a/src/log-stats.h b/src/log-stats.h new file mode 100644 index 0000000000..957104f3c7 --- /dev/null +++ b/src/log-stats.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __LOG_STATS_H__ +#define __LOG_STATS_H__ + +void TmModuleLogStatsLogRegister (void); + +#endif /* __LOG_HTTPLOG_H__ */ diff --git a/src/output-stats.c b/src/output-stats.c new file mode 100644 index 0000000000..df6fa1b09b --- /dev/null +++ b/src/output-stats.c @@ -0,0 +1,234 @@ +/* Copyright (C) 2014 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * Stats Logger Output registration functions + */ + +#include "suricata-common.h" +#include "tm-modules.h" +#include "output-stats.h" + +typedef struct OutputLoggerThreadStore_ { + void *thread_data; + struct OutputLoggerThreadStore_ *next; +} OutputLoggerThreadStore; + +/** per thread data for this module, contains a list of per thread + * data for the packet loggers. */ +typedef struct OutputLoggerThreadData_ { + OutputLoggerThreadStore *store; +} OutputLoggerThreadData; + +/* logger instance, a module + a output ctx, + * it's perfectly valid that have multiple instances of the same + * log module (e.g. http.log) with different output ctx'. */ +typedef struct OutputStatsLogger_ { + StatsLogger LogFunc; + OutputCtx *output_ctx; + struct OutputStatsLogger_ *next; + const char *name; + TmmId module_id; +} OutputStatsLogger; + +static OutputStatsLogger *list = NULL; + +int OutputRegisterStatsLogger(const char *name, StatsLogger LogFunc, OutputCtx *output_ctx) +{ + int module_id = TmModuleGetIdByName(name); + if (module_id < 0) + return -1; + + OutputStatsLogger *op = SCMalloc(sizeof(*op)); + if (op == NULL) + return -1; + memset(op, 0x00, sizeof(*op)); + + op->LogFunc = LogFunc; + op->output_ctx = output_ctx; + op->name = name; + op->module_id = (TmmId) module_id; + + if (list == NULL) + list = op; + else { + OutputStatsLogger *t = list; + while (t->next) + t = t->next; + t->next = op; + } + + SCLogDebug("OutputRegisterStatsLogger happy"); + return 0; +} + +TmEcode OutputStatsLog(ThreadVars *tv, void *thread_data, StatsTable *st) +{ + BUG_ON(thread_data == NULL); + BUG_ON(list == NULL); + + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputStatsLogger *logger = list; + OutputLoggerThreadStore *store = op_thread_data->store; + + BUG_ON(logger == NULL && store != NULL); + BUG_ON(logger != NULL && store == NULL); + BUG_ON(logger == NULL && store == NULL); + + logger = list; + store = op_thread_data->store; + while (logger && store) { + BUG_ON(logger->LogFunc == NULL); + + logger->LogFunc(tv, store->thread_data, st); + + logger = logger->next; + store = store->next; + + BUG_ON(logger == NULL && store != NULL); + BUG_ON(logger != NULL && store == NULL); + } + + return TM_ECODE_OK; +} + +/** \brief thread init for the tx logger + * This will run the thread init functions for the individual registered + * loggers */ +static TmEcode OutputStatsLogThreadInit(ThreadVars *tv, void *initdata, void **data) +{ + OutputLoggerThreadData *td = SCMalloc(sizeof(*td)); + if (td == NULL) + return TM_ECODE_FAILED; + memset(td, 0x00, sizeof(*td)); + + *data = (void *)td; + + SCLogDebug("OutputStatsLogThreadInit happy (*data %p)", *data); + + OutputStatsLogger *logger = list; + while (logger) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadInit) { + void *retptr = NULL; + if (tm_module->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) { + OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts)); +/* todo */ BUG_ON(ts == NULL); + memset(ts, 0x00, sizeof(*ts)); + + /* store thread handle */ + ts->thread_data = retptr; + + if (td->store == NULL) { + td->store = ts; + } else { + OutputLoggerThreadStore *tmp = td->store; + while (tmp->next != NULL) + tmp = tmp->next; + tmp->next = ts; + } + + SCLogDebug("%s is now set up", logger->name); + } + } + + logger = logger->next; + } + + SCLogDebug("OutputStatsLogThreadInit happy (*data %p)", *data); + return TM_ECODE_OK; +} + +static TmEcode OutputStatsLogThreadDeinit(ThreadVars *tv, void *thread_data) +{ + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputLoggerThreadStore *store = op_thread_data->store; + OutputStatsLogger *logger = list; + + while (logger && store) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadDeinit) { + tm_module->ThreadDeinit(tv, store->thread_data); + } + + OutputLoggerThreadStore *next_store = store->next; + SCFree(store); + store = next_store; + logger = logger->next; + } + return TM_ECODE_OK; +} + +static void OutputStatsLogExitPrintStats(ThreadVars *tv, void *thread_data) +{ + OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data; + OutputLoggerThreadStore *store = op_thread_data->store; + OutputStatsLogger *logger = list; + + while (logger && store) { + TmModule *tm_module = TmModuleGetByName((char *)logger->name); + if (tm_module == NULL) { + SCLogError(SC_ERR_INVALID_ARGUMENT, + "TmModuleGetByName for %s failed", logger->name); + exit(EXIT_FAILURE); + } + + if (tm_module->ThreadExitPrintStats) { + tm_module->ThreadExitPrintStats(tv, store->thread_data); + } + + logger = logger->next; + store = store->next; + } +} + +void TmModuleStatsLoggerRegister (void) +{ + tmm_modules[TMM_STATSLOGGER].name = "__stats_logger__"; + tmm_modules[TMM_STATSLOGGER].ThreadInit = OutputStatsLogThreadInit; + //tmm_modules[TMM_STATSLOGGER].Func = OutputStatsLog; + tmm_modules[TMM_STATSLOGGER].ThreadExitPrintStats = OutputStatsLogExitPrintStats; + tmm_modules[TMM_STATSLOGGER].ThreadDeinit = OutputStatsLogThreadDeinit; + tmm_modules[TMM_STATSLOGGER].cap_flags = 0; +} + +void OutputStatsShutdown(void) +{ + OutputStatsLogger *logger = list; + while (logger) { + OutputStatsLogger *next_logger = logger->next; + SCFree(logger); + logger = next_logger; + } + list = NULL; +} diff --git a/src/output-stats.h b/src/output-stats.h new file mode 100644 index 0000000000..adb3d11761 --- /dev/null +++ b/src/output-stats.h @@ -0,0 +1,53 @@ +/* Copyright (C) 2007-2013 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * Stats Logger Output registration functions + */ + +#ifndef __OUTPUT_STATS_H__ +#define __OUTPUT_STATS_H__ + +typedef struct StatsRecord_ { + const char *name; + const char *tm_name; + uint64_t value; /**< total value */ + uint64_t pvalue; /**< prev value (may be higher for memuse counters) */ +} StatsRecord; + +typedef struct StatsTable_ { + StatsRecord *stats; + uint32_t nstats; + time_t start_time; + struct timeval ts; +} StatsTable; + +TmEcode OutputStatsLog(ThreadVars *tv, void *thread_data, StatsTable *st); + +typedef int (*StatsLogger)(ThreadVars *, void *thread_data, const StatsTable *); + +int OutputRegisterStatsLogger(const char *name, StatsLogger LogFunc, OutputCtx *); + +void TmModuleStatsLoggerRegister (void); + +void OutputStatsShutdown(void); + +#endif /* __OUTPUT_STATS_H__ */ diff --git a/src/output.c b/src/output.c index 9229eb17b4..1146b7b226 100644 --- a/src/output.c +++ b/src/output.c @@ -497,6 +497,76 @@ error: exit(EXIT_FAILURE); } +/** + * \brief Register a stats data output module. + * + * This function will register an output module so it can be + * configured with the configuration file. + * + * \retval Returns 0 on success, -1 on failure. + */ +void +OutputRegisterStatsModule(const char *name, const char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), StatsLogger StatsLogFunc) +{ + if (unlikely(StatsLogFunc == NULL)) { + goto error; + } + + OutputModule *module = SCCalloc(1, sizeof(*module)); + if (unlikely(module == NULL)) { + goto error; + } + + module->name = name; + module->conf_name = conf_name; + module->InitFunc = InitFunc; + module->StatsLogFunc = StatsLogFunc; + TAILQ_INSERT_TAIL(&output_modules, module, entries); + + SCLogDebug("Stats logger \"%s\" registered.", name); + return; +error: + SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting..."); + exit(EXIT_FAILURE); +} + +/** + * \brief Register a stats data output sub-module. + * + * This function will register an output module so it can be + * configured with the configuration file. + * + * \retval Returns 0 on success, -1 on failure. + */ +void +OutputRegisterStatsSubModule(const char *parent_name, const char *name, + const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), + StatsLogger StatsLogFunc) +{ + if (unlikely(StatsLogFunc == NULL)) { + goto error; + } + + OutputModule *module = SCCalloc(1, sizeof(*module)); + if (unlikely(module == NULL)) { + goto error; + } + + module->name = name; + module->conf_name = conf_name; + module->parent_name = parent_name; + module->InitSubFunc = InitFunc; + module->StatsLogFunc = StatsLogFunc; + TAILQ_INSERT_TAIL(&output_modules, module, entries); + + SCLogDebug("Stats logger \"%s\" registered.", name); + return; +error: + SCLogError(SC_ERR_FATAL, "Fatal error encountered. Exiting..."); + exit(EXIT_FAILURE); +} + /** * \brief Get an output module by name. * diff --git a/src/output.h b/src/output.h index 85092c3141..9a4add0efb 100644 --- a/src/output.h +++ b/src/output.h @@ -36,6 +36,7 @@ #include "output-filedata.h" #include "output-flow.h" #include "output-streaming.h" +#include "output-stats.h" typedef struct OutputModule_ { const char *name; @@ -51,6 +52,7 @@ typedef struct OutputModule_ { FiledataLogger FiledataLogFunc; FlowLogger FlowLogFunc; StreamingLogger StreamingLogFunc; + StatsLogger StatsLogFunc; AppProto alproto; enum OutputStreamingType stream_type; @@ -98,6 +100,12 @@ void OutputRegisterStreamingSubModule(const char *parent_name, const char *name, const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), StreamingLogger StreamingLogFunc, enum OutputStreamingType stream_type); +void OutputRegisterStatsModule(const char *name, const char *conf_name, + OutputCtx *(*InitFunc)(ConfNode *), StatsLogger StatsLogFunc); +void OutputRegisterStatsSubModule(const char *parent_name, const char *name, + const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), + StatsLogger StatsLogFunc); + OutputModule *OutputGetModuleByConfName(const char *name); void OutputDeregisterAll(void); diff --git a/src/runmodes.c b/src/runmodes.c index 847fadcb10..76f9022f86 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -447,6 +447,7 @@ void RunModeShutDown(void) OutputTxShutdown(); OutputFileShutdown(); OutputFiledataShutdown(); + OutputStatsShutdown(); /* Close any log files. */ RunModeOutput *output; @@ -490,6 +491,11 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu OutputRegisterFlowLogger(module->name, module->FlowLogFunc, output_ctx); return; } + /* stats logger doesn't run in the packet path */ + if (module->StatsLogFunc) { + OutputRegisterStatsLogger(module->name, module->StatsLogFunc, output_ctx); + return; + } TmModule *tm_module = TmModuleGetByName(module->name); if (tm_module == NULL) { @@ -636,9 +642,6 @@ void RunModeInitializeOutputs(void) TAILQ_FOREACH(output, &outputs->head, next) { - if (strcmp(output->val, "stats") == 0) - continue; - output_config = ConfNodeLookupChild(output, output->val); if (output_config == NULL) { /* Shouldn't happen. */ diff --git a/src/suricata.c b/src/suricata.c index d0c084a76c..e7053ab965 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -96,6 +96,7 @@ #include "output-json-smtp.h" #include "log-filestore.h" #include "log-tcp-data.h" +#include "log-stats.h" #include "output-json.h" @@ -892,6 +893,8 @@ void RegisterAllModules() TmModuleJsonDnsLogRegister(); /* tcp streaming data */ TmModuleLogTcpDataLogRegister(); + /* log stats */ + TmModuleLogStatsLogRegister(); TmModuleJsonAlertLogRegister(); /* flow/netflow */ @@ -904,6 +907,7 @@ void RegisterAllModules() TmModuleFileLoggerRegister(); TmModuleFiledataLoggerRegister(); TmModuleStreamingLoggerRegister(); + TmModuleStatsLoggerRegister(); TmModuleDebugList(); /* nflog */ TmModuleReceiveNFLOGRegister(); diff --git a/src/tm-modules.c b/src/tm-modules.c index 8503f02686..19caeca727 100644 --- a/src/tm-modules.c +++ b/src/tm-modules.c @@ -250,6 +250,7 @@ const char * TmModuleTmmIdToString(TmmId id) CASE_CODE (TMM_DECODEAFP); CASE_CODE (TMM_PACKETLOGGER); CASE_CODE (TMM_TXLOGGER); + CASE_CODE (TMM_STATSLOGGER); CASE_CODE (TMM_FILELOGGER); CASE_CODE (TMM_FILEDATALOGGER); CASE_CODE (TMM_STREAMINGLOGGER); @@ -267,6 +268,7 @@ const char * TmModuleTmmIdToString(TmmId id) CASE_CODE (TMM_FLOWMANAGER); CASE_CODE (TMM_FLOWRECYCLER); CASE_CODE (TMM_LUALOG); + CASE_CODE (TMM_LOGSTATSLOG); CASE_CODE (TMM_SIZE); } diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h index 0137e10279..059ba7b12d 100644 --- a/src/tm-threads-common.h +++ b/src/tm-threads-common.h @@ -81,6 +81,7 @@ typedef enum { TMM_DECODENAPATECH, TMM_PACKETLOGGER, TMM_TXLOGGER, + TMM_STATSLOGGER, TMM_FILELOGGER, TMM_FILEDATALOGGER, TMM_STREAMINGLOGGER, @@ -96,6 +97,7 @@ typedef enum { TMM_DECODENFLOG, TMM_JSONFLOWLOG, TMM_JSONNETFLOWLOG, + TMM_LOGSTATSLOG, TMM_FLOWMANAGER, TMM_FLOWRECYCLER,