diff --git a/src/Makefile.am b/src/Makefile.am index 7d6d0ebf0c..121df60ffd 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -343,7 +343,7 @@ noinst_HEADERS = \ ippair-storage.h \ ippair-timeout.h \ ippair.h \ - log-flush.h \ + log-maintenance.h \ log-pcap.h \ log-stats.h \ log-tcp-data.h \ @@ -921,7 +921,7 @@ libsuricata_c_a_SOURCES = \ ippair-storage.c \ ippair-timeout.c \ ippair.c \ - log-flush.c \ + log-maintenance.c \ log-pcap.c \ log-stats.c \ log-tcp-data.c \ diff --git a/src/log-flush.c b/src/log-maintenance.c similarity index 59% rename from src/log-flush.c rename to src/log-maintenance.c index 7022e1359b..170103754b 100644 --- a/src/log-flush.c +++ b/src/log-maintenance.c @@ -23,13 +23,12 @@ #include "suricata-common.h" #include "suricata.h" -#include "log-flush.h" +#include "log-maintenance.h" #include "util-logopenfile.h" #include "tm-threads.h" #include "conf.h" #include "conf-yaml-loader.h" #include "util-privs.h" -#include "util-logopenfile.h" int OutputFlushInterval(void) { @@ -45,20 +44,26 @@ int OutputFlushInterval(void) return (int)output_flush_interval; } -static void *LogFlusherWakeupThread(void *arg) +static void *LogMaintenanceThread(void *arg) { int output_flush_interval = OutputFlushInterval(); - /* This was checked by the logic creating this thread */ - BUG_ON(output_flush_interval == 0); - SCLogConfig("Using output-flush-interval of %d seconds", output_flush_interval); + if (output_flush_interval > 0) { + SCLogConfig("Log maintenance thread started: rotation check every 1s, flush interval %ds", + output_flush_interval); + } else { + SCLogConfig("Log maintenance thread started: rotation check every 1s, flush disabled"); + } + /* * Calculate the number of sleep intervals based on the output flush interval. This is necessary * because this thread pauses a fixed amount of time to react to shutdown situations more * quickly. */ - const int log_flush_sleep_time = 500; /* milliseconds */ - const int flush_wait_count = (1000 * output_flush_interval) / log_flush_sleep_time; + const int maintenance_sleep_time = 500; /* milliseconds */ + const int rotation_wait_count = 1000 / maintenance_sleep_time; /* = 2, check every 1 second */ + const int flush_wait_count = + output_flush_interval > 0 ? (1000 * output_flush_interval) / maintenance_sleep_time : 0; ThreadVars *tv_local = (ThreadVars *)arg; SCSetThreadName(tv_local->name); @@ -72,16 +77,26 @@ static void *LogFlusherWakeupThread(void *arg) TmThreadsSetFlag(tv_local, THV_INIT_DONE | THV_RUNNING); - int wait_count = 0; + int rotation_counter = 0; + int flush_counter = 0; + uint64_t rotation_check_count = 0; uint64_t worker_flush_count = 0; bool run = TmThreadsWaitForUnpause(tv_local); while (run) { - SleepMsec(log_flush_sleep_time); + SleepMsec(maintenance_sleep_time); - if (++wait_count == flush_wait_count) { + /* Check rotation every 1 second */ + if (++rotation_counter >= rotation_wait_count) { + rotation_check_count++; + LogFileRotateAll(); + rotation_counter = 0; + } + + /* Flush at configured interval (if enabled) */ + if (flush_wait_count > 0 && ++flush_counter >= flush_wait_count) { worker_flush_count++; LogFileFlushAll(); - wait_count = 0; + flush_counter = 0; } if (TmThreadsCheckFlag(tv_local, THV_KILL)) { @@ -92,20 +107,16 @@ static void *LogFlusherWakeupThread(void *arg) TmThreadsSetFlag(tv_local, THV_RUNNING_DONE); TmThreadWaitForFlag(tv_local, THV_DEINIT); TmThreadsSetFlag(tv_local, THV_CLOSED); - SCLogInfo("%s: initiated %" PRIu64 " flushes", tv_local->name, worker_flush_count); + SCLogInfo("%s: performed %" PRIu64 " rotation checks, %" PRIu64 " flushes", tv_local->name, + rotation_check_count, worker_flush_count); return NULL; } -void LogFlushThreads(void) +void LogMaintenanceThreadSpawn(void) { - if (0 == OutputFlushInterval()) { - SCLogConfig("log flusher thread not used with heartbeat.output-flush-interval of 0"); - return; - } - - ThreadVars *tv_log_flush = - TmThreadCreateMgmtThread(thread_name_heartbeat, LogFlusherWakeupThread, 1); - if (!tv_log_flush || (TmThreadSpawn(tv_log_flush) != 0)) { - FatalError("Unable to create and start log flush thread"); + ThreadVars *tv_maintenance = + TmThreadCreateMgmtThread(thread_name_heartbeat, LogMaintenanceThread, 1); + if (!tv_maintenance || (TmThreadSpawn(tv_maintenance) != 0)) { + FatalError("Unable to create and start log maintenance thread"); } } diff --git a/src/log-flush.h b/src/log-maintenance.h similarity index 77% rename from src/log-flush.h rename to src/log-maintenance.h index 8676e75745..61728d8d54 100644 --- a/src/log-flush.h +++ b/src/log-maintenance.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2024 Open Information Security Foundation +/* Copyright (C) 2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -20,8 +20,8 @@ * * \author Jeff Lucovsky */ -#ifndef SURICATA_LOG_FLUSH_H__ -#define SURICATA_LOG_FLUSH_H__ -void LogFlushThreads(void); +#ifndef SURICATA_LOG_MAINTENANCE_H__ +#define SURICATA_LOG_MAINTENANCE_H__ +void LogMaintenanceThreadSpawn(void); int OutputFlushInterval(void); -#endif /* SURICATA_LOG_FLUSH_H__ */ +#endif /* SURICATA_LOG_MAINTENANCE_H__ */ diff --git a/src/runmodes.c b/src/runmodes.c index ddf12557ea..c935b914ca 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2024 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -28,7 +28,7 @@ #include "util-debug.h" #include "util-affinity.h" #include "conf.h" -#include "log-flush.h" +#include "log-maintenance.h" #include "runmodes.h" #include "runmode-af-packet.h" #include "runmode-af-xdp.h" @@ -451,7 +451,7 @@ void RunModeDispatch(int runmode, const char *custom_mode, const char *capture_p BypassedFlowManagerThreadSpawn(); } StatsSpawnThreads(); - LogFlushThreads(); + LogMaintenanceThreadSpawn(); TmThreadsSealThreads(); } } diff --git a/src/util-logopenfile.c b/src/util-logopenfile.c index 90b42c3b7b..8386d455e9 100644 --- a/src/util-logopenfile.c +++ b/src/util-logopenfile.c @@ -33,7 +33,7 @@ #include "util-path.h" #include "util-misc.h" #include "util-time.h" -#include "log-flush.h" +#include "log-maintenance.h" #if defined(HAVE_SYS_UN_H) && defined(HAVE_SYS_SOCKET_H) && defined(HAVE_SYS_TYPES_H) #define BUILD_WITH_UNIXSOCKET @@ -54,10 +54,9 @@ static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, // Threaded eve.json identifier static SC_ATOMIC_DECL_AND_INIT_WITH_VAL(uint16_t, eve_file_id, 1); -/* Flush list for heartbeat-triggered flushing */ -static SCMutex log_file_flush_mutex = SCMUTEX_INITIALIZER; -static TAILQ_HEAD(, LogFileFlushEntry_) log_file_flush_list = TAILQ_HEAD_INITIALIZER( - log_file_flush_list); +/* Log file list for heartbeat-triggered flushing and rotation */ +static SCMutex log_file_list_mutex = SCMUTEX_INITIALIZER; +static TAILQ_HEAD(, LogFileEntry_) log_file_list = TAILQ_HEAD_INITIALIZER(log_file_list); #ifdef BUILD_WITH_UNIXSOCKET /** \brief connect to the indicated local stream socket, logging any errors @@ -275,6 +274,20 @@ static int SCLogFileWriteNoLock(const char *buffer, int buffer_len, LogFileCtx * return ret; } +/** + * \brief Check and perform log file rotation if needed. + * + * Called by the maintenance thread to drive rotation independently of + * traffic-driven writes. Takes the file lock so it is safe to run + * concurrently with worker threads writing to the same context. + */ +static void SCLogFileRotate(LogFileCtx *log_ctx) +{ + OutputWriteLock(&log_ctx->fp_mutex); + HandleLogRotation(log_ctx); + SCMutexUnlock(&log_ctx->fp_mutex); +} + /** * \brief Write buffer to log file. * \retval 0 on failure; otherwise, the return value of fwrite (number of @@ -652,9 +665,9 @@ int SCConfLogOpenGeneric( if (rotate) { OutputRegisterFileRotationFlag(&log_ctx->rotation_flag); } - /* Register non-threaded regular files for direct heartbeat flushing */ + /* Register non-threaded regular files for heartbeat maintenance */ if (!log_ctx->threaded && log_ctx->is_regular) { - LogFileRegisterForFlush(log_ctx); + LogFileRegister(log_ctx); } } else { SCLogError("Invalid entry for " @@ -730,6 +743,7 @@ LogFileCtx *LogFileNewCtx(void) lf_ctx->Write = SCLogFileWrite; lf_ctx->Close = SCLogFileClose; lf_ctx->Flush = SCLogFileFlush; + lf_ctx->Rotate = SCLogFileRotate; return lf_ctx; } @@ -898,15 +912,11 @@ static bool LogFileNewThreadedCtx(LogFileCtx *parent_ctx, const char *log_path, goto error; } thread->is_regular = true; - if (OutputFlushInterval() > 0) { - thread->Write = SCLogFileWrite; - thread->Close = SCLogFileClose; - } else { - thread->Write = SCLogFileWriteNoLock; - thread->Close = SCLogFileCloseNoLock; - } + thread->Write = SCLogFileWrite; + thread->Close = SCLogFileClose; + thread->Rotate = SCLogFileRotate; OutputRegisterFileRotationFlag(&thread->rotation_flag); - LogFileRegisterForFlush(thread); + LogFileRegister(thread); } else if (parent_ctx->type == LOGFILE_TYPE_FILETYPE) { entry->slot_number = SC_ATOMIC_ADD(eve_file_id, 1); SCLogDebug("%s - thread %d [slot %d]", log_path, entry->internal_thread_id, @@ -948,7 +958,7 @@ int LogFileFreeCtx(LogFileCtx *lf_ctx) /* Unregister from flush list first, before closing files. * This ensures the heartbeat thread won't try to flush a context * that's being destroyed. */ - LogFileUnregisterForFlush(lf_ctx); + LogFileUnregister(lf_ctx); if (lf_ctx->type == LOGFILE_TYPE_FILETYPE && lf_ctx->filetype.filetype->ThreadDeinit) { lf_ctx->filetype.filetype->ThreadDeinit( @@ -1016,83 +1026,90 @@ void LogFileFlush(LogFileCtx *file_ctx) } /** - * \brief Register a LogFileCtx for flush operations + * \brief Register a LogFileCtx for maintenance operations * - * Adds a LogFileCtx to the global flush list so the heartbeat thread - * can flush it directly without using pseudo packets. + * Adds a LogFileCtx to the global log file list so the heartbeat thread + * can perform flush and rotation on it. * * \param ctx The LogFileCtx to register (must be LOGFILE_TYPE_FILE) */ -void LogFileRegisterForFlush(LogFileCtx *ctx) +void LogFileRegister(LogFileCtx *ctx) { - if (!OutputFlushInterval()) { - SCLogDebug("heartbeat disabled; skipping flush registration"); - return; - } - if (ctx == NULL || ctx->type != LOGFILE_TYPE_FILE) { return; } - LogFileFlushEntry *entry = SCMalloc(sizeof(LogFileFlushEntry)); + LogFileEntry *entry = SCMalloc(sizeof(LogFileEntry)); if (entry == NULL) { - SCLogError("Unable to allocate memory for flush entry"); + SCLogError("Unable to allocate memory for log file entry"); return; } entry->ctx = ctx; - SCMutexLock(&log_file_flush_mutex); - TAILQ_INSERT_TAIL(&log_file_flush_list, entry, entries); - SCMutexUnlock(&log_file_flush_mutex); + SCMutexLock(&log_file_list_mutex); + TAILQ_INSERT_TAIL(&log_file_list, entry, entries); + SCMutexUnlock(&log_file_list_mutex); } /** - * \brief Unregister a LogFileCtx from flush operations + * \brief Unregister a LogFileCtx from maintenance operations * - * Removes a LogFileCtx from the global flush list. + * Removes a LogFileCtx from the global log file list. * * \param ctx The LogFileCtx to unregister */ -void LogFileUnregisterForFlush(LogFileCtx *ctx) +void LogFileUnregister(LogFileCtx *ctx) { - if (!OutputFlushInterval()) { - SCLogDebug("heartbeat disabled; skipping flush deregistration"); - return; - } - if (ctx == NULL) { return; } - SCMutexLock(&log_file_flush_mutex); - LogFileFlushEntry *entry, *safe; - TAILQ_FOREACH_SAFE (entry, &log_file_flush_list, entries, safe) { + SCMutexLock(&log_file_list_mutex); + LogFileEntry *entry, *safe; + TAILQ_FOREACH_SAFE (entry, &log_file_list, entries, safe) { if (entry->ctx == ctx) { - TAILQ_REMOVE(&log_file_flush_list, entry, entries); + TAILQ_REMOVE(&log_file_list, entry, entries); SCFree(entry); break; } } - SCMutexUnlock(&log_file_flush_mutex); + SCMutexUnlock(&log_file_list_mutex); } /** * \brief Flush all registered LogFileCtx instances * - * Called by the heartbeat thread to flush all active file-based loggers. - * Iterates through the flush list and calls LogFileFlush on each context. + * Called by the maintenance thread to flush all active file-based loggers. */ void LogFileFlushAll(void) { - SCMutexLock(&log_file_flush_mutex); - LogFileFlushEntry *entry; - TAILQ_FOREACH (entry, &log_file_flush_list, entries) { + SCMutexLock(&log_file_list_mutex); + LogFileEntry *entry; + TAILQ_FOREACH (entry, &log_file_list, entries) { if (entry->ctx != NULL) { LogFileFlush(entry->ctx); } } - SCMutexUnlock(&log_file_flush_mutex); + SCMutexUnlock(&log_file_list_mutex); +} + +/** + * \brief Check rotation for all registered LogFileCtx instances + * + * Called by the maintenance thread to trigger rotation checks on all + * registered log contexts during zero-traffic periods. + */ +void LogFileRotateAll(void) +{ + SCMutexLock(&log_file_list_mutex); + LogFileEntry *entry; + TAILQ_FOREACH (entry, &log_file_list, entries) { + if (entry->ctx != NULL && entry->ctx->Rotate != NULL) { + entry->ctx->Rotate(entry->ctx); + } + } + SCMutexUnlock(&log_file_list_mutex); } int LogFileWrite(LogFileCtx *file_ctx, MemBuffer *buffer) diff --git a/src/util-logopenfile.h b/src/util-logopenfile.h index d8ca383e10..314673ff37 100644 --- a/src/util-logopenfile.h +++ b/src/util-logopenfile.h @@ -68,10 +68,10 @@ typedef struct LogFileTypeCtx_ { void *thread_data; } LogFileTypeCtx; -typedef struct LogFileFlushEntry_ { +typedef struct LogFileEntry_ { struct LogFileCtx_ *ctx; - TAILQ_ENTRY(LogFileFlushEntry_) entries; -} LogFileFlushEntry; + TAILQ_ENTRY(LogFileEntry_) entries; +} LogFileEntry; /** Global structure for Output Context */ typedef struct LogFileCtx_ { @@ -92,6 +92,7 @@ typedef struct LogFileCtx_ { int (*Write)(const char *buffer, int buffer_len, struct LogFileCtx_ *fp); void (*Close)(struct LogFileCtx_ *fp); void (*Flush)(struct LogFileCtx_ *fp); + void (*Rotate)(struct LogFileCtx_ *fp); LogFileTypeCtx filetype; @@ -192,9 +193,10 @@ int SCConfLogOpenGeneric(SCConfNode *conf, LogFileCtx *, const char *, int); int SCConfLogReopen(LogFileCtx *); bool SCLogOpenThreadedFile(const char *log_path, const char *append, LogFileCtx *parent_ctx); -/* Flush list management functions */ -void LogFileRegisterForFlush(LogFileCtx *ctx); -void LogFileUnregisterForFlush(LogFileCtx *ctx); +/* Log file list management functions */ +void LogFileRegister(LogFileCtx *ctx); +void LogFileUnregister(LogFileCtx *ctx); void LogFileFlushAll(void); +void LogFileRotateAll(void); #endif /* SURICATA_UTIL_LOGOPENFILE_H */