diff --git a/scripts/suricatasc/src/suricatasc.py b/scripts/suricatasc/src/suricatasc.py index 03483f37c2..22501afc25 100644 --- a/scripts/suricatasc/src/suricatasc.py +++ b/scripts/suricatasc/src/suricatasc.py @@ -203,12 +203,13 @@ class SuricataSC: tenant = None if len(parts) > 3: tenant = parts[3] - if cmd != "pcap-file-continuous": + if cmd != "pcap-file": raise SuricataCommandException("Invalid command '%s'" % (command)) else: arguments = {} arguments["filename"] = filename arguments["output-dir"] = output + arguments["continuous"] = True if tenant != None: arguments["tenant"] = int(tenant) elif "iface-stat" in command: diff --git a/src/runmode-unix-socket.c b/src/runmode-unix-socket.c index c28eee836f..8702e9c1eb 100644 --- a/src/runmode-unix-socket.c +++ b/src/runmode-unix-socket.c @@ -34,6 +34,7 @@ #include "flow-timeout.h" #include "stream-tcp.h" #include "stream-tcp-reassemble.h" +#include "source-pcap-file-directory-helper.h" #include "host.h" #include "defrag.h" #include "defrag-hash.h" @@ -132,6 +133,7 @@ static int unix_manager_pcap_task_running = 0; static int unix_manager_pcap_task_failed = 0; static int unix_manager_pcap_task_interrupted = 0; static struct timespec unix_manager_pcap_last_processed; +static SCCtrlMutex unix_manager_pcap_last_processed_mutex; /** * \brief return list of files in the queue @@ -186,7 +188,7 @@ static TmEcode UnixSocketPcapCurrent(json_t *cmd, json_t* answer, void *data) { PcapCommand *this = (PcapCommand *) data; - if (this->current_file && this->current_file->filename) { + if (this->current_file != NULL && this->current_file->filename != NULL) { json_object_set_new(answer, "message", json_string(this->current_file->filename)); } else { @@ -197,8 +199,11 @@ static TmEcode UnixSocketPcapCurrent(json_t *cmd, json_t* answer, void *data) static TmEcode UnixSocketPcapLastProcessed(json_t *cmd, json_t *answer, void *data) { - uint64_t epoch_millis = unix_manager_pcap_last_processed.tv_sec * 1000l + - unix_manager_pcap_last_processed.tv_nsec / 100000l; + json_int_t epoch_millis; + SCCtrlMutexLock(&unix_manager_pcap_last_processed_mutex); + epoch_millis = SCTimespecAsEpochMillis(&unix_manager_pcap_last_processed); + SCCtrlMutexUnlock(&unix_manager_pcap_last_processed_mutex); + json_object_set_new(answer, "message", json_integer(epoch_millis)); @@ -422,7 +427,7 @@ static TmEcode UnixSocketAddPcapFile(json_t *cmd, json_t* answer, void *data) } /** - * \brief Command to add a file to treatment list + * \brief Command to add a file to treatment list, forcing continuous mode * * \param cmd the content of command Arguments as a json_t object * \param answer the json_t object that has to be used to answer @@ -467,6 +472,7 @@ static TmEcode UnixSocketPcapFilesCheck(void *data) } this->current_file = NULL; } + if (TAILQ_EMPTY(&this->files)) { // nothing to do return TM_ECODE_OK; @@ -573,8 +579,10 @@ void RunModeUnixSocketRegister(void) TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed) { #ifdef BUILD_UNIX_SOCKET + SCCtrlMutexLock(&unix_manager_pcap_last_processed_mutex); unix_manager_pcap_last_processed.tv_sec = last_processed->tv_sec; unix_manager_pcap_last_processed.tv_nsec = last_processed->tv_nsec; + SCCtrlMutexUnlock(&unix_manager_pcap_last_processed_mutex); switch (tm) { case TM_ECODE_DONE: SCLogInfo("Marking current task as done"); @@ -1399,6 +1407,10 @@ static int RunModeUnixSocketMaster(void) pcapcmd->running = 0; pcapcmd->current_file = NULL; + memset(&unix_manager_pcap_last_processed, 0, sizeof(struct timespec)); + + SCCtrlMutexInit(&unix_manager_pcap_last_processed_mutex, NULL); + UnixManagerRegisterCommand("pcap-file", UnixSocketAddPcapFile, pcapcmd, UNIX_CMD_TAKE_ARGS); UnixManagerRegisterCommand("pcap-file-continuous", UnixSocketAddPcapFileContinuous, pcapcmd, UNIX_CMD_TAKE_ARGS); UnixManagerRegisterCommand("pcap-file-number", UnixSocketPcapFilesNumber, pcapcmd, 0); diff --git a/src/source-pcap-file-directory-helper.c b/src/source-pcap-file-directory-helper.c index 7367442c96..d026c1f5b7 100644 --- a/src/source-pcap-file-directory-helper.c +++ b/src/source-pcap-file-directory-helper.c @@ -35,13 +35,10 @@ static TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv); static TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv); static int PcapDirectoryGetModifiedTime(char const * file, struct timespec * out); static TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv, - PendingFile *file_to_add, - struct timespec *file_to_add_modified_time); + PendingFile *file_to_add); static TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *ptv, - struct timespec * newer_than, struct timespec * older_than); static TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv, - struct timespec *newer_than, struct timespec *older_than); void GetTime(struct timespec *tm) @@ -49,7 +46,7 @@ void GetTime(struct timespec *tm) struct timeval now; if(gettimeofday(&now, NULL) == 0) { tm->tv_sec = now.tv_sec; - tm->tv_nsec = now.tv_usec * 1000; + tm->tv_nsec = now.tv_usec * 1000L; } } @@ -248,22 +245,20 @@ int PcapDirectoryGetModifiedTime(char const *file, struct timespec *out) #endif #ifdef OS_DARWIN - *out = buf.st_mtimespec; + out->tv_sec = buf.st_mtimespec.tv_sec; + out->tv_nsec = buf.st_mtimespec.tv_nsec; #elif OS_WIN32 - struct timespec ts; - memset(&ts, 0, sizeof(ts)); - ts.tv_sec = buf.st_mtime; - *out = ts; + out->tv_sec = buf.st_mtime; #else - *out = buf.st_mtim; + out->tv_sec = buf.st_mtim.tv_sec; + out->tv_nsec = buf.st_mtim.tv_nsec; #endif return ret; } TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv, - PendingFile *file_to_add, - struct timespec *file_to_add_modified_time + PendingFile *file_to_add ) { PendingFile *file_to_compare = NULL; PendingFile *next_file_to_compare = NULL; @@ -290,14 +285,7 @@ TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv, } else { file_to_compare = TAILQ_FIRST(&pv->directory_content); while(file_to_compare != NULL) { - struct timespec modified_time; - memset(&modified_time, 0, sizeof(struct timespec)); - - if (PcapDirectoryGetModifiedTime(file_to_compare->filename, - &modified_time) == TM_ECODE_FAILED) { - SCReturnInt(TM_ECODE_FAILED); - } - if (CompareTimes(file_to_add_modified_time, &modified_time) < 0) { + if (CompareTimes(&file_to_add->modified_time, &file_to_compare->modified_time) < 0) { TAILQ_INSERT_BEFORE(file_to_compare, file_to_add, next); file_to_compare = NULL; } else { @@ -315,7 +303,6 @@ TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv, } TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv, - struct timespec *newer_than, struct timespec *older_than ) { if (unlikely(pv == NULL)) { @@ -355,11 +342,14 @@ TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv, memset(&temp_time, 0, sizeof(struct timespec)); if (PcapDirectoryGetModifiedTime(pathbuff, &temp_time) == 0) { - SCLogDebug("File %s time (%lu > %lu < %lu)", pathbuff, - newer_than->tv_sec, temp_time.tv_sec, older_than->tv_sec); + SCLogDebug("%" PRIuMAX " < %" PRIuMAX "(%s) < %" PRIuMAX ")", + (uintmax_t)SCTimespecAsEpochMillis(&pv->shared->last_processed), + (uintmax_t)SCTimespecAsEpochMillis(&temp_time), + pathbuff, + (uintmax_t)SCTimespecAsEpochMillis(older_than)); // Skip files outside of our time range - if (CompareTimes(&temp_time, newer_than) < 0) { + if (CompareTimes(&temp_time, &pv->shared->last_processed) <= 0) { SCLogDebug("Skipping old file %s", pathbuff); continue; } @@ -387,9 +377,13 @@ TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv, SCReturnInt(TM_ECODE_FAILED); } - SCLogDebug("Found \"%s\"", file_to_add->filename); + memset(&file_to_add->modified_time, 0, sizeof(struct timespec)); + CopyTime(&temp_time, &file_to_add->modified_time); - if (PcapDirectoryInsertFile(pv, file_to_add, &temp_time) == TM_ECODE_FAILED) { + SCLogInfo("Found \"%s\" at %" PRIuMAX, file_to_add->filename, + (uintmax_t)SCTimespecAsEpochMillis(&file_to_add->modified_time)); + + if (PcapDirectoryInsertFile(pv, file_to_add) == TM_ECODE_FAILED) { SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to add file"); CleanupPendingFile(file_to_add); @@ -403,10 +397,9 @@ TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv, TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv, - struct timespec *newer_than, struct timespec *older_than) { - if (PcapDirectoryPopulateBuffer(pv, newer_than, older_than) == TM_ECODE_FAILED) { + if (PcapDirectoryPopulateBuffer(pv, older_than) == TM_ECODE_FAILED) { SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to populate directory buffer"); SCReturnInt(TM_ECODE_FAILED); } @@ -414,7 +407,7 @@ TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv, TmEcode status = TM_ECODE_OK; if (TAILQ_EMPTY(&pv->directory_content)) { - SCLogInfo("Directory %s has no files to process", pv->filename); + SCLogDebug("Directory %s has no files to process", pv->filename); GetTime(older_than); older_than->tv_sec = older_than->tv_sec - pv->delay; rewinddir(pv->directory); @@ -422,6 +415,9 @@ TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv, } else { PendingFile *current_file = NULL; + struct timespec last_time_seen; + memset(&last_time_seen, 0, sizeof(struct timespec)); + while (status == TM_ECODE_OK && !TAILQ_EMPTY(&pv->directory_content)) { current_file = TAILQ_FIRST(&pv->directory_content); TAILQ_REMOVE(&pv->directory_content, current_file, next); @@ -467,16 +463,13 @@ TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv, SCReturnInt(status); } - struct timespec temp_time; - memset(&temp_time, 0, sizeof(struct timespec)); + SCLogInfo("Processed file %s, processed up to %" PRIuMAX, + current_file->filename, + (uintmax_t)SCTimespecAsEpochMillis(¤t_file->modified_time)); - if (PcapDirectoryGetModifiedTime(current_file->filename, - &temp_time) != 0) { - CopyTime(&temp_time, newer_than); + if(CompareTimes(¤t_file->modified_time, &last_time_seen) > 0) { + CopyTime(¤t_file->modified_time, &last_time_seen); } - SCLogDebug("Processed file %s, processed up to %ld", - current_file->filename, temp_time.tv_sec); - CopyTime(&temp_time, &pv->shared->last_processed); CleanupPendingFile(current_file); pv->current_file = NULL; @@ -486,7 +479,12 @@ TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv, } } - *newer_than = *older_than; + if(CompareTimes(&last_time_seen, &pv->shared->last_processed) > 0) { + SCLogInfo("Updating processed to %" PRIuMAX, + (uintmax_t)SCTimespecAsEpochMillis(&last_time_seen)); + CopyTime(&last_time_seen, &pv->shared->last_processed); + status = PcapRunStatus(pv); + } } GetTime(older_than); older_than->tv_sec = older_than->tv_sec - pv->delay; @@ -500,8 +498,6 @@ TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv) DIR *directory_check = NULL; - struct timespec newer_than; - memset(&newer_than, 0, sizeof(struct timespec)); struct timespec older_than; memset(&older_than, 0, sizeof(struct timespec)); older_than.tv_sec = LONG_MAX; @@ -516,8 +512,9 @@ TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv) while (status == TM_ECODE_OK) { //loop while directory is ok SCLogInfo("Processing pcaps directory %s, files must be newer than %" PRIuMAX " and older than %" PRIuMAX, - ptv->filename, (uintmax_t)newer_than.tv_sec, (uintmax_t)older_than.tv_sec); - status = PcapDirectoryDispatchForTimeRange(ptv, &newer_than, &older_than); + ptv->filename, (uintmax_t)SCTimespecAsEpochMillis(&ptv->shared->last_processed), + (uintmax_t)SCTimespecAsEpochMillis(&older_than)); + status = PcapDirectoryDispatchForTimeRange(ptv, &older_than); if (ptv->should_loop && status == TM_ECODE_OK) { sleep(poll_seconds); //update our status based on suricata control flags or unix command socket diff --git a/src/source-pcap-file-directory-helper.h b/src/source-pcap-file-directory-helper.h index 496e83793b..bc416db2bd 100644 --- a/src/source-pcap-file-directory-helper.h +++ b/src/source-pcap-file-directory-helper.h @@ -31,6 +31,7 @@ typedef struct PendingFile_ { char *filename; + struct timespec modified_time; TAILQ_ENTRY(PendingFile_) next; } PendingFile; /** diff --git a/src/source-pcap-file.c b/src/source-pcap-file.c index 30c5483bde..c08a4d7492 100644 --- a/src/source-pcap-file.c +++ b/src/source-pcap-file.c @@ -193,6 +193,7 @@ TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **d if (unlikely(ptv == NULL)) SCReturnInt(TM_ECODE_FAILED); memset(ptv, 0, sizeof(PcapFileThreadVars)); + memset(&ptv->shared.last_processed, 0, sizeof(struct timespec)); intmax_t tenant = 0; if (ConfGetInt("pcap-file.tenant-id", &tenant) == 1) { diff --git a/src/suricata.c b/src/suricata.c index a2d64eac7e..8b39ea68f5 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2264,6 +2264,7 @@ void PostRunDeinit(const int runmode, struct timeval *start_time) HostCleanup(); StreamTcpFreeConfig(STREAM_VERBOSE); DefragDestroy(); + TmqResetQueues(); #ifdef PROFILING if (profiling_rules_enabled) diff --git a/src/util-time.h b/src/util-time.h index 00c254b97b..a7e53ca989 100644 --- a/src/util-time.h +++ b/src/util-time.h @@ -63,5 +63,6 @@ int SCTimeToStringPattern (time_t epoch, const char *pattern, char *str, uint64_t SCParseTimeSizeString (const char *str); uint64_t SCGetSecondsUntil (const char *str, time_t epoch); uint64_t SCTimespecAsEpochMillis(const struct timespec *ts); + #endif /* __UTIL_TIME_H__ */