aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

source-pcap-file: Directory mode may miss files (bug #2394)

Browse Source 
https://redmine.openinfosecfoundation.org/issues/2394

Certain parameters of delay and poll interval could cause newly added
files in a directory to be missed. Cleaned up how time is handled for
files in a directory and fix which time is used for future directory
traversals. Add a mutex to make sure processing time is not optimized
away.
pull/3263/head
Danny Browning 8 years ago committed by Victor Julien
parent cd98d7ddcc
commit 4b897c9060
7 changed files with 62 additions and 48 deletions
Show Stats Download Patch File Download Diff File

3
scripts/suricatasc/src/suricatasc.py
Unescape Escape View File

@ -203,12 +203,13 @@ class SuricataSC:
tenant = None
if len(parts) > 3:
tenant = parts[3]
if cmd != "pcap-file-continuous":
if cmd != "pcap-file":
raise SuricataCommandException("Invalid command '%s'" % (command))
else:
arguments = {}
arguments["filename"] = filename
arguments["output-dir"] = output
arguments["continuous"] = True
if tenant != None:
arguments["tenant"] = int(tenant)
elif "iface-stat" in command:

20
src/runmode-unix-socket.c
Unescape Escape View File

@ -34,6 +34,7 @@
#include "flow-timeout.h"
#include "stream-tcp.h"
#include "stream-tcp-reassemble.h"
#include "source-pcap-file-directory-helper.h"
#include "host.h"
#include "defrag.h"
#include "defrag-hash.h"
@ -132,6 +133,7 @@ static int unix_manager_pcap_task_running = 0;
static int unix_manager_pcap_task_failed = 0;
static int unix_manager_pcap_task_interrupted = 0;
static struct timespec unix_manager_pcap_last_processed;
static SCCtrlMutex unix_manager_pcap_last_processed_mutex;
/**
* \brief return list of files in the queue
@ -186,7 +188,7 @@ static TmEcode UnixSocketPcapCurrent(json_t *cmd, json_t* answer, void *data)
{
PcapCommand *this = (PcapCommand *) data;
if (this->current_file && this->current_file->filename) {
if (this->current_file != NULL && this->current_file->filename != NULL) {
json_object_set_new(answer, "message",
json_string(this->current_file->filename));
} else {
@ -197,8 +199,11 @@ static TmEcode UnixSocketPcapCurrent(json_t *cmd, json_t* answer, void *data)
static TmEcode UnixSocketPcapLastProcessed(json_t *cmd, json_t *answer, void *data)
{
uint64_t epoch_millis = unix_manager_pcap_last_processed.tv_sec * 1000l +
unix_manager_pcap_last_processed.tv_nsec / 100000l;
json_int_t epoch_millis;
SCCtrlMutexLock(&unix_manager_pcap_last_processed_mutex);
epoch_millis = SCTimespecAsEpochMillis(&unix_manager_pcap_last_processed);
SCCtrlMutexUnlock(&unix_manager_pcap_last_processed_mutex);
json_object_set_new(answer, "message",
json_integer(epoch_millis));
@ -422,7 +427,7 @@ static TmEcode UnixSocketAddPcapFile(json_t *cmd, json_t* answer, void *data)
}
/**
* \brief Command to add a file to treatment list
* \brief Command to add a file to treatment list, forcing continuous mode
*
* \param cmd the content of command Arguments as a json_t object
* \param answer the json_t object that has to be used to answer
@ -467,6 +472,7 @@ static TmEcode UnixSocketPcapFilesCheck(void *data)
}
this->current_file = NULL;
}
if (TAILQ_EMPTY(&this->files)) {
// nothing to do
return TM_ECODE_OK;
@ -573,8 +579,10 @@ void RunModeUnixSocketRegister(void)
TmEcode UnixSocketPcapFile(TmEcode tm, struct timespec *last_processed)
{
#ifdef BUILD_UNIX_SOCKET
SCCtrlMutexLock(&unix_manager_pcap_last_processed_mutex);
unix_manager_pcap_last_processed.tv_sec = last_processed->tv_sec;
unix_manager_pcap_last_processed.tv_nsec = last_processed->tv_nsec;
SCCtrlMutexUnlock(&unix_manager_pcap_last_processed_mutex);
switch (tm) {
case TM_ECODE_DONE:
SCLogInfo("Marking current task as done");
@ -1399,6 +1407,10 @@ static int RunModeUnixSocketMaster(void)
pcapcmd->running = 0;
pcapcmd->current_file = NULL;
memset(&unix_manager_pcap_last_processed, 0, sizeof(struct timespec));
SCCtrlMutexInit(&unix_manager_pcap_last_processed_mutex, NULL);
UnixManagerRegisterCommand("pcap-file", UnixSocketAddPcapFile, pcapcmd, UNIX_CMD_TAKE_ARGS);
UnixManagerRegisterCommand("pcap-file-continuous", UnixSocketAddPcapFileContinuous, pcapcmd, UNIX_CMD_TAKE_ARGS);
UnixManagerRegisterCommand("pcap-file-number", UnixSocketPcapFilesNumber, pcapcmd, 0);

83
src/source-pcap-file-directory-helper.c
Unescape Escape View File

@ -35,13 +35,10 @@ static TmEcode PcapDirectoryFailure(PcapFileDirectoryVars *ptv);
static TmEcode PcapDirectoryDone(PcapFileDirectoryVars *ptv);
static int PcapDirectoryGetModifiedTime(char const * file, struct timespec * out);
static TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
PendingFile *file_to_add,
struct timespec *file_to_add_modified_time);
PendingFile *file_to_add);
static TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *ptv,
struct timespec * newer_than,
struct timespec * older_than);
static TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
struct timespec *newer_than,
struct timespec *older_than);
void GetTime(struct timespec *tm)
@ -49,7 +46,7 @@ void GetTime(struct timespec *tm)
struct timeval now;
if(gettimeofday(&now, NULL) == 0) {
tm->tv_sec = now.tv_sec;
tm->tv_nsec = now.tv_usec * 1000;
tm->tv_nsec = now.tv_usec * 1000L;
}
}
@ -248,22 +245,20 @@ int PcapDirectoryGetModifiedTime(char const *file, struct timespec *out)
#endif
#ifdef OS_DARWIN
*out = buf.st_mtimespec;
out->tv_sec = buf.st_mtimespec.tv_sec;
out->tv_nsec = buf.st_mtimespec.tv_nsec;
#elif OS_WIN32
struct timespec ts;
memset(&ts, 0, sizeof(ts));
ts.tv_sec = buf.st_mtime;
*out = ts;
out->tv_sec = buf.st_mtime;
#else
*out = buf.st_mtim;
out->tv_sec = buf.st_mtim.tv_sec;
out->tv_nsec = buf.st_mtim.tv_nsec;
#endif
return ret;
}
TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
PendingFile *file_to_add,
struct timespec *file_to_add_modified_time
PendingFile *file_to_add
) {
PendingFile *file_to_compare = NULL;
PendingFile *next_file_to_compare = NULL;
@ -290,14 +285,7 @@ TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
} else {
file_to_compare = TAILQ_FIRST(&pv->directory_content);
while(file_to_compare != NULL) {
struct timespec modified_time;
memset(&modified_time, 0, sizeof(struct timespec));
if (PcapDirectoryGetModifiedTime(file_to_compare->filename,
&modified_time) == TM_ECODE_FAILED) {
SCReturnInt(TM_ECODE_FAILED);
}
if (CompareTimes(file_to_add_modified_time, &modified_time) < 0) {
if (CompareTimes(&file_to_add->modified_time, &file_to_compare->modified_time) < 0) {
TAILQ_INSERT_BEFORE(file_to_compare, file_to_add, next);
file_to_compare = NULL;
} else {
@ -315,7 +303,6 @@ TmEcode PcapDirectoryInsertFile(PcapFileDirectoryVars *pv,
}
TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv,
struct timespec *newer_than,
struct timespec *older_than
) {
if (unlikely(pv == NULL)) {
@ -355,11 +342,14 @@ TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv,
memset(&temp_time, 0, sizeof(struct timespec));
if (PcapDirectoryGetModifiedTime(pathbuff, &temp_time) == 0) {
SCLogDebug("File %s time (%lu > %lu < %lu)", pathbuff,
newer_than->tv_sec, temp_time.tv_sec, older_than->tv_sec);
SCLogDebug("%" PRIuMAX " < %" PRIuMAX "(%s) < %" PRIuMAX ")",
(uintmax_t)SCTimespecAsEpochMillis(&pv->shared->last_processed),
(uintmax_t)SCTimespecAsEpochMillis(&temp_time),
pathbuff,
(uintmax_t)SCTimespecAsEpochMillis(older_than));
// Skip files outside of our time range
if (CompareTimes(&temp_time, newer_than) < 0) {
if (CompareTimes(&temp_time, &pv->shared->last_processed) <= 0) {
SCLogDebug("Skipping old file %s", pathbuff);
continue;
}
@ -387,9 +377,13 @@ TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv,
SCReturnInt(TM_ECODE_FAILED);
}
SCLogDebug("Found \"%s\"", file_to_add->filename);
memset(&file_to_add->modified_time, 0, sizeof(struct timespec));
CopyTime(&temp_time, &file_to_add->modified_time);
if (PcapDirectoryInsertFile(pv, file_to_add, &temp_time) == TM_ECODE_FAILED) {
SCLogInfo("Found \"%s\" at %" PRIuMAX, file_to_add->filename,
(uintmax_t)SCTimespecAsEpochMillis(&file_to_add->modified_time));
if (PcapDirectoryInsertFile(pv, file_to_add) == TM_ECODE_FAILED) {
SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to add file");
CleanupPendingFile(file_to_add);
@ -403,10 +397,9 @@ TmEcode PcapDirectoryPopulateBuffer(PcapFileDirectoryVars *pv,
TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
struct timespec *newer_than,
struct timespec *older_than)
{
if (PcapDirectoryPopulateBuffer(pv, newer_than, older_than) == TM_ECODE_FAILED) {
if (PcapDirectoryPopulateBuffer(pv, older_than) == TM_ECODE_FAILED) {
SCLogError(SC_ERR_INVALID_ARGUMENT, "Failed to populate directory buffer");
SCReturnInt(TM_ECODE_FAILED);
}
@ -414,7 +407,7 @@ TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
TmEcode status = TM_ECODE_OK;
if (TAILQ_EMPTY(&pv->directory_content)) {
SCLogInfo("Directory %s has no files to process", pv->filename);
SCLogDebug("Directory %s has no files to process", pv->filename);
GetTime(older_than);
older_than->tv_sec = older_than->tv_sec - pv->delay;
rewinddir(pv->directory);
@ -422,6 +415,9 @@ TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
} else {
PendingFile *current_file = NULL;
struct timespec last_time_seen;
memset(&last_time_seen, 0, sizeof(struct timespec));
while (status == TM_ECODE_OK && !TAILQ_EMPTY(&pv->directory_content)) {
current_file = TAILQ_FIRST(&pv->directory_content);
TAILQ_REMOVE(&pv->directory_content, current_file, next);
@ -467,16 +463,13 @@ TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
SCReturnInt(status);
}
struct timespec temp_time;
memset(&temp_time, 0, sizeof(struct timespec));
SCLogInfo("Processed file %s, processed up to %" PRIuMAX,
current_file->filename,
(uintmax_t)SCTimespecAsEpochMillis(&current_file->modified_time));
if (PcapDirectoryGetModifiedTime(current_file->filename,
&temp_time) != 0) {
CopyTime(&temp_time, newer_than);
if(CompareTimes(&current_file->modified_time, &last_time_seen) > 0) {
CopyTime(&current_file->modified_time, &last_time_seen);
}
SCLogDebug("Processed file %s, processed up to %ld",
current_file->filename, temp_time.tv_sec);
CopyTime(&temp_time, &pv->shared->last_processed);
CleanupPendingFile(current_file);
pv->current_file = NULL;
@ -486,7 +479,12 @@ TmEcode PcapDirectoryDispatchForTimeRange(PcapFileDirectoryVars *pv,
}
}
*newer_than = *older_than;
if(CompareTimes(&last_time_seen, &pv->shared->last_processed) > 0) {
SCLogInfo("Updating processed to %" PRIuMAX,
(uintmax_t)SCTimespecAsEpochMillis(&last_time_seen));
CopyTime(&last_time_seen, &pv->shared->last_processed);
status = PcapRunStatus(pv);
}
}
GetTime(older_than);
older_than->tv_sec = older_than->tv_sec - pv->delay;
@ -500,8 +498,6 @@ TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv)
DIR *directory_check = NULL;
struct timespec newer_than;
memset(&newer_than, 0, sizeof(struct timespec));
struct timespec older_than;
memset(&older_than, 0, sizeof(struct timespec));
older_than.tv_sec = LONG_MAX;
@ -516,8 +512,9 @@ TmEcode PcapDirectoryDispatch(PcapFileDirectoryVars *ptv)
while (status == TM_ECODE_OK) {
//loop while directory is ok
SCLogInfo("Processing pcaps directory %s, files must be newer than %" PRIuMAX " and older than %" PRIuMAX,
ptv->filename, (uintmax_t)newer_than.tv_sec, (uintmax_t)older_than.tv_sec);
status = PcapDirectoryDispatchForTimeRange(ptv, &newer_than, &older_than);
ptv->filename, (uintmax_t)SCTimespecAsEpochMillis(&ptv->shared->last_processed),
(uintmax_t)SCTimespecAsEpochMillis(&older_than));
status = PcapDirectoryDispatchForTimeRange(ptv, &older_than);
if (ptv->should_loop && status == TM_ECODE_OK) {
sleep(poll_seconds);
//update our status based on suricata control flags or unix command socket

1
src/source-pcap-file-directory-helper.h
Unescape Escape View File

@ -31,6 +31,7 @@
typedef struct PendingFile_
{
char *filename;
struct timespec modified_time;
TAILQ_ENTRY(PendingFile_) next;
} PendingFile;
/**

1
src/source-pcap-file.c
Unescape Escape View File

@ -193,6 +193,7 @@ TmEcode ReceivePcapFileThreadInit(ThreadVars *tv, const void *initdata, void **d
if (unlikely(ptv == NULL))
SCReturnInt(TM_ECODE_FAILED);
memset(ptv, 0, sizeof(PcapFileThreadVars));
memset(&ptv->shared.last_processed, 0, sizeof(struct timespec));
intmax_t tenant = 0;
if (ConfGetInt("pcap-file.tenant-id", &tenant) == 1) {

1
src/suricata.c
Unescape Escape View File

@ -2264,6 +2264,7 @@ void PostRunDeinit(const int runmode, struct timeval *start_time)
HostCleanup();
StreamTcpFreeConfig(STREAM_VERBOSE);
DefragDestroy();
TmqResetQueues();
#ifdef PROFILING
if (profiling_rules_enabled)

1
src/util-time.h
Unescape Escape View File

@ -63,5 +63,6 @@ int SCTimeToStringPattern (time_t epoch, const char *pattern, char *str,
uint64_t SCParseTimeSizeString (const char *str);
uint64_t SCGetSecondsUntil (const char *str, time_t epoch);
uint64_t SCTimespecAsEpochMillis(const struct timespec *ts);
#endif /* __UTIL_TIME_H__ */

Write Preview
Loading…
Cancel
Save