mirror of https://github.com/OISF/suricata
Introduce Filedata Logger API
A new logger API for registering file storage handlers. Where the
FileLog handler is called once per file, this handler will be called
for each data chunk so that storing the entire file is possible.
The logger call in the API is as follows:
typedef int (*FiledataLogger)(ThreadVars *, void *thread_data,
const Packet *, const File *, const FileData *, uint8_t flags);
All data is const, thus should be read only. The final flags field
is used to indicate to the caller that the file is new, or if it's
being closed.
Files use an internal unique id 'file_id' which can be used by the
loggers to create unique file names. This id can use the 'waldo'
feature of the log-filestore module. This patch moves that waldo
loading and storing logic to this API's implementation. A new
configuration directive 'file-store-waldo: <filename>' is added,
but the existing waldo settings will also continue to work.
pull/797/head
Victor Julien
12 years ago
parent
b31e0abffe
commit
9ff6608668
@ -0,0 +1,440 @@
|
||||
/* Copyright (C) 2007-2014 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*
|
||||
* AppLayer Filedata Logger Output registration functions
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "tm-modules.h"
|
||||
#include "output-filedata.h"
|
||||
#include "app-layer.h"
|
||||
#include "app-layer-parser.h"
|
||||
#include "detect-filemagic.h"
|
||||
#include "conf.h"
|
||||
|
||||
typedef struct OutputLoggerThreadStore_ {
|
||||
void *thread_data;
|
||||
struct OutputLoggerThreadStore_ *next;
|
||||
} OutputLoggerThreadStore;
|
||||
|
||||
/** per thread data for this module, contains a list of per thread
|
||||
* data for the packet loggers. */
|
||||
typedef struct OutputLoggerThreadData_ {
|
||||
OutputLoggerThreadStore *store;
|
||||
} OutputLoggerThreadData;
|
||||
|
||||
/* logger instance, a module + a output ctx,
|
||||
* it's perfectly valid that have multiple instances of the same
|
||||
* log module (e.g. http.log) with different output ctx'. */
|
||||
typedef struct OutputFiledataLogger_ {
|
||||
FiledataLogger LogFunc;
|
||||
OutputCtx *output_ctx;
|
||||
struct OutputFiledataLogger_ *next;
|
||||
const char *name;
|
||||
} OutputFiledataLogger;
|
||||
|
||||
static OutputFiledataLogger *list = NULL;
|
||||
static char g_waldo[PATH_MAX] = "";
|
||||
static SCMutex g_waldo_mutex = SCMUTEX_INITIALIZER;
|
||||
static int g_waldo_init = 0;
|
||||
static int g_waldo_deinit = 0;
|
||||
|
||||
int OutputRegisterFiledataLogger(char *name, FiledataLogger LogFunc, OutputCtx *output_ctx) {
|
||||
OutputFiledataLogger *op = SCMalloc(sizeof(*op));
|
||||
if (op == NULL)
|
||||
return -1;
|
||||
memset(op, 0x00, sizeof(*op));
|
||||
|
||||
op->LogFunc = LogFunc;
|
||||
op->output_ctx = output_ctx;
|
||||
op->name = SCStrdup(name);
|
||||
if (op->name == NULL) {
|
||||
SCFree(op);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (list == NULL)
|
||||
list = op;
|
||||
else {
|
||||
OutputFiledataLogger *t = list;
|
||||
while (t->next)
|
||||
t = t->next;
|
||||
t->next = op;
|
||||
}
|
||||
|
||||
SCLogDebug("OutputRegisterTxLogger happy");
|
||||
return 0;
|
||||
}
|
||||
|
||||
SC_ATOMIC_DECLARE(unsigned int, file_id);
|
||||
|
||||
static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data, PacketQueue *pq, PacketQueue *postpq) {
|
||||
BUG_ON(thread_data == NULL);
|
||||
BUG_ON(list == NULL);
|
||||
|
||||
OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
|
||||
OutputFiledataLogger *logger = list;
|
||||
OutputLoggerThreadStore *store = op_thread_data->store;
|
||||
|
||||
BUG_ON(logger == NULL && store != NULL);
|
||||
BUG_ON(logger != NULL && store == NULL);
|
||||
BUG_ON(logger == NULL && store == NULL);
|
||||
|
||||
uint8_t flags = 0;
|
||||
Flow * const f = p->flow;
|
||||
|
||||
/* no flow, no files */
|
||||
if (f == NULL) {
|
||||
SCReturnInt(TM_ECODE_OK);
|
||||
}
|
||||
|
||||
if (p->flowflags & FLOW_PKT_TOCLIENT)
|
||||
flags |= STREAM_TOCLIENT;
|
||||
else
|
||||
flags |= STREAM_TOSERVER;
|
||||
|
||||
int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
|
||||
int file_trunc = 0;
|
||||
|
||||
FLOWLOCK_WRLOCK(f); // < need write lock for FiledataPrune below
|
||||
file_trunc = StreamTcpReassembleDepthReached(p);
|
||||
|
||||
FileContainer *ffc = AppLayerParserGetFiles(p->proto, f->alproto,
|
||||
f->alstate, flags);
|
||||
SCLogDebug("ffc %p", ffc);
|
||||
if (ffc != NULL) {
|
||||
File *ff;
|
||||
for (ff = ffc->head; ff != NULL; ff = ff->next) {
|
||||
if (FileForceMagic() && ff->magic == NULL) {
|
||||
FilemagicGlobalLookup(ff);
|
||||
}
|
||||
|
||||
SCLogDebug("ff %p", ff);
|
||||
if (ff->flags & FILE_STORED) {
|
||||
SCLogDebug("stored flag set");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!(ff->flags & FILE_STORE)) {
|
||||
SCLogDebug("ff FILE_STORE not set");
|
||||
continue;
|
||||
}
|
||||
|
||||
FileData *ffd;
|
||||
for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) {
|
||||
uint8_t flags = 0;
|
||||
int file_logged = 0;
|
||||
FileData *write_ffd = ffd;
|
||||
|
||||
SCLogDebug("ffd %p", ffd);
|
||||
|
||||
/* special case: on stream end we may inform the
|
||||
* loggers that the file is truncated. In this
|
||||
* case we already logged the current ffd, which
|
||||
* is the last in our list. */
|
||||
if (ffd->stored == 1) {
|
||||
if (!(file_close == 1 || ffd->next == NULL)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// call writer with NULL ffd, so it can 'close'
|
||||
// so really a 'close' msg
|
||||
write_ffd = NULL;
|
||||
flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
|
||||
}
|
||||
|
||||
/* store */
|
||||
|
||||
/* if file_id == 0, this is the first store of this file */
|
||||
if (ff->file_id == 0) {
|
||||
/* new file */
|
||||
ff->file_id = SC_ATOMIC_ADD(file_id, 1);
|
||||
flags |= OUTPUT_FILEDATA_FLAG_OPEN;
|
||||
} else {
|
||||
/* existing file */
|
||||
}
|
||||
|
||||
/* if file needs to be closed or truncated, inform
|
||||
* loggers */
|
||||
if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
|
||||
ff->state = FILE_STATE_TRUNCATED;
|
||||
}
|
||||
|
||||
/* for the last data chunk we have, also tell the logger
|
||||
* we're closing up */
|
||||
if (ffd->next == NULL && ff->state >= FILE_STATE_CLOSED)
|
||||
flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
|
||||
|
||||
logger = list;
|
||||
store = op_thread_data->store;
|
||||
while (logger && store) {
|
||||
BUG_ON(logger->LogFunc == NULL);
|
||||
|
||||
SCLogDebug("logger %p", logger);
|
||||
logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff,
|
||||
(const FileData *)write_ffd, flags);
|
||||
file_logged = 1;
|
||||
|
||||
logger = logger->next;
|
||||
store = store->next;
|
||||
|
||||
BUG_ON(logger == NULL && store != NULL);
|
||||
BUG_ON(logger != NULL && store == NULL);
|
||||
}
|
||||
|
||||
if (file_logged) {
|
||||
ffd->stored = 1;
|
||||
|
||||
/* all done */
|
||||
if (flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
|
||||
ff->flags |= FILE_STORED;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
FilePrune(ffc);
|
||||
}
|
||||
|
||||
FLOWLOCK_UNLOCK(f);
|
||||
return TM_ECODE_OK;
|
||||
}
|
||||
|
||||
/**
|
||||
* \internal
|
||||
*
|
||||
* \brief Open the waldo file (if available) and load the file_id
|
||||
*
|
||||
* \param path full path for the waldo file
|
||||
*/
|
||||
static void LogFiledataLogLoadWaldo(const char *path) {
|
||||
char line[16] = "";
|
||||
unsigned int id = 0;
|
||||
|
||||
FILE *fp = fopen(path, "r");
|
||||
if (fp == NULL) {
|
||||
SCLogInfo("couldn't open waldo: %s", strerror(errno));
|
||||
SCReturn;
|
||||
}
|
||||
|
||||
if (fgets(line, (int)sizeof(line), fp) != NULL) {
|
||||
if (sscanf(line, "%10u", &id) == 1) {
|
||||
SCLogInfo("id %u", id);
|
||||
(void) SC_ATOMIC_CAS(&file_id, 0, id);
|
||||
}
|
||||
}
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/**
|
||||
* \internal
|
||||
*
|
||||
* \brief Store the waldo file based on the file_id
|
||||
*
|
||||
* \param path full path for the waldo file
|
||||
*/
|
||||
static void LogFiledataLogStoreWaldo(const char *path) {
|
||||
char line[16] = "";
|
||||
|
||||
if (SC_ATOMIC_GET(file_id) == 0) {
|
||||
SCReturn;
|
||||
}
|
||||
|
||||
FILE *fp = fopen(path, "w");
|
||||
if (fp == NULL) {
|
||||
SCLogInfo("couldn't open waldo: %s", strerror(errno));
|
||||
SCReturn;
|
||||
}
|
||||
|
||||
snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(file_id));
|
||||
if (fwrite(line, strlen(line), 1, fp) != 1) {
|
||||
SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno));
|
||||
}
|
||||
fclose(fp);
|
||||
}
|
||||
|
||||
/** \brief thread init for the tx logger
|
||||
* This will run the thread init functions for the individual registered
|
||||
* loggers */
|
||||
static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, void *initdata, void **data) {
|
||||
OutputLoggerThreadData *td = SCMalloc(sizeof(*td));
|
||||
if (td == NULL)
|
||||
return TM_ECODE_FAILED;
|
||||
memset(td, 0x00, sizeof(*td));
|
||||
|
||||
*data = (void *)td;
|
||||
|
||||
SCLogDebug("OutputFiledataLogThreadInit happy (*data %p)", *data);
|
||||
|
||||
OutputFiledataLogger *logger = list;
|
||||
while (logger) {
|
||||
TmModule *tm_module = TmModuleGetByName((char *)logger->name);
|
||||
if (tm_module == NULL) {
|
||||
SCLogError(SC_ERR_INVALID_ARGUMENT,
|
||||
"TmModuleGetByName for %s failed", logger->name);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (tm_module->ThreadInit) {
|
||||
void *retptr = NULL;
|
||||
if (tm_module->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
|
||||
OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
|
||||
/* todo */ BUG_ON(ts == NULL);
|
||||
memset(ts, 0x00, sizeof(*ts));
|
||||
|
||||
/* store thread handle */
|
||||
ts->thread_data = retptr;
|
||||
|
||||
if (td->store == NULL) {
|
||||
td->store = ts;
|
||||
} else {
|
||||
OutputLoggerThreadStore *tmp = td->store;
|
||||
while (tmp->next != NULL)
|
||||
tmp = tmp->next;
|
||||
tmp->next = ts;
|
||||
}
|
||||
|
||||
SCLogDebug("%s is now set up", logger->name);
|
||||
}
|
||||
}
|
||||
|
||||
logger = logger->next;
|
||||
}
|
||||
|
||||
SCMutexLock(&g_waldo_mutex);
|
||||
if (g_waldo_init == 0) {
|
||||
ConfNode *node = ConfGetNode("file-store-waldo");
|
||||
if (node == NULL) {
|
||||
ConfNode *outputs = ConfGetNode("outputs");
|
||||
if (outputs) {
|
||||
ConfNode *output;
|
||||
TAILQ_FOREACH(output, &outputs->head, next) {
|
||||
/* we only care about file and file-store */
|
||||
if (!(strcmp(output->val, "file") == 0 || strcmp(output->val, "file-store") == 0))
|
||||
continue;
|
||||
|
||||
ConfNode *file = ConfNodeLookupChild(output, output->val);
|
||||
BUG_ON(file == NULL);
|
||||
if (file == NULL) {
|
||||
SCLogDebug("file-store failed, lets try 'file'");
|
||||
file = ConfNodeLookupChild(outputs, "file");
|
||||
if (file == NULL)
|
||||
SCLogDebug("file failed as well, giving up");
|
||||
}
|
||||
|
||||
if (file != NULL) {
|
||||
node = ConfNodeLookupChild(file, "waldo");
|
||||
if (node == NULL)
|
||||
SCLogDebug("no waldo node");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (node != NULL) {
|
||||
char *s_default_log_dir = NULL;
|
||||
s_default_log_dir = ConfigGetLogDirectory();
|
||||
|
||||
const char *waldo = node->val;
|
||||
SCLogDebug("loading waldo %s", waldo);
|
||||
if (waldo != NULL && strlen(waldo) > 0) {
|
||||
if (PathIsAbsolute(waldo)) {
|
||||
snprintf(g_waldo, sizeof(g_waldo), "%s", waldo);
|
||||
} else {
|
||||
snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo);
|
||||
}
|
||||
|
||||
SCLogDebug("loading waldo file %s", g_waldo);
|
||||
LogFiledataLogLoadWaldo(g_waldo);
|
||||
}
|
||||
}
|
||||
g_waldo_init = 1;
|
||||
}
|
||||
SCMutexUnlock(&g_waldo_mutex);
|
||||
return TM_ECODE_OK;
|
||||
}
|
||||
|
||||
static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data) {
|
||||
OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
|
||||
OutputLoggerThreadStore *store = op_thread_data->store;
|
||||
OutputFiledataLogger *logger = list;
|
||||
|
||||
while (logger && store) {
|
||||
TmModule *tm_module = TmModuleGetByName((char *)logger->name);
|
||||
if (tm_module == NULL) {
|
||||
SCLogError(SC_ERR_INVALID_ARGUMENT,
|
||||
"TmModuleGetByName for %s failed", logger->name);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (tm_module->ThreadDeinit) {
|
||||
tm_module->ThreadDeinit(tv, store->thread_data);
|
||||
}
|
||||
|
||||
logger = logger->next;
|
||||
store = store->next;
|
||||
}
|
||||
|
||||
SCMutexLock(&g_waldo_mutex);
|
||||
if (g_waldo_deinit == 0) {
|
||||
if (strlen(g_waldo) > 0) {
|
||||
SCLogDebug("we have a waldo at %s", g_waldo);
|
||||
LogFiledataLogStoreWaldo(g_waldo);
|
||||
}
|
||||
g_waldo_deinit = 1;
|
||||
}
|
||||
SCMutexUnlock(&g_waldo_mutex);
|
||||
return TM_ECODE_OK;
|
||||
}
|
||||
|
||||
static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data) {
|
||||
OutputLoggerThreadData *op_thread_data = (OutputLoggerThreadData *)thread_data;
|
||||
OutputLoggerThreadStore *store = op_thread_data->store;
|
||||
OutputFiledataLogger *logger = list;
|
||||
|
||||
while (logger && store) {
|
||||
TmModule *tm_module = TmModuleGetByName((char *)logger->name);
|
||||
if (tm_module == NULL) {
|
||||
SCLogError(SC_ERR_INVALID_ARGUMENT,
|
||||
"TmModuleGetByName for %s failed", logger->name);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
if (tm_module->ThreadExitPrintStats) {
|
||||
tm_module->ThreadExitPrintStats(tv, store->thread_data);
|
||||
}
|
||||
|
||||
logger = logger->next;
|
||||
store = store->next;
|
||||
}
|
||||
}
|
||||
|
||||
void TmModuleFiledataLoggerRegister (void) {
|
||||
tmm_modules[TMM_FILEDATALOGGER].name = "__filedata_logger__";
|
||||
tmm_modules[TMM_FILEDATALOGGER].ThreadInit = OutputFiledataLogThreadInit;
|
||||
tmm_modules[TMM_FILEDATALOGGER].Func = OutputFiledataLog;
|
||||
tmm_modules[TMM_FILEDATALOGGER].ThreadExitPrintStats = OutputFiledataLogExitPrintStats;
|
||||
tmm_modules[TMM_FILEDATALOGGER].ThreadDeinit = OutputFiledataLogThreadDeinit;
|
||||
tmm_modules[TMM_FILEDATALOGGER].cap_flags = 0;
|
||||
|
||||
SC_ATOMIC_INIT(file_id);
|
||||
}
|
||||
@ -0,0 +1,48 @@
|
||||
/* Copyright (C) 2007-2014 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*
|
||||
* AppLayer Filedata Logger Output registration functions
|
||||
*/
|
||||
|
||||
#ifndef __OUTPUT_FILEDATA_H__
|
||||
#define __OUTPUT_FILEDATA_H__
|
||||
|
||||
#include "decode.h"
|
||||
#include "util-file.h"
|
||||
|
||||
#define OUTPUT_FILEDATA_FLAG_OPEN 0x01
|
||||
#define OUTPUT_FILEDATA_FLAG_CLOSE 0x02
|
||||
|
||||
/** filedata logger function pointer type */
|
||||
typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *,
|
||||
const File *, const FileData *, uint8_t);
|
||||
|
||||
/** packet logger condition function pointer type,
|
||||
* must return true for packets that should be logged
|
||||
*/
|
||||
//typedef int (*TxLogCondition)(ThreadVars *, const Packet *);
|
||||
|
||||
int OutputRegisterFiledataLogger(char *name, FiledataLogger LogFunc, OutputCtx *);
|
||||
|
||||
void TmModuleFiledataLoggerRegister (void);
|
||||
|
||||
#endif /* __OUTPUT_FILE_H__ */
|
||||
Loading…
Reference in New Issue