diff --git a/src/Makefile.am b/src/Makefile.am index 74c477d76c..6050d1a082 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -252,6 +252,7 @@ log-droplog.c log-droplog.h \ log-httplog.c log-httplog.h \ log-pcap.c log-pcap.h \ log-file.c log-file.h \ +log-filestore.c log-filestore.h \ stream.c stream.h \ stream-tcp.c stream-tcp.h stream-tcp-private.h \ stream-tcp-reassemble.c stream-tcp-reassemble.h \ diff --git a/src/log-file.c b/src/log-file.c index 3d24058d23..4dd757ecc1 100644 --- a/src/log-file.c +++ b/src/log-file.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2011 Open Information Security Foundation +/* Copyright (C) 2007-2012 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -20,6 +20,8 @@ * * \author Victor Julien * + * Log files we track. + * */ #include "suricata-common.h" @@ -67,10 +69,6 @@ int LogFileLogOpenFileCtx(LogFileCtx* , const char *, const char *); static OutputCtx *LogFileLogInitCtx(ConfNode *); static void LogFileLogDeInitCtx(OutputCtx *); -SC_ATOMIC_DECLARE(unsigned int, file_id); -static char g_logfile_base_dir[PATH_MAX] = "/tmp"; -static char g_waldo[PATH_MAX] = ""; - void TmModuleLogFileLogRegister (void) { tmm_modules[TMM_FILELOG].name = MODULE_NAME; tmm_modules[TMM_FILELOG].ThreadInit = LogFileLogThreadInit; @@ -80,11 +78,9 @@ void TmModuleLogFileLogRegister (void) { tmm_modules[TMM_FILELOG].RegisterTests = NULL; tmm_modules[TMM_FILELOG].cap_flags = 0; - OutputRegisterModule(MODULE_NAME, "file", LogFileLogInitCtx); + OutputRegisterModule(MODULE_NAME, "file-log", LogFileLogInitCtx); SCLogDebug("registered"); - - SC_ATOMIC_INIT(file_id); } typedef struct LogFileLogThread_ { @@ -163,106 +159,6 @@ static void LogFileMetaGetReferer(FILE *fp, Packet *p, File *ff) { fprintf(fp, ""); } -static void LogFileLogCreateMetaFile(Packet *p, File *ff, char *filename, int ipver) { - char metafilename[PATH_MAX] = ""; - snprintf(metafilename, sizeof(metafilename), "%s.meta", filename); - FILE *fp = fopen(metafilename, "w+"); - if (fp != NULL) { - char timebuf[64]; - - CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); - - fprintf(fp, "TIME: %s\n", timebuf); - if (p->pcap_cnt > 0) { - fprintf(fp, "PCAP PKT NUM: %"PRIu64"\n", p->pcap_cnt); - } - - char srcip[46], dstip[46]; - Port sp, dp; - switch (ipver) { - case AF_INET: - PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); - break; - case AF_INET6: - PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); - PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); - break; - default: - strlcpy(srcip, "", sizeof(srcip)); - strlcpy(dstip, "", sizeof(dstip)); - break; - } - sp = p->sp; - dp = p->dp; - - fprintf(fp, "SRC IP: %s\n", srcip); - fprintf(fp, "DST IP: %s\n", dstip); - fprintf(fp, "PROTO: %" PRIu32 "\n", p->proto); - if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) { - fprintf(fp, "SRC PORT: %" PRIu16 "\n", sp); - fprintf(fp, "DST PORT: %" PRIu16 "\n", dp); - } - fprintf(fp, "HTTP URI: "); - LogFileMetaGetUri(fp, p, ff); - fprintf(fp, "\n"); - fprintf(fp, "HTTP HOST: "); - LogFileMetaGetHost(fp, p, ff); - fprintf(fp, "\n"); - fprintf(fp, "HTTP REFERER: "); - LogFileMetaGetReferer(fp, p, ff); - fprintf(fp, "\n"); - fprintf(fp, "FILENAME: "); - PrintRawUriFp(fp, ff->name, ff->name_len); - fprintf(fp, "\n"); - - fclose(fp); - } -} - -static void LogFileLogCloseMetaFile(File *ff) { - char filename[PATH_MAX] = ""; - snprintf(filename, sizeof(filename), "%s/file.%u", - g_logfile_base_dir, ff->file_id); - char metafilename[PATH_MAX] = ""; - snprintf(metafilename, sizeof(metafilename), "%s.meta", filename); - FILE *fp = fopen(metafilename, "a"); - if (fp != NULL) { - fprintf(fp, "MAGIC: %s\n", - ff->magic ? ff->magic : ""); - - switch (ff->state) { - case FILE_STATE_CLOSED: - fprintf(fp, "STATE: CLOSED\n"); -#ifdef HAVE_NSS - if (ff->flags & FILE_MD5) { - fprintf(fp, "MD5: "); - size_t x; - for (x = 0; x < sizeof(ff->md5); x++) { - fprintf(fp, "%02x", ff->md5[x]); - } - fprintf(fp, "\n"); - } -#endif - break; - case FILE_STATE_TRUNCATED: - fprintf(fp, "STATE: TRUNCATED\n"); - break; - case FILE_STATE_ERROR: - fprintf(fp, "STATE: ERROR\n"); - break; - default: - fprintf(fp, "STATE: UNKNOWN\n"); - break; - } - fprintf(fp, "SIZE: %"PRIu64"\n", ff->size); - - fclose(fp); - } else { - SCLogInfo("opening %s failed: %s", metafilename, strerror(errno)); - } -} - /** * \internal * \brief Write meta data on a single line json record @@ -274,7 +170,11 @@ static void LogFileWriteJsonRecord(LogFileLogThread *aft, Packet *p, File *ff, i char timebuf[64]; CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); - fprintf(fp, "{ \"id\": %u, ", ff->file_id); + fprintf(fp, "{ "); + + if (ff->file_id > 0) + fprintf(fp, "\"id\": %u, ", ff->file_id); + fprintf(fp, "\"timestamp\": \""); PrintRawJsonFp(fp, (uint8_t *)timebuf, strlen(timebuf)); fprintf(fp, "\", "); @@ -359,6 +259,7 @@ static void LogFileWriteJsonRecord(LogFileLogThread *aft, Packet *p, File *ff, i fprintf(fp, "\"state\": \"UNKOWN\", "); break; } + fprintf(fp, "\"stored\": %s, ", ff->flags & FILE_STORED ? "true" : "false"); fprintf(fp, "\"size\": %"PRIu64" ", ff->size); fprintf(fp, "}\n"); fflush(fp); @@ -390,90 +291,23 @@ static TmEcode LogFileLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue if (ffc != NULL) { File *ff; for (ff = ffc->head; ff != NULL; ff = ff->next) { - int file_fd = -1; + if (ff->flags & FILE_LOGGED) + continue; if (FileForceMagic() && ff->magic == NULL) { FilemagicLookup(ff); } SCLogDebug("ff %p", ff); - if (ff->state == FILE_STATE_STORED) { - SCLogDebug("ff->state == FILE_STATE_STORED"); - continue; - } - - if (ff->store != 1) { - SCLogDebug("ff->store %d, so not 1", ff->store); - continue; - } - - FileData *ffd; - for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) { - SCLogDebug("ffd %p", ffd); - if (ffd->stored == 1) { - if (file_close == 1 && ffd->next == NULL) { - LogFileWriteJsonRecord(aft, p, ff, ipver); - LogFileLogCloseMetaFile(ff); - ff->state = FILE_STATE_STORED; - } - continue; - } - - /* store */ - SCLogDebug("trying to open file"); - - char filename[PATH_MAX] = ""; - - if (ff->file_id == 0) { - ff->file_id = SC_ATOMIC_ADD(file_id, 1); - - snprintf(filename, sizeof(filename), "%s/file.%u", - g_logfile_base_dir, ff->file_id); - - file_fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644); - if (file_fd == -1) { - SCLogDebug("failed to open file"); - continue; - } - - /* create a .meta file that contains time, src/dst/sp/dp/proto */ - LogFileLogCreateMetaFile(p, ff, filename, ipver); - aft->file_cnt++; - } else { - snprintf(filename, sizeof(filename), "%s/file.%u", - g_logfile_base_dir, ff->file_id); - - file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY); - if (file_fd == -1) { - SCLogDebug("failed to open file %s: %s", filename, strerror(errno)); - continue; - } - } - - ssize_t r = write(file_fd, (const void *)ffd->data, (size_t)ffd->len); - if (r == -1) { - SCLogDebug("write failed: %s", strerror(errno)); - - close(file_fd); - continue; - } - close(file_fd); - - if (ff->state == FILE_STATE_CLOSED || - ff->state == FILE_STATE_TRUNCATED || - ff->state == FILE_STATE_ERROR || + if (ff->state == FILE_STATE_CLOSED || + ff->state == FILE_STATE_TRUNCATED || ff->state == FILE_STATE_ERROR || (file_close == 1 && ff->state < FILE_STATE_CLOSED)) - { - if (ffd->next == NULL) { - LogFileLogCloseMetaFile(ff); - LogFileWriteJsonRecord(aft, p, ff, ipver); - - ff->state = FILE_STATE_STORED; - } - } + { + LogFileWriteJsonRecord(aft, p, ff, ipver); - ffd->stored = 1; + ff->flags |= FILE_LOGGED; + aft->file_cnt++; } } @@ -534,14 +368,6 @@ TmEcode LogFileLogThreadInit(ThreadVars *t, void *initdata, void **data) /* Use the Ouptut Context (file pointer and mutex) */ aft->file_ctx = ((OutputCtx *)initdata)->data; - struct stat stat_buf; - if (stat(g_logfile_base_dir, &stat_buf) != 0) { - SCLogError(SC_ERR_LOGDIR_CONFIG, "The file drop directory \"%s\" " - "supplied doesn't exist. Shutting down the engine", - g_logfile_base_dir); - exit(EXIT_FAILURE); - } - *data = (void *)aft; return TM_ECODE_OK; } @@ -566,60 +392,7 @@ void LogFileLogExitPrintStats(ThreadVars *tv, void *data) { return; } - SCLogInfo("(%s) Files extracted %" PRIu32 "", tv->name, aft->file_cnt); -} - -/** - * \internal - * - * \brief Open the waldo file (if available) and load the file_id - * - * \param path full path for the waldo file - */ -static void LogFileLogLoadWaldo(const char *path) { - char line[16] = ""; - unsigned int id = 0; - - FILE *fp = fopen(path, "r"); - if (fp == NULL) { - SCLogInfo("couldn't open waldo: %s", strerror(errno)); - SCReturn; - } - - if (fgets(line, (int)sizeof(line), fp) != NULL) { - if (sscanf(line, "%10u", &id) == 1) { - SCLogInfo("id %u", id); - SC_ATOMIC_CAS(&file_id, 0, id); - } - } - fclose(fp); -} - -/** - * \internal - * - * \brief Store the waldo file based on the file_id - * - * \param path full path for the waldo file - */ -static void LogFileLogStoreWaldo(const char *path) { - char line[16] = ""; - - if (SC_ATOMIC_GET(file_id) == 0) { - SCReturn; - } - - FILE *fp = fopen(path, "w"); - if (fp == NULL) { - SCLogInfo("couldn't open waldo: %s", strerror(errno)); - SCReturn; - } - - snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(file_id)); - if (fwrite(line, strlen(line), 1, fp) != 1) { - SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno)); - } - fclose(fp); + SCLogInfo("(%s) Files logged: %" PRIu32 "", tv->name, aft->file_cnt); } /** \brief Create a new http log LogFileCtx. @@ -646,49 +419,21 @@ static OutputCtx *LogFileLogInitCtx(ConfNode *conf) output_ctx->data = logfile_ctx; output_ctx->DeInit = LogFileLogDeInitCtx; - char *s_default_log_dir = NULL; - if (ConfGet("default-log-dir", &s_default_log_dir) != 1) - s_default_log_dir = DEFAULT_LOG_DIR; - - const char *s_base_dir = NULL; - s_base_dir = ConfNodeLookupChildValue(conf, "log-dir"); - if (s_base_dir == NULL || strlen(s_base_dir) == 0) { - strlcpy(g_logfile_base_dir, - s_default_log_dir, sizeof(g_logfile_base_dir)); - } else { - if (PathIsAbsolute(s_base_dir)) { - strlcpy(g_logfile_base_dir, - s_base_dir, sizeof(g_logfile_base_dir)); - } else { - snprintf(g_logfile_base_dir, sizeof(g_logfile_base_dir), - "%s/%s", s_default_log_dir, s_base_dir); - } - } - const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic"); if (force_magic != NULL && ConfValIsTrue(force_magic)) { FileForceMagicEnable(); - SCLogInfo("forcing magic lookup for stored files"); + SCLogInfo("forcing magic lookup for logged files"); } const char *force_md5 = ConfNodeLookupChildValue(conf, "force-md5"); if (force_md5 != NULL && ConfValIsTrue(force_md5)) { +#ifdef HAVE_NSS FileForceMd5Enable(); - SCLogInfo("forcing md5 calculation for stored files"); - } - - const char *waldo = ConfNodeLookupChildValue(conf, "waldo"); - if (waldo != NULL && strlen(waldo) > 0) { - if (PathIsAbsolute(waldo)) { - snprintf(g_waldo, sizeof(g_waldo), "%s", waldo); - } else { - snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo); - } - - SCLogInfo("loading waldo file %s", g_waldo); - LogFileLogLoadWaldo(g_waldo); + SCLogInfo("forcing md5 calculation for logged files"); +#else + SCLogInfo("md5 calculation requires linking against libnss"); +#endif } - SCLogInfo("storing files in %s", g_logfile_base_dir); SCReturnPtr(output_ctx, "OutputCtx"); } @@ -705,10 +450,6 @@ static void LogFileLogDeInitCtx(OutputCtx *output_ctx) LogFileCtx *logfile_ctx = (LogFileCtx *)output_ctx->data; LogFileFreeCtx(logfile_ctx); free(output_ctx); - - if (g_waldo != NULL) { - LogFileLogStoreWaldo(g_waldo); - } } /** \brief Read the config set the file pointer, open the file diff --git a/src/log-filestore.c b/src/log-filestore.c new file mode 100644 index 0000000000..c9abc7b029 --- /dev/null +++ b/src/log-filestore.c @@ -0,0 +1,617 @@ +/* Copyright (C) 2007-2012 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + */ + +#include "suricata-common.h" +#include "debug.h" +#include "detect.h" +#include "pkt-var.h" +#include "conf.h" + +#include "threadvars.h" +#include "tm-modules.h" + +#include "threads.h" + +#include "app-layer-parser.h" + +#include "detect-filemagic.h" + +#include "stream.h" + +#include "util-print.h" +#include "util-unittest.h" +#include "util-privs.h" +#include "util-debug.h" +#include "util-atomic.h" +#include "util-file.h" + +#include "output.h" + +#include "log-file.h" +#include "util-logopenfile.h" + +#include "app-layer-htp.h" + +#define MODULE_NAME "LogFilestoreLog" + +TmEcode LogFilestoreLog (ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); +TmEcode LogFilestoreLogIPv4(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); +TmEcode LogFilestoreLogIPv6(ThreadVars *, Packet *, void *, PacketQueue *, PacketQueue *); +TmEcode LogFilestoreLogThreadInit(ThreadVars *, void *, void **); +TmEcode LogFilestoreLogThreadDeinit(ThreadVars *, void *); +void LogFilestoreLogExitPrintStats(ThreadVars *, void *); +int LogFilestoreLogOpenFileCtx(LogFileCtx* , const char *, const char *); +static OutputCtx *LogFilestoreLogInitCtx(ConfNode *); +static void LogFilestoreLogDeInitCtx(OutputCtx *); + +SC_ATOMIC_DECLARE(unsigned int, file_id); +static char g_logfile_base_dir[PATH_MAX] = "/tmp"; +static char g_waldo[PATH_MAX] = ""; + +void TmModuleLogFilestoreRegister (void) { + tmm_modules[TMM_FILESTORE].name = MODULE_NAME; + tmm_modules[TMM_FILESTORE].ThreadInit = LogFilestoreLogThreadInit; + tmm_modules[TMM_FILESTORE].Func = LogFilestoreLog; + tmm_modules[TMM_FILESTORE].ThreadExitPrintStats = LogFilestoreLogExitPrintStats; + tmm_modules[TMM_FILESTORE].ThreadDeinit = LogFilestoreLogThreadDeinit; + tmm_modules[TMM_FILESTORE].RegisterTests = NULL; + tmm_modules[TMM_FILESTORE].cap_flags = 0; + + OutputRegisterModule(MODULE_NAME, "file", LogFilestoreLogInitCtx); + OutputRegisterModule(MODULE_NAME, "file-store", LogFilestoreLogInitCtx); + + SCLogDebug("registered"); + + SC_ATOMIC_INIT(file_id); +} + +typedef struct LogFilestoreLogThread_ { + LogFileCtx *file_ctx; + /** LogFilestoreCtx has the pointer to the file and a mutex to allow multithreading */ + uint32_t file_cnt; +} LogFilestoreLogThread; + +static void CreateTimeString (const struct timeval *ts, char *str, size_t size) { + time_t time = ts->tv_sec; + struct tm local_tm; + struct tm *t = (struct tm *)localtime_r(&time, &local_tm); + + snprintf(str, size, "%02d/%02d/%02d-%02d:%02d:%02d.%06u", + t->tm_mon + 1, t->tm_mday, t->tm_year + 1900, t->tm_hour, + t->tm_min, t->tm_sec, (uint32_t) ts->tv_usec); +} + +static void LogFilestoreMetaGetUri(FILE *fp, Packet *p, File *ff) { + HtpState *htp_state = (HtpState *)p->flow->alstate; + if (htp_state != NULL) { + htp_tx_t *tx = list_get(htp_state->connp->conn->transactions, ff->txid); + if (tx != NULL && tx->request_uri_normalized != NULL) { + PrintRawUriFp(fp, (uint8_t *)bstr_ptr(tx->request_uri_normalized), + bstr_len(tx->request_uri_normalized)); + return; + } + } + + fprintf(fp, ""); +} + +static void LogFilestoreMetaGetHost(FILE *fp, Packet *p, File *ff) { + HtpState *htp_state = (HtpState *)p->flow->alstate; + if (htp_state != NULL) { + htp_tx_t *tx = list_get(htp_state->connp->conn->transactions, ff->txid); + if (tx != NULL) { + table_t *headers; + headers = tx->request_headers; + htp_header_t *h = NULL; + + table_iterator_reset(headers); + while (table_iterator_next(headers, (void **)&h) != NULL) { + if (strcasecmp("Host", bstr_tocstr(h->name)) == 0) { + PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value), + bstr_len(h->value)); + return; + } + } + } + } + + fprintf(fp, ""); +} + +static void LogFilestoreMetaGetReferer(FILE *fp, Packet *p, File *ff) { + HtpState *htp_state = (HtpState *)p->flow->alstate; + if (htp_state != NULL) { + htp_tx_t *tx = list_get(htp_state->connp->conn->transactions, ff->txid); + if (tx != NULL) { + table_t *headers; + headers = tx->request_headers; + htp_header_t *h = NULL; + + table_iterator_reset(headers); + while (table_iterator_next(headers, (void **)&h) != NULL) { + if (strcasecmp("Referer", bstr_tocstr(h->name)) == 0) { + PrintRawUriFp(fp, (uint8_t *)bstr_ptr(h->value), + bstr_len(h->value)); + return; + } + } + } + } + + fprintf(fp, ""); +} + +static void LogFilestoreLogCreateMetaFile(Packet *p, File *ff, char *filename, int ipver) { + char metafilename[PATH_MAX] = ""; + snprintf(metafilename, sizeof(metafilename), "%s.meta", filename); + FILE *fp = fopen(metafilename, "w+"); + if (fp != NULL) { + char timebuf[64]; + + CreateTimeString(&p->ts, timebuf, sizeof(timebuf)); + + fprintf(fp, "TIME: %s\n", timebuf); + if (p->pcap_cnt > 0) { + fprintf(fp, "PCAP PKT NUM: %"PRIu64"\n", p->pcap_cnt); + } + + char srcip[46], dstip[46]; + Port sp, dp; + switch (ipver) { + case AF_INET: + PrintInet(AF_INET, (const void *)GET_IPV4_SRC_ADDR_PTR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET, (const void *)GET_IPV4_DST_ADDR_PTR(p), dstip, sizeof(dstip)); + break; + case AF_INET6: + PrintInet(AF_INET6, (const void *)GET_IPV6_SRC_ADDR(p), srcip, sizeof(srcip)); + PrintInet(AF_INET6, (const void *)GET_IPV6_DST_ADDR(p), dstip, sizeof(dstip)); + break; + default: + strlcpy(srcip, "", sizeof(srcip)); + strlcpy(dstip, "", sizeof(dstip)); + break; + } + sp = p->sp; + dp = p->dp; + + fprintf(fp, "SRC IP: %s\n", srcip); + fprintf(fp, "DST IP: %s\n", dstip); + fprintf(fp, "PROTO: %" PRIu32 "\n", p->proto); + if (PKT_IS_TCP(p) || PKT_IS_UDP(p)) { + fprintf(fp, "SRC PORT: %" PRIu16 "\n", sp); + fprintf(fp, "DST PORT: %" PRIu16 "\n", dp); + } + fprintf(fp, "HTTP URI: "); + LogFilestoreMetaGetUri(fp, p, ff); + fprintf(fp, "\n"); + fprintf(fp, "HTTP HOST: "); + LogFilestoreMetaGetHost(fp, p, ff); + fprintf(fp, "\n"); + fprintf(fp, "HTTP REFERER: "); + LogFilestoreMetaGetReferer(fp, p, ff); + fprintf(fp, "\n"); + fprintf(fp, "FILENAME: "); + PrintRawUriFp(fp, ff->name, ff->name_len); + fprintf(fp, "\n"); + + fclose(fp); + } +} + +static void LogFilestoreLogCloseMetaFile(File *ff) { + char filename[PATH_MAX] = ""; + snprintf(filename, sizeof(filename), "%s/file.%u", + g_logfile_base_dir, ff->file_id); + char metafilename[PATH_MAX] = ""; + snprintf(metafilename, sizeof(metafilename), "%s.meta", filename); + FILE *fp = fopen(metafilename, "a"); + if (fp != NULL) { + fprintf(fp, "MAGIC: %s\n", + ff->magic ? ff->magic : ""); + + switch (ff->state) { + case FILE_STATE_CLOSED: + fprintf(fp, "STATE: CLOSED\n"); +#ifdef HAVE_NSS + if (ff->flags & FILE_MD5) { + fprintf(fp, "MD5: "); + size_t x; + for (x = 0; x < sizeof(ff->md5); x++) { + fprintf(fp, "%02x", ff->md5[x]); + } + fprintf(fp, "\n"); + } +#endif + break; + case FILE_STATE_TRUNCATED: + fprintf(fp, "STATE: TRUNCATED\n"); + break; + case FILE_STATE_ERROR: + fprintf(fp, "STATE: ERROR\n"); + break; + default: + fprintf(fp, "STATE: UNKNOWN\n"); + break; + } + fprintf(fp, "SIZE: %"PRIu64"\n", ff->size); + + fclose(fp); + } else { + SCLogInfo("opening %s failed: %s", metafilename, strerror(errno)); + } +} + +static TmEcode LogFilestoreLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq, int ipver) +{ + SCEnter(); + LogFilestoreLogThread *aft = (LogFilestoreLogThread *)data; + uint8_t flags = 0; + + /* no flow, no htp state */ + if (p->flow == NULL) { + SCReturnInt(TM_ECODE_OK); + } + + if (p->flowflags & FLOW_PKT_TOCLIENT) + flags |= STREAM_TOCLIENT; + else + flags |= STREAM_TOSERVER; + + int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0; + + SCMutexLock(&p->flow->m); + + FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow, flags); + SCLogDebug("ffc %p", ffc); + if (ffc != NULL) { + File *ff; + for (ff = ffc->head; ff != NULL; ff = ff->next) { + int file_fd = -1; + + if (FileForceMagic() && ff->magic == NULL) { + FilemagicLookup(ff); + } + + SCLogDebug("ff %p", ff); + if (ff->flags & FILE_STORED) { + SCLogDebug("stored flag set"); + continue; + } + + if (ff->store != 1) { + SCLogDebug("ff->store %d, so not 1", ff->store); + continue; + } + + FileData *ffd; + for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) { + SCLogDebug("ffd %p", ffd); + if (ffd->stored == 1) { + if (file_close == 1 && ffd->next == NULL) { + LogFilestoreLogCloseMetaFile(ff); + ff->flags |= FILE_STORED; + } + continue; + } + + /* store */ + SCLogDebug("trying to open file"); + + char filename[PATH_MAX] = ""; + + if (ff->file_id == 0) { + ff->file_id = SC_ATOMIC_ADD(file_id, 1); + + snprintf(filename, sizeof(filename), "%s/file.%u", + g_logfile_base_dir, ff->file_id); + + file_fd = open(filename, O_CREAT | O_TRUNC | O_NOFOLLOW | O_WRONLY, 0644); + if (file_fd == -1) { + SCLogDebug("failed to open file"); + continue; + } + + /* create a .meta file that contains time, src/dst/sp/dp/proto */ + LogFilestoreLogCreateMetaFile(p, ff, filename, ipver); + aft->file_cnt++; + } else { + snprintf(filename, sizeof(filename), "%s/file.%u", + g_logfile_base_dir, ff->file_id); + + file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY); + if (file_fd == -1) { + SCLogDebug("failed to open file %s: %s", filename, strerror(errno)); + continue; + } + } + + ssize_t r = write(file_fd, (const void *)ffd->data, (size_t)ffd->len); + if (r == -1) { + SCLogDebug("write failed: %s", strerror(errno)); + + close(file_fd); + continue; + } + + close(file_fd); + + if (ff->state == FILE_STATE_CLOSED || + ff->state == FILE_STATE_TRUNCATED || + ff->state == FILE_STATE_ERROR || + (file_close == 1 && ff->state < FILE_STATE_CLOSED)) + { + if (ffd->next == NULL) { + LogFilestoreLogCloseMetaFile(ff); + + ff->flags |= FILE_STORED; + } + } + + ffd->stored = 1; + } + } + + FilePrune(ffc); + } + + SCMutexUnlock(&p->flow->m); + SCReturnInt(TM_ECODE_OK); +} + +TmEcode LogFilestoreLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { + return LogFilestoreLogWrap(tv, p, data, NULL, NULL, AF_INET); +} + +TmEcode LogFilestoreLogIPv6(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) { + return LogFilestoreLogWrap(tv, p, data, NULL, NULL, AF_INET6); +} + +TmEcode LogFilestoreLog (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq) +{ + SCEnter(); + int r = TM_ECODE_OK; + + /* no flow, no htp state */ + if (p->flow == NULL) { + SCReturnInt(TM_ECODE_OK); + } + + if (!(PKT_IS_TCP(p))) { + SCReturnInt(TM_ECODE_OK); + } + + SCLogDebug("p->pcap_cnt %"PRIu64, p->pcap_cnt); + + if (PKT_IS_IPV4(p)) { + r = LogFilestoreLogIPv4(tv, p, data, pq, postpq); + } else if (PKT_IS_IPV6(p)) { + r = LogFilestoreLogIPv6(tv, p, data, pq, postpq); + } + + SCReturnInt(r); +} + +TmEcode LogFilestoreLogThreadInit(ThreadVars *t, void *initdata, void **data) +{ + LogFilestoreLogThread *aft = SCMalloc(sizeof(LogFilestoreLogThread)); + if (aft == NULL) + return TM_ECODE_FAILED; + memset(aft, 0, sizeof(LogFilestoreLogThread)); + + if (initdata == NULL) + { + SCLogDebug("Error getting context for LogFilestore. \"initdata\" argument NULL"); + SCFree(aft); + return TM_ECODE_FAILED; + } + + /* Use the Ouptut Context (file pointer and mutex) */ + aft->file_ctx = ((OutputCtx *)initdata)->data; + + struct stat stat_buf; + if (stat(g_logfile_base_dir, &stat_buf) != 0) { + SCLogError(SC_ERR_LOGDIR_CONFIG, "The file drop directory \"%s\" " + "supplied doesn't exist. Shutting down the engine", + g_logfile_base_dir); + exit(EXIT_FAILURE); + } + + *data = (void *)aft; + return TM_ECODE_OK; +} + +TmEcode LogFilestoreLogThreadDeinit(ThreadVars *t, void *data) +{ + LogFilestoreLogThread *aft = (LogFilestoreLogThread *)data; + if (aft == NULL) { + return TM_ECODE_OK; + } + + /* clear memory */ + memset(aft, 0, sizeof(LogFilestoreLogThread)); + + SCFree(aft); + return TM_ECODE_OK; +} + +void LogFilestoreLogExitPrintStats(ThreadVars *tv, void *data) { + LogFilestoreLogThread *aft = (LogFilestoreLogThread *)data; + if (aft == NULL) { + return; + } + + SCLogInfo("(%s) Files extracted %" PRIu32 "", tv->name, aft->file_cnt); +} + +/** + * \internal + * + * \brief Open the waldo file (if available) and load the file_id + * + * \param path full path for the waldo file + */ +static void LogFilestoreLogLoadWaldo(const char *path) { + char line[16] = ""; + unsigned int id = 0; + + FILE *fp = fopen(path, "r"); + if (fp == NULL) { + SCLogInfo("couldn't open waldo: %s", strerror(errno)); + SCReturn; + } + + if (fgets(line, (int)sizeof(line), fp) != NULL) { + if (sscanf(line, "%10u", &id) == 1) { + SCLogInfo("id %u", id); + SC_ATOMIC_CAS(&file_id, 0, id); + } + } + fclose(fp); +} + +/** + * \internal + * + * \brief Store the waldo file based on the file_id + * + * \param path full path for the waldo file + */ +static void LogFilestoreLogStoreWaldo(const char *path) { + char line[16] = ""; + + if (SC_ATOMIC_GET(file_id) == 0) { + SCReturn; + } + + FILE *fp = fopen(path, "w"); + if (fp == NULL) { + SCLogInfo("couldn't open waldo: %s", strerror(errno)); + SCReturn; + } + + snprintf(line, sizeof(line), "%u\n", SC_ATOMIC_GET(file_id)); + if (fwrite(line, strlen(line), 1, fp) != 1) { + SCLogError(SC_ERR_FWRITE, "fwrite failed: %s", strerror(errno)); + } + fclose(fp); +} + +/** \brief Create a new http log LogFilestoreCtx. + * \param conf Pointer to ConfNode containing this loggers configuration. + * \return NULL if failure, LogFilestoreCtx* to the file_ctx if succesful + * */ +static OutputCtx *LogFilestoreLogInitCtx(ConfNode *conf) +{ + LogFileCtx *logfile_ctx = LogFileNewCtx(); + if (logfile_ctx == NULL) { + SCLogDebug("Could not create new LogFilestoreCtx"); + return NULL; + } + + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); + if (output_ctx == NULL) + return NULL; + + output_ctx->data = NULL; + output_ctx->DeInit = LogFilestoreLogDeInitCtx; + + char *s_default_log_dir = NULL; + if (ConfGet("default-log-dir", &s_default_log_dir) != 1) + s_default_log_dir = DEFAULT_LOG_DIR; + + const char *s_base_dir = NULL; + s_base_dir = ConfNodeLookupChildValue(conf, "log-dir"); + if (s_base_dir == NULL || strlen(s_base_dir) == 0) { + strlcpy(g_logfile_base_dir, + s_default_log_dir, sizeof(g_logfile_base_dir)); + } else { + if (PathIsAbsolute(s_base_dir)) { + strlcpy(g_logfile_base_dir, + s_base_dir, sizeof(g_logfile_base_dir)); + } else { + snprintf(g_logfile_base_dir, sizeof(g_logfile_base_dir), + "%s/%s", s_default_log_dir, s_base_dir); + } + } + + const char *force_magic = ConfNodeLookupChildValue(conf, "force-magic"); + if (force_magic != NULL && ConfValIsTrue(force_magic)) { + FileForceMagicEnable(); + SCLogInfo("forcing magic lookup for stored files"); + } + + const char *force_md5 = ConfNodeLookupChildValue(conf, "force-md5"); + if (force_md5 != NULL && ConfValIsTrue(force_md5)) { +#ifdef HAVE_NSS + FileForceMd5Enable(); + SCLogInfo("forcing md5 calculation for stored files"); +#else + SCLogInfo("md5 calculation requires linking against libnss"); +#endif + } + + const char *waldo = ConfNodeLookupChildValue(conf, "waldo"); + if (waldo != NULL && strlen(waldo) > 0) { + if (PathIsAbsolute(waldo)) { + snprintf(g_waldo, sizeof(g_waldo), "%s", waldo); + } else { + snprintf(g_waldo, sizeof(g_waldo), "%s/%s", s_default_log_dir, waldo); + } + + SCLogInfo("loading waldo file %s", g_waldo); + LogFilestoreLogLoadWaldo(g_waldo); + } + SCLogInfo("storing files in %s", g_logfile_base_dir); + + SCReturnPtr(output_ctx, "OutputCtx"); +} + +/** + * \internal + * + * \brief deinit the log ctx and write out the waldo + * + * \param output_ctx output context to deinit + */ +static void LogFilestoreLogDeInitCtx(OutputCtx *output_ctx) +{ + LogFileCtx *logfile_ctx = (LogFileCtx *)output_ctx->data; + LogFileFreeCtx(logfile_ctx); + free(output_ctx); + + if (g_waldo != NULL) { + LogFilestoreLogStoreWaldo(g_waldo); + } +} + +/** \brief Read the config set the file pointer, open the file + * \param file_ctx pointer to a created LogFilestoreCtx using LogFilestoreNewCtx() + * \param config_file for loading separate configs + * \return -1 if failure, 0 if succesful + * */ +int LogFilestoreLogOpenFileCtx(LogFileCtx *file_ctx, const char *filename, const + char *mode) +{ + return 0; +} diff --git a/src/log-filestore.h b/src/log-filestore.h new file mode 100644 index 0000000000..bfbfd5fcfa --- /dev/null +++ b/src/log-filestore.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __LOG_FILESTORE_H__ +#define __LOG_FILESTORE_H__ + +void TmModuleLogFilestoreRegister (void); + +#endif /* __LOG_FILELOG_H__ */ diff --git a/src/suricata.c b/src/suricata.c index dc36d85760..8b9895b964 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -97,6 +97,7 @@ #include "log-httplog.h" #include "log-pcap.h" #include "log-file.h" +#include "log-filestore.h" #include "stream-tcp.h" @@ -1363,6 +1364,7 @@ int main(int argc, char **argv) TmModuleLogHttpLogIPv6Register(); TmModulePcapLogRegister(); TmModuleLogFileLogRegister(); + TmModuleLogFilestoreRegister(); #ifdef __SC_CUDA_SUPPORT__ TmModuleCudaMpmB2gRegister(); TmModuleCudaPacketBatcherRegister(); diff --git a/src/tm-threads-common.h b/src/tm-threads-common.h index 1a1d23cafe..acf4528a95 100644 --- a/src/tm-threads-common.h +++ b/src/tm-threads-common.h @@ -57,6 +57,7 @@ typedef enum { TMM_LOGHTTPLOG6, TMM_PCAPLOG, TMM_FILELOG, + TMM_FILESTORE, TMM_STREAMTCP, TMM_DECODEIPFW, TMM_VERDICTIPFW, diff --git a/src/util-file.h b/src/util-file.h index b36f780134..bfecfd9043 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -34,6 +34,8 @@ #define FILE_NOMAGIC 0x04 #define FILE_STORE 0x08 #define FILE_MD5 0x10 +#define FILE_LOGGED 0x20 +#define FILE_STORED 0x40 typedef enum FileState_ { FILE_STATE_NONE = 0, /**< no state */ @@ -42,7 +44,6 @@ typedef enum FileState_ { there will be no more data. */ FILE_STATE_TRUNCATED, /**< flow file is not complete, but there will be no more data. */ - FILE_STATE_STORED, /**< all fully written to disk */ FILE_STATE_ERROR, /**< file is in an error state */ FILE_STATE_MAX } FileState; diff --git a/suricata.yaml.in b/suricata.yaml.in index 94427f342c..2adf0c46ae 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -156,12 +156,23 @@ outputs: # - stream reassembly depth. For optimal results, set this to 0 (unlimited) # - http request / response body sizes. Again set to 0 for optimal results. # - rules that contain the "filestore" keyword. - - file: + - file-store: enabled: no # set to yes to enable log-dir: files # directory to store the files force-magic: no # force logging magic on all stored files + force-md5: no # force logging of md5 checksums #waldo: file.waldo # waldo file to store the file_id across runs + # output module to log files tracked in a easily parsable json format + - file-log: + enabled: no + filename: files-json.log + append: yes + #filetype: regular # 'regular', 'unix_stream' or 'unix_dgram' + + force-magic: no # force logging magic on all logged files + force-md5: no # force logging of md5 checksums + # Magic file. The extension .mgc is added to the value here. #magic-file: /usr/share/file/magic magic-file: @e_magic_file@