From 18d79c4215c4b9d45e845ec48a680853e271f537 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 15 Dec 2011 17:01:53 +0100 Subject: [PATCH] file store: respect flowbits and other keywords The filestore keyword until now flagged a file, tx or ssn for storage as soon as the keyword was inspected. This happens before flowbits and some other keywords, so files were stored that weren't supposed to. This patch makes the filestore keyword fill an array in the detect engine thread ctx. Then if the full signature matches, a post-match filestore function makes the store final. --- src/detect-filestore.c | 232 +++++++++++++++++++++++++++-------------- src/detect-filestore.h | 1 + src/detect.c | 4 + src/detect.h | 34 +++--- src/util-file.c | 20 ++++ src/util-file.h | 1 + 6 files changed, 195 insertions(+), 97 deletions(-) diff --git a/src/detect-filestore.c b/src/detect-filestore.c index 128fdd13f9..590d048122 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -96,6 +96,146 @@ error: return; } +/** + * \brief apply the post match filestore with options + */ +static int FilestorePostMatchWithOptions(Packet *p, Flow *f, DetectFilestoreData *filestore, FileContainer *fc, + uint16_t file_id, uint16_t tx_id) +{ + if (filestore == NULL) { + SCReturnInt(0); + } + + int this_file = 0; + int this_tx = 0; + int this_flow = 0; + int rule_dir = 0; + int toserver_dir = 0; + int toclient_dir = 0; + + switch (filestore->direction) { + case FILESTORE_DIR_DEFAULT: + rule_dir = 1; + break; + case FILESTORE_DIR_BOTH: + toserver_dir = 1; + toclient_dir = 1; + break; + case FILESTORE_DIR_TOSERVER: + toserver_dir = 1; + break; + case FILESTORE_DIR_TOCLIENT: + toclient_dir = 1; + break; + } + + switch (filestore->scope) { + case FILESTORE_SCOPE_DEFAULT: + if (rule_dir) { + this_file = 1; + } else if (p->flowflags & FLOW_PKT_TOCLIENT && toclient_dir) { + this_file = 1; + } else if (p->flowflags & FLOW_PKT_TOSERVER && toserver_dir) { + this_file = 1; + } + break; + case FILESTORE_SCOPE_TX: + this_tx = 1; + break; + case FILESTORE_SCOPE_SSN: + this_flow = 1; + break; + } + + if (this_file) { + FileStoreFileById(fc, file_id); + } else if (this_tx) { + /* flag tx all files will be stored */ + if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) { + HtpState *htp_state = f->alstate; + if (toserver_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS; + FileStoreAllFilesForTx(htp_state->files_ts, tx_id); + } + if (toclient_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC; + FileStoreAllFilesForTx(htp_state->files_tc, tx_id); + } + htp_state->store_tx_id = tx_id; + } + } else if (this_flow) { + /* flag flow all files will be stored */ + if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) { + HtpState *htp_state = f->alstate; + if (toserver_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TS; + FileStoreAllFiles(htp_state->files_ts); + } + if (toclient_dir) { + htp_state->flags |= HTP_FLAG_STORE_FILES_TC; + FileStoreAllFiles(htp_state->files_tc); + } + } + } else { + FileStoreFileById(fc, file_id); + } + + SCReturnInt(0); +} + +/** + * \brief post-match function for filestore + * + * The match function for filestore records store candidates in the det_ctx. + * When we are sure all parts of the signature matched, we run this function + * to finalize the filestore. + */ +int DetectFilestorePostMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p) { + uint8_t flags = 0; + + SCEnter(); + + if (det_ctx->filestore_cnt == 0) { + SCReturnInt(0); + } + + if (det_ctx->filestore_sm == NULL || p->flow == NULL) { +#ifndef DEBUG + SCReturnInt(0); +#else + BUG_ON(1); +#endif + } + + if (p->flowflags & FLOW_PKT_TOCLIENT) + flags |= STREAM_TOCLIENT; + else + flags |= STREAM_TOSERVER; + + SCMutexLock(&p->flow->m); + + FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow, flags); + + /* filestore for single files only */ + if (det_ctx->filestore_sm->ctx == NULL) { + uint16_t u; + for (u = 0; u < det_ctx->filestore_cnt; u++) { + FileStoreFileById(ffc, det_ctx->filestore[u].file_id); + } + } else { + DetectFilestoreData *filestore = det_ctx->filestore_sm->ctx; + uint16_t u; + + for (u = 0; u < det_ctx->filestore_cnt; u++) { + FilestorePostMatchWithOptions(p, p->flow, filestore, ffc, + det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id); + } + } + + SCMutexUnlock(&p->flow->m); + SCReturnInt(0); +} + /** * \brief match the specified filestore * @@ -114,88 +254,22 @@ int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f uint8_t flags, void *state, Signature *s, SigMatch *m) { SCEnter(); - DetectFilestoreData *filestore = m->ctx; - if (filestore != NULL) { - int this_file = 0; - int this_tx = 0; - int this_flow = 0; - int rule_dir = 0; - int toserver_dir = 0; - int toclient_dir = 0; - - switch (filestore->direction) { - case FILESTORE_DIR_DEFAULT: - rule_dir = 1; - break; - case FILESTORE_DIR_BOTH: - toserver_dir = 1; - toclient_dir = 1; - break; - case FILESTORE_DIR_TOSERVER: - toserver_dir = 1; - break; - case FILESTORE_DIR_TOCLIENT: - toclient_dir = 1; - break; - } + if (det_ctx->filestore_cnt > DETECT_FILESTORE_MAX) { + SCReturnInt(1); + } - switch (filestore->scope) { - case FILESTORE_SCOPE_DEFAULT: - if (rule_dir) { - this_file = 1; - } else if (flags & STREAM_TOCLIENT && toclient_dir) { - this_file = 1; - } else if (flags & STREAM_TOSERVER && toserver_dir) { - this_file = 1; - } - break; - case FILESTORE_SCOPE_TX: - this_tx = 1; - break; - case FILESTORE_SCOPE_SSN: - this_flow = 1; - break; - } + File *file = (File *)state; - if (this_file) { - File *file = (File *)state; - FileStore(file); - } else if (this_tx) { - /* flag tx all files will be stored */ - if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) { - HtpState *htp_state = f->alstate; - if (toserver_dir) { - htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS; - FileStoreAllFilesForTx(htp_state->files_ts, det_ctx->tx_id); - } - if (toclient_dir) { - htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC; - FileStoreAllFilesForTx(htp_state->files_tc, det_ctx->tx_id); - } - htp_state->store_tx_id = det_ctx->tx_id; - } + det_ctx->filestore[det_ctx->filestore_cnt].file_id = file->file_id; + det_ctx->filestore[det_ctx->filestore_cnt].tx_id = det_ctx->tx_id; - } else if (this_flow) { - /* flag flow all files will be stored */ - if (f->alproto == ALPROTO_HTTP && f->alstate != NULL) { - HtpState *htp_state = f->alstate; - if (toserver_dir) { - htp_state->flags |= HTP_FLAG_STORE_FILES_TS; - FileStoreAllFiles(htp_state->files_ts); - } - if (toclient_dir) { - htp_state->flags |= HTP_FLAG_STORE_FILES_TC; - FileStoreAllFiles(htp_state->files_tc); - } - } - } else { - File *file = (File *)state; - FileStore(file); - } - } else { - File *file = (File *)state; - FileStore(file); - } + SCLogDebug("%u, file %u, tx %u", det_ctx->filestore_cnt, + det_ctx->filestore[det_ctx->filestore_cnt].file_id, + det_ctx->filestore[det_ctx->filestore_cnt].tx_id); + + det_ctx->filestore_cnt++; + + det_ctx->filestore_sm = m; SCReturnInt(1); } diff --git a/src/detect-filestore.h b/src/detect-filestore.h index f394842fbd..cd3f014400 100644 --- a/src/detect-filestore.h +++ b/src/detect-filestore.h @@ -41,4 +41,5 @@ typedef struct DetectFilestoreData_ { /* prototypes */ void DetectFilestoreRegister (void); +int DetectFilestorePostMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx, Packet *p); #endif /* __DETECT_FILESTORE_H__ */ diff --git a/src/detect.c b/src/detect.c index 9793339425..8f755875f5 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1218,6 +1218,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh p->alerts.cnt = 0; det_ctx->pkts++; + det_ctx->filestore_cnt = 0; /* No need to perform any detection on this packet, if the the given flag is set.*/ if (p->flags & PKT_NOPACKET_INSPECTION) { @@ -1583,6 +1584,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh DetectReplaceExecute(p, det_ctx->replist); det_ctx->replist = NULL; + DetectFilestorePostMatch(th_v, det_ctx,p); if (!(s->flags & SIG_FLAG_NOALERT)) { PacketAlertAppend(det_ctx, s, p, alert_flags, NULL); } @@ -1604,6 +1606,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh DetectReplaceExecute(p, det_ctx->replist); det_ctx->replist = NULL; + DetectFilestorePostMatch(th_v, det_ctx,p); if (!(s->flags & SIG_FLAG_NOALERT)) { /* only add once */ if (rmatch == 0) { @@ -1644,6 +1647,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh fmatch = 1; DetectReplaceExecute(p, det_ctx->replist); det_ctx->replist = NULL; + DetectFilestorePostMatch(th_v, det_ctx,p); if (!(s->flags & SIG_FLAG_NOALERT)) { PacketAlertAppend(det_ctx, s, p, alert_flags, alert_msg); diff --git a/src/detect.h b/src/detect.h index e49d3b502a..1068283446 100644 --- a/src/detect.h +++ b/src/detect.h @@ -662,6 +662,8 @@ enum { ENGINE_SGH_MPM_FACTORY_CONTEXT_AUTO }; +#define DETECT_FILESTORE_MAX 15 + /** * Detection engine thread data. */ @@ -677,33 +679,20 @@ typedef struct DetectionEngineThreadCtx_ { /* used by pcre match function alone */ uint32_t pcre_match_start_offset; - /* http_uri stuff for uricontent */ - //char de_have_httpuri; - //char de_mpm_scanned_uri; - - /* detectione engine context for hcbd mpm */ - //char de_have_hcbd; - //char de_mpm_scanned_hcbd; - - /* detectione engine context for hhd mpm */ - //char de_have_hhd; - //char de_mpm_scanned_hhd; - - /* detectione engine context for hrhd mpm */ - //char de_have_hrhd; - //char de_mpm_scanned_hrhd; - uint8_t **hcbd_buffers; uint32_t *hcbd_buffers_len; uint16_t hcbd_buffers_list_len; + /* counter for the filestore array below -- up here for cache reasons. */ + uint16_t filestore_cnt; + + uint16_t hhd_buffers_list_len; + uint16_t hsbd_buffers_list_len; uint8_t **hsbd_buffers; uint32_t *hsbd_buffers_len; - uint16_t hsbd_buffers_list_len; uint8_t **hhd_buffers; uint32_t *hhd_buffers_len; - uint16_t hhd_buffers_list_len; /** id for alert counter */ uint16_t counter_alerts; @@ -773,6 +762,15 @@ typedef struct DetectionEngineThreadCtx_ { /* string to replace */ DetectReplaceList *replist; + /* Array in which the filestore keyword stores file id and tx id. If the + * full signature matches, these are processed by a post-match filestore + * function to finalize the store. */ + struct { + uint16_t file_id; + uint16_t tx_id; + } filestore[DETECT_FILESTORE_MAX]; + SigMatch *filestore_sm; + DetectEngineCtx *de_ctx; #ifdef __SC_CUDA_SUPPORT__ /* each detection thread would have it's own queue where the cuda dispatcher diff --git a/src/util-file.c b/src/util-file.c index 3b3b68d9cd..f3c5497d1d 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -636,6 +636,26 @@ void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint16_t tx_id SCReturn; } +/** + * \brief flag a file with id "file_id" to be stored. + * + * \param fc file store + * \param file_id the file's id + */ +void FileStoreFileById(FileContainer *fc, uint16_t file_id) { + File *ptr = NULL; + + SCEnter(); + + if (fc != NULL) { + for (ptr = fc->head; ptr != NULL; ptr = ptr->next) { + if (ptr->file_id == file_id) { + ptr->store = 1; + } + } + } +} + void FileStoreAllFilesForTx(FileContainer *fc, uint16_t tx_id) { File *ptr = NULL; diff --git a/src/util-file.h b/src/util-file.h index ba692107d3..bac71a361b 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -163,5 +163,6 @@ int FileForceMagic(void); void FileStoreAllFiles(FileContainer *); void FileStoreAllFilesForTx(FileContainer *, uint16_t); +void FileStoreFileById(FileContainer *fc, uint16_t); #endif /* __UTIL_FILE_H__ */