diff --git a/src/Makefile.am b/src/Makefile.am index 947c8f55af..f190a5bd03 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -83,6 +83,7 @@ detect-engine-hmd.c detect-engine-hmd.h \ detect-engine-hcd.c detect-engine-hcd.h \ detect-engine-hrud.c detect-engine-hrud.h \ detect-engine-state.c detect-engine-state.h \ +detect-engine-file.c detect-engine-file.h \ detect-parse.c detect-parse.h \ detect-ack.c detect-ack.h \ detect-seq.c detect-seq.h \ @@ -152,6 +153,7 @@ detect-http-stat-msg.c detect-http-stat-msg.h \ detect-asn1.c detect-asn1.h \ detect-filename.c detect-filename.h \ detect-fileext.c detect-fileext.h \ +detect-filestore.c detect-filestore.h \ detect-http-stat-code.c detect-http-stat-code.h \ detect-ssl-version.c detect-ssl-version.h \ detect-ssl-state.c detect-ssl-state.h \ diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index 1ffee598ab..b1d22621d9 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -72,18 +72,26 @@ * \param data data chunk (if any) * \param data_len length of the data portion * - * \retval 0 ok + * \retval 0 ok * \retval -1 error + * \retval -2 not handling files on this flow */ int HTPFileOpen(Flow *f, uint8_t *filename, uint16_t filename_len, uint8_t *data, uint32_t data_len) { int retval = 0; + uint16_t txid; if (f == NULL) { - return -1; + SCReturnInt(-1); + } + + if (f->flags & FLOW_FILE_NO_HANDLING) { + SCReturnInt(-2); } + txid = AppLayerTransactionGetAvailId(f) - 1; + SCMutexLock(&f->files_m); { if (f->files == NULL) { @@ -99,11 +107,13 @@ int HTPFileOpen(Flow *f, uint8_t *filename, uint16_t filename_len, { retval = -1; } + + FlowFileSetTx(f->files->tail, txid); } end: SCMutexUnlock(&f->files_m); - return retval; + SCReturnInt(retval); } /** @@ -115,16 +125,22 @@ end: * * \retval 0 ok * \retval -1 error + * \retval -2 file doesn't need storing */ int HTPFileStoreChunk(Flow *f, uint8_t *data, uint32_t data_len) { SCEnter(); int retval = 0; + int result = 0; if (f == NULL) { SCReturnInt(-1); } + if (f->flags & FLOW_FILE_NO_HANDLING) { + SCReturnInt(-2); + } + SCMutexLock(&f->files_m); { if (f->files == NULL) { @@ -133,10 +149,12 @@ int HTPFileStoreChunk(Flow *f, uint8_t *data, uint32_t data_len) { goto end; } - if (FlowFileAppendData(f->files, data, data_len) == -1) - { + result = FlowFileAppendData(f->files, data, data_len); + if (result == -1) { SCLogDebug("appending data failed"); retval = -1; + } else if (result == -2) { + retval = -2; } } @@ -158,12 +176,14 @@ end: * * \retval 0 ok * \retval -1 error + * \retval -2 not storing files on this flow/tx */ int HTPFileClose(Flow *f, uint8_t *data, uint32_t data_len, uint8_t flags) { int retval = 0; + int result = 0; if (f == NULL) { - return -1; + SCReturnInt(-1); } SCMutexLock(&f->files_m); @@ -173,15 +193,17 @@ int HTPFileClose(Flow *f, uint8_t *data, uint32_t data_len, uint8_t flags) { goto end; } - if (FlowFileCloseFile(f->files, data, data_len, flags) == -1) - { + result = FlowFileCloseFile(f->files, data, data_len, flags); + if (result == -1) { retval = -1; + } else if (result == -2) { + retval = -2; } } end: SCMutexUnlock(&f->files_m); - return retval; + SCReturnInt(retval); } #ifdef UNITTESTS diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 98fd056637..c25541d4a7 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1020,17 +1020,18 @@ static void HtpRequestBodyReassemble(SCHtpTxUserData *htud, int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, uint8_t *chunks_buffer, uint32_t chunks_buffer_len) { + int result = 0; + uint8_t *expected_boundary = NULL; + uint8_t *expected_boundary_end = NULL; + uint8_t expected_boundary_len = 0; + uint8_t expected_boundary_end_len = 0; + #ifdef PRINT printf("CHUNK START: \n"); PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len); printf("CHUNK END: \n"); #endif - uint8_t *expected_boundary = NULL; - uint8_t *expected_boundary_end = NULL; - uint8_t expected_boundary_len = 0; - uint8_t expected_boundary_end_len = 0; - if (HtpRequestBodySetupBoundary(htud, &expected_boundary, &expected_boundary_len, &expected_boundary_end, &expected_boundary_end_len) < 0) { goto end; @@ -1073,9 +1074,11 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, PrintRawDataFp(stdout, filedata, filedata_len); printf("FILEDATA (final chunk) END: \n"); #endif - if (HTPFileClose(hstate->f, filedata, filedata_len, flags) == -1) - { - goto end; + if (!(htud->flags & HTP_DONTSTORE)) { + if (HTPFileClose(hstate->f, filedata, filedata_len, flags) == -1) + { + goto end; + } } htud->flags &=~ HTP_FILENAME_SET; @@ -1092,8 +1095,15 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, PrintRawDataFp(stdout, filedata, filedata_len); printf("FILEDATA (part) END: \n"); #endif - if (HTPFileStoreChunk(hstate->f, filedata, filedata_len) == -1) { - goto end; + + if (!(htud->flags & HTP_DONTSTORE)) { + result = HTPFileStoreChunk(hstate->f, filedata, filedata_len); + if (result == -1) { + goto end; + } else if (result == -2) { + /* we know for sure we're not storing the file */ + htud->flags |= HTP_DONTSTORE; + } } htud->body_parsed += filedata_len; @@ -1132,6 +1142,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, SCLogDebug("we have a filename"); htud->flags |= HTP_FILENAME_SET; + htud->flags &= ~HTP_DONTSTORE; SCLogDebug("header_end %p", header_end); SCLogDebug("form_end %p", form_end); @@ -1156,12 +1167,16 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, printf("FILEDATA END: \n"); #endif - if (HTPFileOpen(hstate->f, filename, filename_len, - filedata, filedata_len) == -1) { - goto end; - } - if (HTPFileClose(hstate->f, NULL, 0, 0) == -1) { + result = HTPFileOpen(hstate->f, filename, filename_len, + filedata, filedata_len); + if (result == -1) { goto end; + } else if (result == -2) { + htud->flags |= HTP_DONTSTORE; + } else { + if (HTPFileClose(hstate->f, NULL, 0, 0) == -1) { + goto end; + } } } else { SCLogDebug("more file data to come"); @@ -1170,9 +1185,12 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud, SCLogDebug("offset %u", offset); htud->body_parsed = offset; - if (HTPFileOpen(hstate->f, filename, filename_len, - filedata, filedata_len) == -1) { + result = HTPFileOpen(hstate->f, filename, filename_len, + filedata, filedata_len); + if (result == -1) { goto end; + } else if (result == -2) { + htud->flags |= HTP_DONTSTORE; } } @@ -1217,6 +1235,8 @@ int HtpRequestBodySetupPUT(htp_tx_data_t *d, SCHtpTxUserData *htud) { int HtpRequestBodyHandlePUT(HtpState *hstate, SCHtpTxUserData *htud, htp_tx_t *tx, uint8_t *data, uint32_t data_len) { + int result = 0; + /* see if we need to open the file */ if (!(htud->flags & HTP_FILENAME_SET)) { @@ -1229,19 +1249,29 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, SCHtpTxUserData *htud, filename_len = bstr_len(tx->parsed_uri->path); } - if (HTPFileOpen(hstate->f, filename, filename_len, - data, data_len) == -1) { + result = HTPFileOpen(hstate->f, filename, filename_len, + data, data_len); + if (result == -1) { goto end; + } else if (result == -2) { + htud->flags |= HTP_DONTSTORE; + } else { + htud->flags |= HTP_FILENAME_SET; + htud->flags &= ~HTP_DONTSTORE; } - - htud->flags |= HTP_FILENAME_SET; } else { /* otherwise, just store the data */ - if (HTPFileStoreChunk(hstate->f, data, data_len) == -1) { - goto end; + if (!(htud->flags & HTP_DONTSTORE)) { + result = HTPFileStoreChunk(hstate->f, data, data_len); + if (result == -1) { + goto end; + } else if (result == -2) { + /* we know for sure we're not storing the file */ + htud->flags |= HTP_DONTSTORE; + } } } @@ -1264,7 +1294,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) SCReturnInt(HOOK_ERROR); } - SCLogDebug("New response body data available at %p -> %p -> %p, bodylen " + SCLogDebug("New request body data available at %p -> %p -> %p, bodylen " "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len); SCHtpTxUserData *htud = (SCHtpTxUserData *) htp_tx_get_user_data(d->tx); @@ -1274,15 +1304,15 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) SCReturnInt(HOOK_OK); } memset(htud, 0, sizeof(SCHtpTxUserData)); - htud->body.operation = HTP_BODY_NONE; + htud->body.operation = HTP_BODY_REQUEST; if (d->tx->request_method_number == M_POST) { if (HtpRequestBodySetupMultipart(d, htud) == 0) { - htud->body.operation = HTP_BODY_REQUEST_MULTIPART; + htud->body.type = HTP_BODY_REQUEST_MULTIPART; } } else if (d->tx->request_method_number == M_PUT) { if (HtpRequestBodySetupPUT(d, htud) == 0) { - htud->body.operation = HTP_BODY_REQUEST_PUT; + htud->body.type = HTP_BODY_REQUEST_PUT; } } @@ -1320,7 +1350,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) uint8_t *chunks_buffer = NULL; uint32_t chunks_buffer_len = 0; - if (htud->body.operation == HTP_BODY_REQUEST_MULTIPART) { + if (htud->body.type == HTP_BODY_REQUEST_MULTIPART) { /* multi-part body handling starts here */ if (!(htud->flags & HTP_BOUNDARY_SET)) { goto end; @@ -1336,7 +1366,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) if (chunks_buffer != NULL) { SCFree(chunks_buffer); } - } else if (htud->body.operation == HTP_BODY_REQUEST_PUT) { + } else if (htud->body.type == HTP_BODY_REQUEST_PUT) { HtpRequestBodyHandlePUT(hstate, htud, d->tx, (uint8_t *)d->data, (uint32_t)d->len); } diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index 735d7a91d3..ad39ca4a35 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -56,16 +56,20 @@ has parsed a new body (for pcre) */ enum { - HTP_BODY_NONE, /**< Flag to indicate the current + HTP_BODY_NONE = 0, /**< Flag to indicate the current operation */ HTP_BODY_REQUEST, /**< Flag to indicate that the current operation is a request */ - HTP_BODY_REQUEST_MULTIPART, - HTP_BODY_REQUEST_PUT, HTP_BODY_RESPONSE /**< Flag to indicate that the current * operation is a response */ }; +enum { + HTP_BODY_REQUEST_NONE = 0, + HTP_BODY_REQUEST_MULTIPART, + HTP_BODY_REQUEST_PUT, +}; + #define HTP_PCRE_NONE 0x00 /**< No pcre executed yet */ #define HTP_PCRE_DONE 0x01 /**< Flag to indicate that pcre has done some inspection in the @@ -89,6 +93,7 @@ typedef struct HtpBody_ { uint32_t nchunks; /**< Number of chunks in the current operation */ uint8_t operation; /**< This flag indicate if it's a request or a response */ + uint8_t type; /* pahole: padding: 3 */ } HtpBody; @@ -99,6 +104,17 @@ typedef struct HtpBody_ { #define HTP_BOUNDARY_SET 0x04 /**< We have a boundary string */ #define HTP_BOUNDARY_OPEN 0x08 /**< We have a boundary string */ #define HTP_FILENAME_SET 0x10 /**< filename is registered in the flow */ +#define HTP_DONTSTORE 0x20 /**< not storing this file */ + +#define HTP_TX_HAS_FILE 0x01 +#define HTP_TX_HAS_FILENAME 0x02 /**< filename is known at this time */ +#define HTP_TX_HAS_TYPE 0x04 +#define HTP_TX_HAS_FILECONTENT 0x08 /**< file has content so we can do type detect */ + +#define HTP_RULE_NEED_FILE HTP_TX_HAS_FILE +#define HTP_RULE_NEED_FILENAME HTP_TX_HAS_FILENAME +#define HTP_RULE_NEED_TYPE HTP_TX_HAS_TYPE +#define HTP_RULE_NEED_FILECONTENT HTP_TX_HAS_FILECONTENT /** Now the Body Chunks will be stored per transaction, at * the tx user data */ diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index f16037332b..37fd2eb912 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -1016,6 +1016,26 @@ error: SCReturnInt(-1); } +uint16_t AppLayerTransactionGetAvailId(Flow *f) { + SCEnter(); + + /* Get the parser state (if any) */ + if (f->aldata == NULL) { + SCLogDebug("no aldata"); + SCReturnUInt(0); + } + + AppLayerParserStateStore *parser_state_store = + (AppLayerParserStateStore *)f->aldata[app_layer_sid]; + + if (parser_state_store == NULL) { + SCLogDebug("no state store"); + SCReturnUInt(0); + } + + SCReturnUInt(parser_state_store->avail_id); +} + /** \brief get the highest loggable transaction id */ int AppLayerTransactionGetLoggableId(Flow *f) { SCEnter(); diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index e748eb1237..a5028757eb 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -267,10 +267,12 @@ int AppLayerTransactionGetLoggableId(Flow *f); int AppLayerTransactionGetLoggedId(Flow *f); int AppLayerTransactionGetBaseId(Flow *f); int AppLayerTransactionGetInspectId(Flow *f); -void AppLayerSetEOF(Flow *); +uint16_t AppLayerTransactionGetAvailId(Flow *f); uint16_t AppLayerGetStateVersion(Flow *f); +void AppLayerSetEOF(Flow *); + /* cleanup */ void AppLayerParserCleanupState(Flow *); void AppLayerFreeProbingParsers(AppLayerProbingParser *); diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c new file mode 100644 index 0000000000..92a8804c68 --- /dev/null +++ b/src/detect-engine-file.c @@ -0,0 +1,186 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#include "suricata-common.h" + +#include "decode.h" + +#include "detect.h" +#include "detect-engine.h" +#include "detect-parse.h" +#include "detect-engine-state.h" + +#include "detect-engine-uri.h" +#include "detect-engine-hcbd.h" +#include "detect-engine-hhd.h" +#include "detect-engine-hrhd.h" +#include "detect-engine-hmd.h" +#include "detect-engine-hcd.h" +#include "detect-engine-hrud.h" +#include "detect-engine-dcepayload.h" + +#include "stream-tcp.h" +#include "stream-tcp-private.h" +#include "stream-tcp-reassemble.h" + +#include "app-layer-parser.h" +#include "app-layer-protos.h" +#include "app-layer-htp.h" +#include "app-layer-smb.h" +#include "app-layer-dcerpc-common.h" +#include "app-layer-dcerpc.h" + +#include "util-unittest.h" +#include "util-unittest-helper.h" +#include "util-profiling.h" + + +/** + * \brief Inspect the file inspecting keywords. + * + * \param tv thread vars + * \param det_ctx detection engine thread ctx + * \param f flow + * \param s signature to inspect + * + * \retval 0 no match + * \retval 1 match + * \retval 2 can't match + * \retval 3 can't match filestore signature + * + * \note flow is not locked at this time + */ +static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s) { + SigMatch *sm = NULL; + int r = 0; + int match = 0; + + SCLogDebug("file inspection..."); + + SCMutexLock(&f->files_m); + if (f->files != NULL) { + FlowFile *file = f->files->head; + for (; file != NULL; file = file->next) { + SCLogDebug("file"); + + if (file->state == FLOWFILE_STATE_NONE) { + SCLogDebug("file state FLOWFILE_STATE_NONE"); + continue; + } + + if (file->txid < det_ctx->tx_id) { + SCLogDebug("file->txid < det_ctx->tx_id == %u < %u", file->txid, det_ctx->tx_id); + continue; + } + + if (file->txid > det_ctx->tx_id) { + SCLogDebug("file->txid > det_ctx->tx_id == %u > %u", file->txid, det_ctx->tx_id); + break; + } + + if (s->file_flags & FILE_SIG_NEED_FILENAME && file->name == NULL) { + SCLogDebug("sig needs filename, but we don't have any"); + r = 0; + break; + } + + /* run the file match functions. */ + for (sm = s->sm_lists[DETECT_SM_LIST_FILEMATCH]; sm != NULL; sm = sm->next) { + SCLogDebug("sm %p, sm->next %p", sm, sm->next); + + if (sigmatch_table[sm->type].AppLayerMatch != NULL) { + match = sigmatch_table[sm->type]. + AppLayerMatch(tv, det_ctx, f, 0, (void *)file, s, sm); + if (match == 0) { + r = 2; + break; + } else if (sm->next == NULL) { + r = 1; + break; + } + } + } + + if (r == 1) + break; + + /* if this is a filestore sig, and the sig can't match + * return 3 so we can distinguish */ + if (s->init_flags & SIG_FLAG_FILESTORE && r == 2) + r = 3; + } + } + + SCMutexUnlock(&f->files_m); + SCReturnInt(r); +} + +int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s, void *alstate) { + SCEnter(); + + int r = 0; + HtpState *htp_state = NULL; + size_t idx = 0; + size_t start_tx = 0; + size_t end_tx = 0; + int match = 0; + + htp_state = (HtpState *)alstate; + if (htp_state == NULL) { + SCLogDebug("no HTTP state"); + SCReturnInt(0); + } + + /* locking the flow, we will inspect the htp state */ + SCMutexLock(&f->m); + if (htp_state->connp != NULL && htp_state->connp->conn != NULL) + { + start_tx = AppLayerTransactionGetInspectId(f); + end_tx = list_size(htp_state->connp->conn->transactions); + + } + SCMutexUnlock(&f->m); + + for (idx = start_tx ; idx < end_tx; idx++) + { + /* inspect files for this transaction */ + det_ctx->tx_id = (uint16_t)idx; + + match = DetectFileInspect(tv, det_ctx, f, s); + if (match == 1) { + r = 1; + } else if (match == 2) { + if (r != 1) { + SCLogDebug("sid %u can't match on this transaction", s->id); + r = 2; + } + } else if (match == 3) { + if (r != 1) { + SCLogDebug("sid %u can't match on this transaction (filestore sig)", s->id); + r = 3; + } + } + } + + SCReturnInt(r); +} diff --git a/src/detect-engine-file.h b/src/detect-engine-file.h new file mode 100644 index 0000000000..b8cb1a2f03 --- /dev/null +++ b/src/detect-engine-file.h @@ -0,0 +1,29 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __DETECT_ENGINE_FILE_H__ +#define __DETECT_ENGINE_FILE_H__ + +int DetectFileInspectHttp(ThreadVars *, DetectEngineThreadCtx *, Flow *, Signature *, void *); + +#endif /* __DETECT_ENGINE_FILE_H__ */ diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 437d9e5a07..6fc1273df1 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -1585,6 +1585,32 @@ int SigGroupHeadBuildMatchArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh, return 0; } +/** + * \brief Set the filestore_cnt in the sgh. + * + * \param de_ctx detection engine ctx for the signatures + * \param sgh sig group head to set the counter in + */ +void SigGroupHeadSetFilestoreCount(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { + Signature *s = NULL; + uint32_t sig = 0; + + if (sgh == NULL) + return; + + for (sig = 0; sig < sgh->sig_cnt; sig++) { + s = sgh->match_array[sig]; + if (s == NULL) + continue; + + if (SignatureIsFilestoring(s)) { + sgh->filestore_cnt++; + } + } + + return; +} + int SigGroupHeadBuildHeadArray(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { Signature *s = NULL; diff --git a/src/detect-engine-siggroup.h b/src/detect-engine-siggroup.h index 0d33a91279..5054a7b263 100644 --- a/src/detect-engine-siggroup.h +++ b/src/detect-engine-siggroup.h @@ -84,5 +84,8 @@ void SigGroupHeadRegisterTests(void); void SigGroupHeadPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh); void SigGroupHeadStore(DetectEngineCtx *, SigGroupHead *); + int SigGroupHeadBuildHeadArray(DetectEngineCtx *, SigGroupHead *); +void SigGroupHeadSetFilestoreCount(DetectEngineCtx *, SigGroupHead *); + #endif /* __DETECT_ENGINE_SIGGROUP_H__ */ diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c index 5a3cb0a3cd..2dfb59f4c7 100644 --- a/src/detect-engine-state.c +++ b/src/detect-engine-state.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2011 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -64,6 +64,7 @@ #include "detect-engine-hcd.h" #include "detect-engine-hrud.h" #include "detect-engine-dcepayload.h" +#include "detect-engine-file.h" #include "stream-tcp.h" #include "stream-tcp-private.h" @@ -83,6 +84,22 @@ /** convert enum to string */ #define CASE_CODE(E) case E: return #E +int DeStateStoreFilestoreSigsCantMatch(SigGroupHead *sgh, + DetectEngineState *de_state, uint8_t direction) +{ + if (direction & STREAM_TOSERVER) { + if (de_state->toserver_filestore_cnt == sgh->filestore_cnt) { + SCReturnInt(1); + } + } else if (direction & STREAM_TOCLIENT) { + if (de_state->toclient_filestore_cnt == sgh->filestore_cnt) { + SCReturnInt(1); + } + } + + SCReturnInt(0); +} + /** \brief get string for match enum */ const char *DeStateMatchResultToString(DeStateMatchResult res) { @@ -91,6 +108,7 @@ const char *DeStateMatchResultToString(DeStateMatchResult res) CASE_CODE (DE_STATE_MATCH_FULL); CASE_CODE (DE_STATE_MATCH_PARTIAL); CASE_CODE (DE_STATE_MATCH_NEW); + CASE_CODE (DE_STATE_MATCH_NOMATCH); } return NULL; @@ -313,6 +331,26 @@ void DeStateStoreStateVersion(DetectEngineState *de_state, uint8_t direction, } } +/** + * \brief Increment de_state filestore_cnt in the proper direction. + * + * \param de_state flow's locked de_state + * \param direction flags containing direction + * \param file_no_match number of sigs that are identified as "can't match" + * with filestore. + */ +void DeStateStoreFileNoMatch(DetectEngineState *de_state, uint8_t direction, + uint16_t file_no_match) +{ + if (direction & STREAM_TOSERVER) { + SCLogDebug("STREAM_TOSERVER added %"PRIu16, file_no_match); + de_state->toserver_filestore_cnt += file_no_match; + } else { + SCLogDebug("STREAM_TOCLIENT added %"PRIu16, file_no_match); + de_state->toclient_filestore_cnt += file_no_match; + } +} + /** * \brief Check if a flow already contains a flow detect state * @@ -326,9 +364,9 @@ int DeStateFlowHasState(Flow *f, uint8_t flags, uint16_t alversion) { int r = 0; SCMutexLock(&f->de_state_m); - if (f->de_state == NULL || f->de_state->cnt == 0) + if (f->de_state == NULL || f->de_state->cnt == 0) { r = 0; - else if (DeStateGetStateVersion(f->de_state, flags) == alversion) + } else if (DeStateGetStateVersion(f->de_state, flags) == alversion) r = 2; else r = 1; @@ -353,6 +391,7 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, int r = 0; uint16_t inspect_flags = 0; uint16_t match_flags = 0; + uint16_t file_no_match = 0; if (alstate == NULL) { SCReturnInt(0); @@ -425,6 +464,19 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, } SCLogDebug("inspecting http raw uri"); } + if (s->sm_lists[DETECT_SM_LIST_FILEMATCH] != NULL) { + inspect_flags |= DE_STATE_FLAG_FILE_INSPECT; + + match = DetectFileInspectHttp(tv, det_ctx, f, s, alstate); + if (match == 1) { + match_flags |= DE_STATE_FLAG_FILE_MATCH; + } else if (match == 2) { + match_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; + } else if (match == 3) { + match_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; + file_no_match++; + } + } } else if (flags & STREAM_TOCLIENT) { /* For to client set the flags in inspect so it can't match * if the sig requires something only the request has. The rest @@ -465,6 +517,9 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, if (s->sm_lists[DETECT_SM_LIST_HRUDMATCH] != NULL) { inspect_flags |= DE_STATE_FLAG_HRUD_INSPECT; } + if (s->sm_lists[DETECT_SM_LIST_FILEMATCH] != NULL) { + inspect_flags |= DE_STATE_FLAG_FILE_INSPECT; + } } } else if (alproto == ALPROTO_DCERPC || alproto == ALPROTO_SMB || alproto == ALPROTO_SMB2) { if (s->sm_lists[DETECT_SM_LIST_DMATCH] != NULL) { @@ -548,6 +603,7 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, /* \todo shift to an array to transfer these match values*/ DeStateSignatureAppend(f->de_state, s, sm, match_flags); DeStateStoreStateVersion(f->de_state, flags, alversion); + DeStateStoreFileNoMatch(f->de_state, flags, file_no_match); } SCMutexUnlock(&f->de_state_m); @@ -568,6 +624,7 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete uint16_t inspect_flags = 0; uint16_t match_flags = 0; int match = 0; + uint16_t file_no_match = 0; if (f == NULL || alstate == NULL || alproto == ALPROTO_UNKNOWN) { return 0; @@ -578,10 +635,6 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete if (f->de_state == NULL || f->de_state->cnt == 0) goto end; - if (DeStateGetStateVersion(f->de_state, flags) == alversion) { - goto end; - } - /* loop through the stores */ for (store = f->de_state->head; store != NULL; store = store->next) { @@ -611,6 +664,12 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete goto next_sig; } + /* if we know for sure we can't ever match, detect that here */ + if (item->flags & DE_STATE_FLAG_SIG_CANT_MATCH) { + det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NOMATCH; + goto next_sig; + } + /* let's continue detection */ /* first, check uricontent */ @@ -705,6 +764,22 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete } } + if (s->sm_lists[DETECT_SM_LIST_FILEMATCH] != NULL) { + if (!(item->flags & DE_STATE_FLAG_FILE_MATCH)) { + inspect_flags |= DE_STATE_FLAG_FILE_INSPECT; + + match = DetectFileInspectHttp(tv, det_ctx, f, s, alstate); + if (match == 1) { + match_flags |= DE_STATE_FLAG_FILE_MATCH; + } else if (match == 2) { + match_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; + } else if (match == 3) { + match_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; + file_no_match++; + } + } + } + } else if (alproto == ALPROTO_DCERPC || alproto == ALPROTO_SMB || alproto == ALPROTO_SMB2) { if (s->sm_lists[DETECT_SM_LIST_DMATCH] != NULL) { if (!(item->flags & DE_STATE_FLAG_DCE_MATCH)) { @@ -803,6 +878,16 @@ next_sig: } DeStateStoreStateVersion(f->de_state, flags, alversion); + DeStateStoreFileNoMatch(f->de_state, flags, file_no_match); + + if (!(f->de_state->flags & DE_STATE_FILE_STORE_DISABLED)) { + if (DeStateStoreFilestoreSigsCantMatch(det_ctx->sgh, f->de_state, flags) == 1) { + SCLogDebug("disabling file storage for transaction"); + FlowFileDisableStoringForTransaction(f, det_ctx->tx_id); + f->de_state->flags |= DE_STATE_FILE_STORE_DISABLED; + } + } + end: SCMutexUnlock(&f->de_state_m); SCReturnInt(0); diff --git a/src/detect-engine-state.h b/src/detect-engine-state.h index 8de5d6232c..724dec86a0 100644 --- a/src/detect-engine-state.h +++ b/src/detect-engine-state.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2011 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -48,18 +48,21 @@ #define __DETECT_ENGINE_STATE_H__ /** number of DeStateStoreItem's in one DeStateStore object */ -#define DE_STATE_CHUNK_SIZE 15 - -#define DE_STATE_FLAG_PAYLOAD_MATCH 0x0001 /**< payload part of the sig matched */ -#define DE_STATE_FLAG_URI_MATCH 0x0002 /**< uri part of the sig matched */ -#define DE_STATE_FLAG_DCE_MATCH 0x0004 /**< dce payload inspection part matched */ -#define DE_STATE_FLAG_HCBD_MATCH 0x0008 /**< hcbd payload inspection part matched */ -#define DE_STATE_FLAG_HHD_MATCH 0x0010 /**< hhd payload inspection part matched */ -#define DE_STATE_FLAG_HRHD_MATCH 0x0020 /**< hrhd payload inspection part matched */ -#define DE_STATE_FLAG_HMD_MATCH 0x0040 /**< hmd payload inspection part matched */ -#define DE_STATE_FLAG_HCD_MATCH 0x0080 /**< hcd payload inspection part matched */ -#define DE_STATE_FLAG_HRUD_MATCH 0x0100 /**< hrud payload inspection part matched */ -#define DE_STATE_FLAG_FULL_MATCH 0x0200 /**< sig already fully matched */ +#define DE_STATE_CHUNK_SIZE 15 + +/* per stored sig flags */ +#define DE_STATE_FLAG_PAYLOAD_MATCH 0x0001 /**< payload part of the sig matched */ +#define DE_STATE_FLAG_URI_MATCH 0x0002 /**< uri part of the sig matched */ +#define DE_STATE_FLAG_DCE_MATCH 0x0004 /**< dce payload inspection part matched */ +#define DE_STATE_FLAG_HCBD_MATCH 0x0008 /**< hcbd payload inspection part matched */ +#define DE_STATE_FLAG_HHD_MATCH 0x0010 /**< hhd payload inspection part matched */ +#define DE_STATE_FLAG_HRHD_MATCH 0x0020 /**< hrhd payload inspection part matched */ +#define DE_STATE_FLAG_HMD_MATCH 0x0040 /**< hmd payload inspection part matched */ +#define DE_STATE_FLAG_HCD_MATCH 0x0080 /**< hcd payload inspection part matched */ +#define DE_STATE_FLAG_HRUD_MATCH 0x0100 /**< hrud payload inspection part matched */ +#define DE_STATE_FLAG_FILE_MATCH 0x0200 +#define DE_STATE_FLAG_FULL_MATCH 0x0400 /**< sig already fully matched */ +#define DE_STATE_FLAG_SIG_CANT_MATCH 0x0800 /**< signature has no chance of matching */ #define DE_STATE_FLAG_URI_INSPECT DE_STATE_FLAG_URI_MATCH /**< uri part of the sig inspected */ #define DE_STATE_FLAG_DCE_INSPECT DE_STATE_FLAG_DCE_MATCH /**< dce payload inspection part inspected */ @@ -69,7 +72,10 @@ #define DE_STATE_FLAG_HMD_INSPECT DE_STATE_FLAG_HMD_MATCH /**< hmd payload inspection part inspected */ #define DE_STATE_FLAG_HCD_INSPECT DE_STATE_FLAG_HCD_MATCH /**< hcd payload inspection part inspected */ #define DE_STATE_FLAG_HRUD_INSPECT DE_STATE_FLAG_HRUD_MATCH /**< hrud payload inspection part inspected */ +#define DE_STATE_FLAG_FILE_INSPECT DE_STATE_FLAG_FILE_MATCH +/* state flags */ +#define DE_STATE_FILE_STORE_DISABLED 0x0001 /** per signature detection engine state */ typedef enum { @@ -77,6 +83,7 @@ typedef enum { DE_STATE_MATCH_FULL, /**< sig already fully matched */ DE_STATE_MATCH_PARTIAL, /**< partial state match */ DE_STATE_MATCH_NEW, /**< new (full) match this run */ + DE_STATE_MATCH_NOMATCH, /**< not a match */ } DeStateMatchResult; /** \brief State storage for a single signature */ @@ -95,11 +102,18 @@ typedef struct DeStateStore_ { /** \brief State store main object */ typedef struct DetectEngineState_ { - DeStateStore *head; /**< signature state storage */ - DeStateStore *tail; /**< tail item of the storage list */ - SigIntId cnt; /**< number of sigs in the storage */ - uint16_t toclient_version; - uint16_t toserver_version; + DeStateStore *head; /**< signature state storage */ + DeStateStore *tail; /**< tail item of the storage list */ + SigIntId cnt; /**< number of sigs in the storage */ + uint16_t toclient_version; /**< app layer state "version" inspected + * last in to client direction */ + uint16_t toserver_version; /**< app layer state "version" inspected + * last in to server direction */ + uint16_t toclient_filestore_cnt;/**< number of sigs with filestore that + * cannot match in to client direction. */ + uint16_t toserver_filestore_cnt;/**< number of sigs with filestore that + * cannot match in to server direction. */ + uint16_t flags; } DetectEngineState; void DeStateRegisterTests(void); @@ -111,8 +125,6 @@ void DetectEngineStateReset(DetectEngineState *state); DetectEngineState *DetectEngineStateAlloc(void); void DetectEngineStateFree(DetectEngineState *); -//void DeStateSignatureAppend(DetectEngineState *, Signature *, SigMatch *, char); - int DeStateFlowHasState(Flow *, uint8_t, uint16_t); int DeStateDetectStartDetection(ThreadVars *, DetectEngineCtx *, diff --git a/src/detect-fileext.c b/src/detect-fileext.c index c7dfc022b8..710cecfaea 100644 --- a/src/detect-fileext.c +++ b/src/detect-fileext.c @@ -89,43 +89,36 @@ int DetectFileextMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, int ret = 0; DetectFileextData *fileext = (DetectFileextData *)m->ctx; + FlowFile *file = (FlowFile *)state; - SCMutexLock(&f->files_m); - if (f->files != NULL) { - FlowFile *file = f->files->head; + if (file->name == NULL) + SCReturnInt(0); - for (; file != NULL; file = file->next) { - if (file->state == FLOWFILE_STATE_NONE) - continue; + if (file->txid < det_ctx->tx_id) + SCReturnInt(0); - if (file->name == NULL) - continue; + if (file->txid > det_ctx->tx_id) + SCReturnInt(0); - if (file->name_len <= fileext->len) - continue; + if (file->name_len <= fileext->len) + SCReturnInt(0); - int offset = file->name_len - fileext->len; + int offset = file->name_len - fileext->len; - if (file->name[offset - 1] == '.' && - SCMemcmp(file->name + offset, fileext->ext, fileext->len) == 0) - { - if (!(fileext->flags & DETECT_CONTENT_NEGATED)) { - ret = 1; - SCLogDebug("File ext found"); - - /* Stop searching */ - break; - } - } - } - - if (ret == 0 && fileext->flags & DETECT_CONTENT_NEGATED) { - SCLogDebug("negated match"); + if (file->name[offset - 1] == '.' && + SCMemcmp(file->name + offset, fileext->ext, fileext->len) == 0) + { + if (!(fileext->flags & DETECT_CONTENT_NEGATED)) { ret = 1; + SCLogDebug("File ext found"); } } - SCMutexUnlock(&f->files_m); + if (ret == 0 && fileext->flags & DETECT_CONTENT_NEGATED) { + SCLogDebug("negated match"); + ret = 1; + } + SCReturnInt(ret); } @@ -204,7 +197,8 @@ static int DetectFileextSetup (DetectEngineCtx *de_ctx, Signature *s, char *str) sm->type = DETECT_FILEEXT; sm->ctx = (void *)fileext; - SigMatchAppendAppLayer(s, sm); + SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH); + if (s->alproto != ALPROTO_UNKNOWN && s->alproto != ALPROTO_HTTP) { SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords."); @@ -213,6 +207,7 @@ static int DetectFileextSetup (DetectEngineCtx *de_ctx, Signature *s, char *str) AppLayerHtpNeedFileInspection(); s->alproto = ALPROTO_HTTP; + s->file_flags |= (FILE_SIG_NEED_FILE|FILE_SIG_NEED_FILENAME); return 0; error: diff --git a/src/detect-filename.c b/src/detect-filename.c index 40838ff7af..c6a80ea342 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -88,45 +88,40 @@ int DetectFilenameMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, int ret = 0; DetectFilenameData *filename = m->ctx; + FlowFile *file = (FlowFile *)state; - SCMutexLock(&f->files_m); - if (f->files != NULL) { - FlowFile *file = f->files->head; - for (; file != NULL; file = file->next) { - if (file->state == FLOWFILE_STATE_NONE) - continue; + if (file->name == NULL) + SCReturnInt(0); - if (file->name == NULL) - continue; + if (file->txid < det_ctx->tx_id) + SCReturnInt(0); - if (BoyerMooreNocase(filename->name, filename->len, file->name, + if (file->txid > det_ctx->tx_id) + SCReturnInt(0); + + if (BoyerMooreNocase(filename->name, filename->len, file->name, file->name_len, filename->bm_ctx->bmGs, filename->bm_ctx->bmBc) != NULL) - { + { #ifdef DEBUG - if (SCLogDebugEnabled()) { - char *name = SCMalloc(filename->len + 1); - memcpy(name, filename->name, filename->len); - name[filename->len] = '\0'; - SCLogDebug("will look for filename %s", name); - } -#endif - - if (!(filename->flags & DETECT_CONTENT_NEGATED)) { - ret = 1; - /* Stop searching */ - break; - } - } + if (SCLogDebugEnabled()) { + char *name = SCMalloc(filename->len + 1); + memcpy(name, filename->name, filename->len); + name[filename->len] = '\0'; + SCLogDebug("will look for filename %s", name); } +#endif - if (ret == 0 && filename->flags & DETECT_CONTENT_NEGATED) { - SCLogDebug("negated match"); + if (!(filename->flags & DETECT_CONTENT_NEGATED)) { ret = 1; } } - SCMutexUnlock(&f->files_m); + if (ret == 0 && filename->flags & DETECT_CONTENT_NEGATED) { + SCLogDebug("negated match"); + ret = 1; + } + SCReturnInt(ret); } @@ -210,7 +205,7 @@ static int DetectFilenameSetup (DetectEngineCtx *de_ctx, Signature *s, char *str sm->type = DETECT_FILENAME; sm->ctx = (void *)filename; - SigMatchAppendAppLayer(s, sm); + SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH); if (s->alproto != ALPROTO_UNKNOWN && s->alproto != ALPROTO_HTTP) { SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords."); @@ -220,6 +215,8 @@ static int DetectFilenameSetup (DetectEngineCtx *de_ctx, Signature *s, char *str AppLayerHtpNeedFileInspection(); s->alproto = ALPROTO_HTTP; + + s->file_flags |= (FILE_SIG_NEED_FILE|FILE_SIG_NEED_FILENAME); return 0; error: diff --git a/src/detect-filestore.c b/src/detect-filestore.c new file mode 100644 index 0000000000..67534e76ca --- /dev/null +++ b/src/detect-filestore.c @@ -0,0 +1,131 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + */ + +#include "suricata-common.h" +#include "threads.h" +#include "debug.h" +#include "decode.h" + +#include "detect.h" +#include "detect-parse.h" + +#include "detect-engine.h" +#include "detect-engine-mpm.h" +#include "detect-engine-state.h" + +#include "flow.h" +#include "flow-var.h" +#include "flow-util.h" + +#include "util-debug.h" +#include "util-spm-bm.h" +#include "util-unittest.h" +#include "util-unittest-helper.h" + +#include "app-layer.h" + +#include "stream-tcp.h" + +#include "detect-filestore.h" + +int DetectFilestoreMatch (ThreadVars *, DetectEngineThreadCtx *, Flow *, uint8_t, void *, Signature *, SigMatch *); +static int DetectFilestoreSetup (DetectEngineCtx *, Signature *, char *); + +/** + * \brief Registration function for keyword: filestore + */ +void DetectFilestoreRegister(void) { + sigmatch_table[DETECT_FILESTORE].name = "filestore"; + sigmatch_table[DETECT_FILESTORE].Match = NULL; + sigmatch_table[DETECT_FILESTORE].AppLayerMatch = DetectFilestoreMatch; + sigmatch_table[DETECT_FILESTORE].alproto = ALPROTO_HTTP; + sigmatch_table[DETECT_FILESTORE].Setup = DetectFilestoreSetup; + sigmatch_table[DETECT_FILESTORE].Free = NULL; + sigmatch_table[DETECT_FILESTORE].RegisterTests = NULL; + + SCLogDebug("registering filestore rule option"); + return; +} + +/** + * \brief match the specified filestore + * + * \param t pointer to thread vars + * \param det_ctx pointer to the pattern matcher thread + * \param p pointer to the current packet + * \param m pointer to the sigmatch that we will cast into DetectFilestoreData + * + * \retval 0 no match + * \retval 1 match + */ +int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, + uint8_t flags, void *state, Signature *s, SigMatch *m) +{ + SCEnter(); + FlowFile *file = (FlowFile *)state; + FlowFileStore(file); + SCReturnInt(1); +} + +/** + * \brief this function is used to parse filestore options + * \brief into the current signature + * + * \param de_ctx pointer to the Detection Engine Context + * \param s pointer to the Current Signature + * \param str pointer to the user provided "filestore" option + * + * \retval 0 on Success + * \retval -1 on Failure + */ +static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, char *str) +{ + SigMatch *sm = NULL; + + sm = SigMatchAlloc(); + if (sm == NULL) + goto error; + + sm->type = DETECT_FILESTORE; + sm->ctx = NULL; + + SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH); + + if (s->alproto != ALPROTO_UNKNOWN && s->alproto != ALPROTO_HTTP) { + SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords."); + goto error; + } + + AppLayerHtpNeedFileInspection(); + + s->alproto = ALPROTO_HTTP; + + s->init_flags |= SIG_FLAG_FILESTORE; + return 0; + +error: + if (sm != NULL) + SCFree(sm); + return -1; +} diff --git a/src/detect-filestore.h b/src/detect-filestore.h new file mode 100644 index 0000000000..bacc1e6d4b --- /dev/null +++ b/src/detect-filestore.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __DETECT_FILESTORE_H__ +#define __DETECT_FILESTORE_H__ + +/* prototypes */ +void DetectFilestoreRegister (void); + +#endif /* __DETECT_FILESTORE_H__ */ diff --git a/src/detect-parse.c b/src/detect-parse.c index a00c7dacb7..97cf013bce 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -1510,6 +1510,8 @@ Signature *SigInit(DetectEngineCtx *de_ctx, char *sigstr) { sig->flags |= SIG_FLAG_STATE_MATCH; if (sig->sm_lists[DETECT_SM_LIST_HRUDMATCH]) sig->flags |= SIG_FLAG_STATE_MATCH; + if (sig->sm_lists[DETECT_SM_LIST_FILEMATCH]) + sig->flags |= SIG_FLAG_STATE_MATCH; SCLogDebug("sig %"PRIu32" SIG_FLAG_APPLAYER: %s, SIG_FLAG_PACKET: %s", sig->id, sig->flags & SIG_FLAG_APPLAYER ? "set" : "not set", @@ -1712,6 +1714,8 @@ Signature *SigInitReal(DetectEngineCtx *de_ctx, char *sigstr) { sig->flags |= SIG_FLAG_STATE_MATCH; if (sig->sm_lists[DETECT_SM_LIST_HCDMATCH]) sig->flags |= SIG_FLAG_STATE_MATCH; + if (sig->sm_lists[DETECT_SM_LIST_FILEMATCH]) + sig->flags |= SIG_FLAG_STATE_MATCH; SigBuildAddressMatchArray(sig); diff --git a/src/detect.c b/src/detect.c index 13a4b4b979..e36c55d6c6 100644 --- a/src/detect.c +++ b/src/detect.c @@ -92,6 +92,7 @@ #include "detect-asn1.h" #include "detect-filename.h" #include "detect-fileext.h" +#include "detect-filestore.h" #include "detect-dsize.h" #include "detect-flowvar.h" #include "detect-flowint.h" @@ -1772,6 +1773,17 @@ end: p->flow->sgh_toclient = det_ctx->sgh; p->flow->flags |= FLOW_SGH_TOCLIENT; } + + /* if we know both sides of the flow have had their sgh check + * and both are null, we will never decide to store. So disable + * storage completely. */ + if (p->flow->flags & FLOW_SGH_TOCLIENT && + p->flow->flags & FLOW_SGH_TOSERVER && + (p->flow->sgh_toserver == NULL || p->flow->sgh_toserver->filestore_cnt == 0) && + (p->flow->sgh_toclient == NULL || p->flow->sgh_toclient->filestore_cnt == 0)) + { + FlowFileDisableStoring(p->flow); + } } /* if we had no alerts that involved the smsgs, @@ -1889,6 +1901,24 @@ int SignatureIsAppLayer(DetectEngineCtx *de_ctx, Signature *s) { return 0; } +/** + * \brief Check if a signature contains the filestore keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFilestoring(Signature *s) { + if (s == NULL) + return 0; + + if (s->init_flags & SIG_FLAG_FILESTORE) + return 1; + + return 0; +} + /** \brief Test is a initialized signature is IP only * \param de_ctx detection engine ctx * \param s the signature @@ -3734,10 +3764,14 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) { continue; SigGroupHeadBuildHeadArray(de_ctx, sgh); + SigGroupHeadSetFilestoreCount(de_ctx, sgh); + SCLogInfo("filestore count %u", sgh->filestore_cnt); } if (de_ctx->decoder_event_sgh != NULL) { SigGroupHeadBuildHeadArray(de_ctx, de_ctx->decoder_event_sgh); + /* no need to set filestore count here as that would make a + * signature not decode event only. */ } SCFree(de_ctx->sgh_array); @@ -4298,6 +4332,7 @@ void SigTableSetup(void) { DetectByteExtractRegister(); DetectFilenameRegister(); DetectFileextRegister(); + DetectFilestoreRegister(); uint8_t i = 0; for (i = 0; i < DETECT_TBLSIZE; i++) { diff --git a/src/detect.h b/src/detect.h index a5d942b5ab..f993bec4de 100644 --- a/src/detect.h +++ b/src/detect.h @@ -98,6 +98,8 @@ enum { DETECT_SM_LIST_HCDMATCH, /* list for http_raw_uri keyword and the ones relative to it */ DETECT_SM_LIST_HRUDMATCH, + + DETECT_SM_LIST_FILEMATCH, DETECT_SM_LIST_MAX, }; @@ -268,6 +270,7 @@ typedef struct DetectPort_ { #define SIG_FLAG_FLOW (1<<2) /**< signature has a flow setting */ #define SIG_FLAG_BIDIREC (1<<3) /**< signature has bidirectional operator */ #define SIG_FLAG_PAYLOAD (1<<4) /**< signature is inspecting the packet payload */ +#define SIG_FLAG_FILESTORE (1<<5) /**< signature has filestore keyword */ /* signature mask flags */ #define SIG_MASK_REQUIRE_PAYLOAD 1 @@ -285,6 +288,10 @@ typedef struct DetectPort_ { #define DETECT_ENGINE_THREAD_CTX_INSPECTING_PACKET 0x0001 #define DETECT_ENGINE_THREAD_CTX_INSPECTING_STREAM 0x0002 +#define FILE_SIG_NEED_FILE 0x01 +#define FILE_SIG_NEED_FILENAME 0x02 +#define FILE_SIG_NEED_TYPE 0x04 +#define FILE_SIG_NEED_FILECONTENT 0x08 /* Detection Engine flags */ #define DE_QUIET 0x01 /**< DE is quiet (esp for unittests) */ @@ -462,6 +469,8 @@ typedef struct Signature_ { /* holds all sm lists' tails */ struct SigMatch_ *sm_lists_tail[DETECT_SM_LIST_MAX]; + uint8_t file_flags; + /** address settings for this signature */ DetectAddressHead src, dst; @@ -774,6 +783,9 @@ typedef struct DetectionEngineThreadCtx_ { PatternMatcherQueue pmq; PatternMatcherQueue smsg_pmq[256]; + /** ID of the transaction currently being inspected. */ + uint16_t tx_id; + /* counters */ uint32_t pkts; uint32_t pkts_searched; @@ -903,6 +915,10 @@ typedef struct SigGroupHead_ { uint16_t mpm_streamcontent_maxlen; uint16_t mpm_uricontent_maxlen; + + /** the number of signatures in this sgh that have the filestore keyword + * set. */ + uint16_t filestore_cnt; #if __WORDSIZE == 64 uint32_t pad2; #endif @@ -1025,6 +1041,7 @@ enum { DETECT_FILENAME, DETECT_FILEEXT, + DETECT_FILESTORE, /* make sure this stays last */ DETECT_TBLSIZE, @@ -1057,5 +1074,7 @@ SigGroupHead *SigMatchSignaturesGetSgh(DetectEngineCtx *de_ctx, DetectEngineThre Signature *DetectGetTagSignature(void); +int SignatureIsFilestoring(Signature *); + #endif /* __DETECT_H__ */ diff --git a/src/flow-file.c b/src/flow-file.c index aad4f71555..84bd735744 100644 --- a/src/flow-file.c +++ b/src/flow-file.c @@ -138,14 +138,9 @@ static void FlowFileDataFree(FlowFileData *ffd) { SCFree(ffd); } -static int FlowFileAppendFlowFileData(FlowFileContainer *ffc, FlowFileData *ffd) { +static int FlowFileAppendFlowFileDataFilePtr(FlowFile *ff, FlowFileData *ffd) { SCEnter(); - if (ffc == NULL) { - SCReturnInt(-1); - } - - FlowFile *ff = ffc->tail; if (ff == NULL) { SCReturnInt(-1); } @@ -161,6 +156,21 @@ static int FlowFileAppendFlowFileData(FlowFileContainer *ffc, FlowFileData *ffd) SCReturnInt(0); } +static int FlowFileAppendFlowFileData(FlowFileContainer *ffc, FlowFileData *ffd) { + SCEnter(); + + if (ffc == NULL) { + SCReturnInt(-1); + } + + if (FlowFileAppendFlowFileDataFilePtr(ffc->tail, ffd) == -1) + { + SCReturnInt(-1); + } + + SCReturnInt(0); +} + /** * \brief Alloc a new FlowFile * @@ -262,27 +272,16 @@ FlowFile *FlowFileOpenFile(FlowFileContainer *ffc, uint8_t *name, SCReturnPtr(ff, "FlowFile"); } -/** - * \brief Close a FlowFile - * - * \param ffc the container - * \param data final data if any - * \param data_len data len if any - * \param flags flags - * - * \retval 0 ok - * \retval -1 error - */ -int FlowFileCloseFile(FlowFileContainer *ffc, uint8_t *data, +static int FlowFileCloseFilePtr(FlowFile *ff, uint8_t *data, uint32_t data_len, uint8_t flags) { SCEnter(); - if (ffc == NULL || ffc->tail == NULL) { + if (ff == NULL) { SCReturnInt(-1); } - if (ffc->tail->state != FLOWFILE_STATE_OPENED) { + if (ff->state != FLOWFILE_STATE_OPENED) { SCReturnInt(-1); } @@ -291,29 +290,60 @@ int FlowFileCloseFile(FlowFileContainer *ffc, uint8_t *data, FlowFileData *ffd = FlowFileDataAlloc(data, data_len); if (ffd == NULL) { - ffc->tail->state = FLOWFILE_STATE_ERROR; + ff->state = FLOWFILE_STATE_ERROR; SCReturnInt(-1); } /* append the data */ - if (FlowFileAppendFlowFileData(ffc, ffd) < 0) { - ffc->tail->state = FLOWFILE_STATE_ERROR; + if (FlowFileAppendFlowFileDataFilePtr(ff, ffd) < 0) { + ff->state = FLOWFILE_STATE_ERROR; FlowFileDataFree(ffd); SCReturnInt(-1); } } if (flags & FLOW_FILE_TRUNCATED) { - ffc->tail->state = FLOWFILE_STATE_TRUNCATED; + ff->state = FLOWFILE_STATE_TRUNCATED; SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_TRUNCATED"); + + if (flags & FLOW_FILE_NOSTORE) { + ff->store = -1; + } } else { - ffc->tail->state = FLOWFILE_STATE_CLOSED; + ff->state = FLOWFILE_STATE_CLOSED; SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_CLOSED"); } SCReturnInt(0); } +/** + * \brief Close a FlowFile + * + * \param ffc the container + * \param data final data if any + * \param data_len data len if any + * \param flags flags + * + * \retval 0 ok + * \retval -1 error + */ +int FlowFileCloseFile(FlowFileContainer *ffc, uint8_t *data, + uint32_t data_len, uint8_t flags) +{ + SCEnter(); + + if (ffc == NULL || ffc->tail == NULL) { + SCReturnInt(-1); + } + + if (FlowFileCloseFilePtr(ffc->tail, data, data_len, flags) == -1) { + SCReturnInt(-1); + } + + SCReturnInt(0); +} + /** * \brief Store a chunk of file data in the flow. The open "flowfile" * will be used. @@ -322,8 +352,9 @@ int FlowFileCloseFile(FlowFileContainer *ffc, uint8_t *data, * \param data data chunk * \param data_len data chunk len * - * \retval 0 ok + * \retval 0 ok * \retval -1 error + * \retval -2 no store for this file */ int FlowFileAppendData(FlowFileContainer *ffc, uint8_t *data, uint32_t data_len) { SCEnter(); @@ -333,9 +364,14 @@ int FlowFileAppendData(FlowFileContainer *ffc, uint8_t *data, uint32_t data_len) } if (ffc->tail->state != FLOWFILE_STATE_OPENED) { + if (ffc->tail->store == -1) { + SCReturnInt(-2); + } SCReturnInt(-1); } + SCLogDebug("appending %"PRIu32" bytes", data_len); + FlowFileData *ffd = FlowFileDataAlloc(data, data_len); if (ffd == NULL) { ffc->tail->state = FLOWFILE_STATE_ERROR; @@ -350,3 +386,79 @@ int FlowFileAppendData(FlowFileContainer *ffc, uint8_t *data, uint32_t data_len) } SCReturnInt(0); } + +/** + * \brief Tag a file for storing + * + * \param ff The file to store + */ +int FlowFileStore(FlowFile *ff) { + ff->store = 1; + SCReturnInt(0); +} + +/** + * \brief Set the TX id for a file + * + * \param ff The file to store + * \param txid the tx id + */ +int FlowFileSetTx(FlowFile *ff, uint16_t txid) { + ff->txid = txid; + SCReturnInt(0); +} + +/** + * \brief disable file storage for a flow + * + * \param f *LOCKED* flow + */ +void FlowFileDisableStoring(Flow *f) { + FlowFile *ptr = NULL; + + SCEnter(); + + f->flags |= FLOW_FILE_NO_HANDLING; + + SCMutexLock(&f->files_m); + if (f->files != NULL) { + for (ptr = f->files->head; ptr != NULL; ptr = ptr->next) { + if (ptr->state == FLOWFILE_STATE_OPENED) { + ptr->state = FLOWFILE_STATE_CLOSED; + SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_CLOSED"); + } + } + } + SCMutexUnlock(&f->files_m); + SCReturn; +} + +/** + * \brief disable file storing for files in a transaction + * + * \param f flow + * \param tx_id transaction id + */ +void FlowFileDisableStoringForTransaction(struct Flow_ *f, uint16_t tx_id) { + FlowFile *ptr = NULL; + + SCEnter(); + + SCMutexLock(&f->files_m); + if (f->files != NULL) { + for (ptr = f->files->head; ptr != NULL; ptr = ptr->next) { + if (ptr->state == FLOWFILE_STATE_OPENED && ptr->txid == tx_id) { + if (ptr->store == 1) { + /* weird, already storing -- let it continue*/ + SCLogDebug("file is already being stored"); + } else { + (void)FlowFileCloseFilePtr(ptr, NULL, 0, + (FLOW_FILE_TRUNCATED|FLOW_FILE_NOSTORE)); + } + } + } + } + SCMutexUnlock(&f->files_m); + + SCReturn; +} diff --git a/src/flow-file.h b/src/flow-file.h index 8482dbb055..cdcb403dbc 100644 --- a/src/flow-file.h +++ b/src/flow-file.h @@ -29,6 +29,7 @@ #include "util-hash.h" #define FLOW_FILE_TRUNCATED 0x01 +#define FLOW_FILE_NOSTORE 0x02 typedef enum FlowFileState_ { FLOWFILE_STATE_NONE = 0, /**< no state */ @@ -51,10 +52,12 @@ typedef struct FlowFileData_ { } FlowFileData; typedef struct FlowFile_ { + int16_t store; /**< need storing? 0: no, 1: yes, -1: won't */ + uint16_t txid; /**< tx this file is part of */ uint8_t *name; uint16_t name_len; int16_t state; - int fd; /**< file discriptor for storing files */ + int fd; /**< file discriptor for storing files */ FlowFileData *chunks_head; FlowFileData *chunks_tail; struct FlowFile_ *next; @@ -113,4 +116,34 @@ int FlowFileCloseFile(FlowFileContainer *, uint8_t *data, uint32_t data_len, uin */ int FlowFileAppendData(FlowFileContainer *, uint8_t *data, uint32_t data_len); +/** + * \brief Tag a file for storing + * + * \param ff The file to store + */ +int FlowFileStore(FlowFile *); + +/** + * \brief Set the TX id for a file + * + * \param ff The file to store + * \param txid the tx id + */ +int FlowFileSetTx(FlowFile *, uint16_t txid); + +/** + * \brief disable file storage for a flow + * + * \param f *LOCKED* flow + */ +void FlowFileDisableStoring(struct Flow_ *); + +/** + * \brief disable file storing for a transaction + * + * \param f flow + * \param tx_id transaction id + */ +void FlowFileDisableStoringForTransaction(struct Flow_ *, uint16_t); + #endif /* __FLOW_FILE_H__ */ diff --git a/src/flow.h b/src/flow.h index 27da8050c9..1f47dda0fc 100644 --- a/src/flow.h +++ b/src/flow.h @@ -91,6 +91,8 @@ /** Both pattern matcher and probing parser alproto detection done */ #define FLOW_TC_PM_PP_ALPROTO_DETECT_DONE 0x00400000 #define FLOW_TIMEOUT_REASSEMBLY_DONE 0x00800000 +/** even if the flow has files, don't store 'm */ +#define FLOW_FILE_NO_HANDLING 0x01000000 /** flow is ipv4 */ #define FLOW_IPV4 0x01000000 diff --git a/src/log-file.c b/src/log-file.c index 8f3d6ca031..4dded34179 100644 --- a/src/log-file.c +++ b/src/log-file.c @@ -54,6 +54,9 @@ void LogFileLogExitPrintStats(ThreadVars *, void *); int LogFileLogOpenFileCtx(LogFileCtx* , const char *, const char *); static void LogFileLogDeInitCtx(OutputCtx *); +SC_ATOMIC_DECLARE(unsigned int, file_id); +static char g_logfile_base_dir[PATH_MAX] = "/tmp"; + void TmModuleLogFileLogRegister (void) { tmm_modules[TMM_FILELOG].name = MODULE_NAME; tmm_modules[TMM_FILELOG].ThreadInit = LogFileLogThreadInit; @@ -66,6 +69,8 @@ void TmModuleLogFileLogRegister (void) { OutputRegisterModule(MODULE_NAME, "file", LogFileLogInitCtx); SCLogDebug("registered"); + + SC_ATOMIC_INIT(file_id); } typedef struct LogFileLogThread_ { @@ -74,9 +79,6 @@ typedef struct LogFileLogThread_ { uint32_t file_cnt; } LogFileLogThread; -SC_ATOMIC_DECL_AND_INIT(unsigned int, file_id); -static char g_logfile_base_dir[PATH_MAX] = "/tmp"; - static void CreateTimeString (const struct timeval *ts, char *str, size_t size) { time_t time = ts->tv_sec; @@ -107,6 +109,10 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P if (ff->state == FLOWFILE_STATE_STORED) continue; + if (ff->store != 1) { + continue; + } + FlowFileData *ffd; for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) { if (ffd->stored == 1)