From 84dc73d9dee4ddc7f5d33aea861541e2d0450d30 Mon Sep 17 00:00:00 2001 From: Giuseppe Longo Date: Wed, 21 Jan 2015 20:39:34 +0100 Subject: [PATCH] mpm: implement prefiltering for smtp --- src/detect-engine-filedata-smtp.c | 31 +++++++++++ src/detect-engine-filedata-smtp.h | 7 +++ src/detect-engine-mpm.c | 90 +++++++++++++++++++++++++++++-- src/detect-engine-mpm.h | 1 + src/detect.c | 50 ++++++++++++++++- src/detect.h | 3 ++ src/suricata-common.h | 1 + 7 files changed, 178 insertions(+), 5 deletions(-) diff --git a/src/detect-engine-filedata-smtp.c b/src/detect-engine-filedata-smtp.c index 6cd1adb868..7f0ff13e7f 100644 --- a/src/detect-engine-filedata-smtp.c +++ b/src/detect-engine-filedata-smtp.c @@ -215,3 +215,34 @@ void DetectEngineCleanSMTPBuffers(DetectEngineThreadCtx *det_ctx) return; } + +int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, Flow *f, + SMTPState *smtp_state, uint8_t flags, + void *tx, uint64_t idx) +{ + FileContainer *ffc = smtp_state->files_ts; + uint32_t cnt = 0; + uint32_t buffer_len = 0; + uint32_t stream_start_offset = 0; + uint8_t *buffer = 0; + + if (ffc != NULL) { + File *file = ffc->head; + for (; file != NULL; file = file->next) { + buffer = DetectEngineSMTPGetBufferForTX(idx, + de_ctx, det_ctx, + f, file, + flags, + &buffer_len, + &stream_start_offset); + + if (buffer_len == 0) + goto end; + + cnt = SMTPFiledataPatternSearch(det_ctx, buffer, buffer_len, flags); + } + } +end: + return cnt; +} diff --git a/src/detect-engine-filedata-smtp.h b/src/detect-engine-filedata-smtp.h index b574b974c2..666c9e56b4 100644 --- a/src/detect-engine-filedata-smtp.h +++ b/src/detect-engine-filedata-smtp.h @@ -23,6 +23,8 @@ #ifndef __DETECT_ENGINE_FILEDATA_SMTP_H__ #define __DETECT_ENGINE_FILEDATA_SMTP_H__ +#include "app-layer-smtp.h" + int DetectEngineInspectSMTPFiledata(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, @@ -31,4 +33,9 @@ int DetectEngineInspectSMTPFiledata(ThreadVars *tv, void *tx, uint64_t tx_id); void DetectEngineCleanSMTPBuffers(DetectEngineThreadCtx *det_ctx); +int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, Flow *f, + SMTPState *smtp_state, uint8_t flags, + void *tx, uint64_t idx); + #endif /* __DETECT_ENGINE_FILEDATA_SMTP_H__ */ diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 78f7f8a0c3..1cbf277097 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -737,6 +737,36 @@ uint32_t StreamPatternSearch(DetectEngineThreadCtx *det_ctx, Packet *p, SCReturnInt(ret); } +/** + * \brief SMTP Filedata match -- searches for one pattern per signature. + * + * \param det_ctx Detection engine thread ctx. + * \param buffer Buffer to inspect. + * \param buffer_len buffer length. + * \param flags Flags + * + * \retval ret Number of matches. + */ +uint32_t SMTPFiledataPatternSearch(DetectEngineThreadCtx *det_ctx, + uint8_t *buffer, uint32_t buffer_len, + uint8_t flags) +{ + SCEnter(); + + uint32_t ret = 0; + + if (flags & STREAM_TOSERVER) { + if (det_ctx->sgh->mpm_smtp_filedata_ctx_ts == NULL) + SCReturnUInt(0); + + ret = mpm_table[det_ctx->sgh->mpm_smtp_filedata_ctx_ts->mpm_type]. + Search(det_ctx->sgh->mpm_smtp_filedata_ctx_ts, &det_ctx->mtcu, + &det_ctx->pmq, buffer, buffer_len); + } + + SCReturnUInt(ret); +} + /** \brief cleans up the mpm instance after a match */ void PacketPatternCleanup(ThreadVars *t, DetectEngineThreadCtx *det_ctx) { @@ -944,6 +974,16 @@ void PatternMatchDestroyGroup(SigGroupHead *sh) } } + if (sh->mpm_smtp_filedata_ctx_ts != NULL) { + if (sh->mpm_smtp_filedata_ctx_ts != NULL) { + if (!sh->mpm_smtp_filedata_ctx_ts->global) { + mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].DestroyCtx(sh->mpm_smtp_filedata_ctx_ts); + SCFree(sh->mpm_smtp_filedata_ctx_ts); + } + sh->mpm_smtp_filedata_ctx_ts = NULL; + } + } + if (sh->mpm_hhd_ctx_ts != NULL || sh->mpm_hhd_ctx_tc != NULL) { if (sh->mpm_hhd_ctx_ts != NULL) { if (!sh->mpm_hhd_ctx_ts->global) { @@ -1478,9 +1518,14 @@ static void PopulateMpmAddPatternToMpm(DetectEngineCtx *de_ctx, if (cd->flags & DETECT_CONTENT_NEGATED) s->flags |= SIG_FLAG_MPM_APPLAYER_NEG; } else if (sm_list == DETECT_SM_LIST_FILEDATA) { - if (s->flags & SIG_FLAG_TOCLIENT) + if (s->flags & SIG_FLAG_TOCLIENT) { mpm_ctx_tc = sgh->mpm_hsbd_ctx_tc; - sgh->flags |= SIG_GROUP_HEAD_MPM_HSBD; + sgh->flags |= SIG_GROUP_HEAD_MPM_HSBD; + } + if (s->flags & SIG_FLAG_TOSERVER) { + mpm_ctx_ts = sgh->mpm_smtp_filedata_ctx_ts; + sgh->flags |= SIG_GROUP_HEAD_MPM_FD_SMTP; + } s->flags |= SIG_FLAG_MPM_APPLAYER; if (cd->flags & DETECT_CONTENT_NEGATED) s->flags |= SIG_FLAG_MPM_APPLAYER_NEG; @@ -1934,6 +1979,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) uint32_t has_co_hcbd = 0; /* used to indicate if sgh has atleast one sig with http_server_body */ uint32_t has_co_hsbd = 0; + /* used to indicate if sgh has smtp file_data inspecting content */ + uint32_t has_co_smtp = 0; /* used to indicate if sgh has atleast one sig with http_header */ uint32_t has_co_hhd = 0; /* used to indicate if sgh has atleast one sig with http_raw_header */ @@ -1981,7 +2028,14 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) } if (s->sm_lists[DETECT_SM_LIST_FILEDATA] != NULL) { - has_co_hsbd = 1; + if (s->alproto == ALPROTO_SMTP) + has_co_smtp = 1; + else if (s->alproto == ALPROTO_HTTP) + has_co_hsbd = 1; + else if (s->alproto == ALPROTO_UNKNOWN) { + has_co_smtp = 1; + has_co_hsbd = 1; + } } if (s->sm_lists[DETECT_SM_LIST_HHDMATCH] != NULL) { @@ -2131,6 +2185,20 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) MpmInitCtx(sh->mpm_hsbd_ctx_tc, de_ctx->mpm_matcher); } + if (has_co_smtp) { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE) { + sh->mpm_smtp_filedata_ctx_ts = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_smtp, 0); + } else { + sh->mpm_smtp_filedata_ctx_ts = MpmFactoryGetMpmCtxForProfile(de_ctx, MPM_CTX_FACTORY_UNIQUE_CONTEXT, 0); + } + if (sh->mpm_smtp_filedata_ctx_ts == NULL) { + SCLogDebug("sh->mpm_smtp_filedata_ctx_ts == NULL. This should never happen"); + exit(EXIT_FAILURE); + } + + MpmInitCtx(sh->mpm_smtp_filedata_ctx_ts, de_ctx->mpm_matcher); + } + if (has_co_hhd) { if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE) { sh->mpm_hhd_ctx_ts = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_hhd, 0); @@ -2299,6 +2367,7 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) has_co_uri || has_co_hcbd || has_co_hsbd || + has_co_smtp || has_co_hhd || has_co_hrhd || has_co_hmd || @@ -2442,6 +2511,19 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) } } + if (sh->mpm_smtp_filedata_ctx_ts != NULL) { + if (sh->mpm_smtp_filedata_ctx_ts->pattern_cnt == 0) { + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_smtp_filedata_ctx_ts); + sh->mpm_smtp_filedata_ctx_ts = NULL; + } else { + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_FULL) { + if (mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare != NULL) { + mpm_table[sh->mpm_smtp_filedata_ctx_ts->mpm_type].Prepare(sh->mpm_smtp_filedata_ctx_ts); + } + } + } + } + if (sh->mpm_hhd_ctx_ts != NULL) { if (sh->mpm_hhd_ctx_ts->pattern_cnt == 0) { MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_hhd_ctx_ts); @@ -2639,6 +2721,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh) sh->mpm_hrhhd_ctx_ts = NULL; MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_dnsquery_ctx_ts); sh->mpm_dnsquery_ctx_ts = NULL; + MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_smtp_filedata_ctx_ts); + sh->mpm_smtp_filedata_ctx_ts = NULL; MpmFactoryReClaimMpmCtx(de_ctx, sh->mpm_proto_tcp_ctx_tc); sh->mpm_proto_tcp_ctx_tc = NULL; diff --git a/src/detect-engine-mpm.h b/src/detect-engine-mpm.h index 05178e46fc..02df56f47b 100644 --- a/src/detect-engine-mpm.h +++ b/src/detect-engine-mpm.h @@ -52,6 +52,7 @@ uint32_t HttpUAPatternSearch(DetectEngineThreadCtx *, uint8_t *, uint32_t, uint8 uint32_t HttpHHPatternSearch(DetectEngineThreadCtx *, uint8_t *, uint32_t, uint8_t); uint32_t HttpHRHPatternSearch(DetectEngineThreadCtx *, uint8_t *, uint32_t, uint8_t); uint32_t DnsQueryPatternSearch(DetectEngineThreadCtx *det_ctx, uint8_t *buffer, uint32_t buffer_len, uint8_t flags); +uint32_t SMTPFiledataPatternSearch(DetectEngineThreadCtx *det_ctx, uint8_t *buffer, uint32_t buffer_len, uint8_t flags); void PacketPatternCleanup(ThreadVars *, DetectEngineThreadCtx *); void StreamPatternCleanup(ThreadVars *t, DetectEngineThreadCtx *det_ctx, StreamMsg *smsg); diff --git a/src/detect.c b/src/detect.c index ec665944e1..9b08e8852a 100644 --- a/src/detect.c +++ b/src/detect.c @@ -49,6 +49,7 @@ #include "detect-dns-query.h" #include "detect-engine-state.h" #include "detect-engine-analyzer.h" +#include "detect-engine-filedata-smtp.h" #include "detect-http-cookie.h" #include "detect-http-method.h" @@ -160,6 +161,7 @@ #include "app-layer.h" #include "app-layer-protos.h" #include "app-layer-htp.h" +#include "app-layer-smtp.h" #include "detect-tls.h" #include "detect-tls-version.h" #include "detect-ssh-proto-version.h" @@ -988,6 +990,32 @@ static inline void DetectMpmPrefilter(DetectEngineCtx *de_ctx, FLOWLOCK_UNLOCK(p->flow); } } + } else if (alproto == ALPROTO_SMTP && has_state) { + if (p->flowflags & FLOW_PKT_TOSERVER) { + if (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_FD_SMTP) { + FLOWLOCK_RDLOCK(p->flow); + void *alstate = FlowGetAppState(p->flow); + if (alstate == NULL) { + SCLogDebug("no alstate"); + FLOWLOCK_UNLOCK(p->flow); + return; + } + + SMTPState *smtp_state = (SMTPState *)alstate; + uint64_t idx = AppLayerParserGetTransactionInspectId(p->flow->alparser, flags); + uint64_t total_txs = AppLayerParserGetTxCnt(p->flow->proto, alproto, alstate); + for (; idx < total_txs; idx++) { + void *tx = AppLayerParserGetTx(p->flow->proto, alproto, alstate, idx); + if (tx == NULL) + continue; + + PACKET_PROFILING_DETECT_START(p, PROF_DETECT_MPM_FD_SMTP); + DetectEngineRunSMTPMpm(de_ctx, det_ctx, p->flow, smtp_state, flags, tx, idx); + PACKET_PROFILING_DETECT_END(p, PROF_DETECT_MPM_FD_SMTP); + } + FLOWLOCK_UNLOCK(p->flow); + } + } } if (smsg != NULL && (det_ctx->sgh->flags & SIG_GROUP_HEAD_MPM_STREAM)) { @@ -2333,8 +2361,13 @@ static int SignatureCreateMask(Signature *s) } if (s->sm_lists[DETECT_SM_LIST_FILEDATA] != NULL) { - s->mask |= SIG_MASK_REQUIRE_HTTP_STATE; - SCLogDebug("sig requires http app state"); + /* set the state depending from the protocol */ + if (s->alproto == ALPROTO_HTTP) + s->mask |= SIG_MASK_REQUIRE_HTTP_STATE; + else if (s->alproto == ALPROTO_SMTP) + s->mask |= SIG_MASK_REQUIRE_SMTP_STATE; + + SCLogDebug("sig requires http or smtp app state"); } if (s->sm_lists[DETECT_SM_LIST_HHDMATCH] != NULL) { @@ -2561,6 +2594,9 @@ static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx) de_ctx->sgh_mpm_context_hsbd = MpmFactoryRegisterMpmCtxProfile(de_ctx, "hsbd", MPM_CTX_FACTORY_FLAGS_PREPARE_WITH_SIG_GROUP_BUILD); + de_ctx->sgh_mpm_context_smtp = + MpmFactoryRegisterMpmCtxProfile(de_ctx, "smtp", + MPM_CTX_FACTORY_FLAGS_PREPARE_WITH_SIG_GROUP_BUILD); de_ctx->sgh_mpm_context_hhd = MpmFactoryRegisterMpmCtxProfile(de_ctx, "hhd", MPM_CTX_FACTORY_FLAGS_PREPARE_WITH_SIG_GROUP_BUILD); @@ -4698,6 +4734,16 @@ int SigGroupBuild(DetectEngineCtx *de_ctx) } //printf("hsbd- %d\n", mpm_ctx->pattern_cnt); + mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_smtp, 0); + if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { + mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + } + mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_smtp, 1); + if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { + mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); + } + //printf("smtp- %d\n"; mpm_ctx->pattern_cnt); + mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_hhd, 0); if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) { mpm_table[de_ctx->mpm_matcher].Prepare(mpm_ctx); diff --git a/src/detect.h b/src/detect.h index f34b46c7d0..3b5ff6005c 100644 --- a/src/detect.h +++ b/src/detect.h @@ -705,6 +705,7 @@ typedef struct DetectEngineCtx_ { int32_t sgh_mpm_context_hrhhd; int32_t sgh_mpm_context_app_proto_detect; int32_t sgh_mpm_context_dnsquery; + int32_t sgh_mpm_context_smtp; /* the max local id used amongst all sigs */ int32_t byte_extract_max_local_id; @@ -962,6 +963,7 @@ typedef struct SigTableElmt_ { #define SIG_GROUP_HEAD_HAVEFILEMD5 (1 << 21) #define SIG_GROUP_HEAD_HAVEFILESIZE (1 << 22) #define SIG_GROUP_HEAD_MPM_DNSQUERY (1 << 23) +#define SIG_GROUP_HEAD_MPM_FD_SMTP (1 << 24) typedef struct SigGroupHeadInitData_ { /* list of content containers @@ -1024,6 +1026,7 @@ typedef struct SigGroupHead_ { MpmCtx *mpm_hhhd_ctx_ts; MpmCtx *mpm_hrhhd_ctx_ts; MpmCtx *mpm_dnsquery_ctx_ts; + MpmCtx *mpm_smtp_filedata_ctx_ts; MpmCtx *mpm_proto_tcp_ctx_tc; MpmCtx *mpm_proto_udp_ctx_tc; diff --git a/src/suricata-common.h b/src/suricata-common.h index d7126a874f..98545fd803 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -313,6 +313,7 @@ typedef enum PacketProfileDetectId_ { PROF_DETECT_ALERT, PROF_DETECT_CLEANUP, PROF_DETECT_GETSGH, + PROF_DETECT_MPM_FD_SMTP, PROF_DETECT_SIZE, } PacketProfileDetectId;