From 70157b1e156fdf068ad8413eac626b7cc91c7a6e Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Wed, 4 Sep 2024 15:57:02 +0200 Subject: [PATCH 1/5] ci/live: test for sid 1 existence, not in last position --- qa/live/afp-ids.sh | 4 ++-- qa/live/pcap.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/qa/live/afp-ids.sh b/qa/live/afp-ids.sh index 541ca982c1..e938e06cfe 100755 --- a/qa/live/afp-ids.sh +++ b/qa/live/afp-ids.sh @@ -70,8 +70,8 @@ if [ $STATSCHECK = false ]; then echo "ERROR no packets captured" RES=1 fi -SID1CHECK=$(jq -c 'select(.event_type == "alert")' ./eve.json | tail -n1 | jq '.alert.signature_id == 1') -if [ $SID1CHECK = false ]; then +SID1CHECK=$(jq -c 'select(.alert.signature_id == 1)' ./eve.json | wc -l) +if [ $SID1CHECK -eq 0 ]; then echo "ERROR no alerts for sid 1" RES=1 fi diff --git a/qa/live/pcap.sh b/qa/live/pcap.sh index 06f59ee896..8dfae07825 100755 --- a/qa/live/pcap.sh +++ b/qa/live/pcap.sh @@ -61,8 +61,8 @@ if [ $STATSCHECK = false ]; then echo "ERROR no packets captured" RES=1 fi -SID1CHECK=$(jq -c 'select(.event_type == "alert")' ./eve.json | tail -n1 | jq '.alert.signature_id == 1') -if [ $SID1CHECK = false ]; then +SID1CHECK=$(jq -c 'select(.alert.signature_id == 1)' ./eve.json | wc -l) +if [ $SID1CHECK -eq 0 ]; then echo "ERROR no alerts for sid 1" RES=1 fi From 2d5e922125fb2e6517807e61f26f11ecd6d2d390 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Wed, 4 Sep 2024 15:57:53 +0200 Subject: [PATCH 2/5] detect: postmatch can run AppLayerTxMatch callbacks Will be useful for dataset, when it needs to find a transaction buffer again. --- src/detect.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/detect.c b/src/detect.c index 537ecfa41b..d219e901bc 100644 --- a/src/detect.c +++ b/src/detect.c @@ -261,9 +261,8 @@ end: SCReturn; } -static void DetectRunPostMatch(ThreadVars *tv, - DetectEngineThreadCtx *det_ctx, Packet *p, - const Signature *s) +static void DetectRunPostMatch(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Packet *p, + const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv) { /* run the packet match functions */ const SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_POSTMATCH]; @@ -274,6 +273,10 @@ static void DetectRunPostMatch(ThreadVars *tv, while (1) { KEYWORD_PROFILING_START; + if (txv && sigmatch_table[smd->type].AppLayerTxMatch != NULL) { + sigmatch_table[smd->type].AppLayerTxMatch( + det_ctx, f, flags, alstate, txv, s, smd->ctx); + } (void)sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx); KEYWORD_PROFILING_END(det_ctx, smd->type, 1); if (smd->is_last) @@ -885,7 +888,7 @@ static inline uint8_t DetectRulePacketRules(ThreadVars *const tv, #ifdef PROFILE_RULES smatch = true; #endif - DetectRunPostMatch(tv, det_ctx, p, s); + DetectRunPostMatch(tv, det_ctx, p, s, NULL, 0, NULL, NULL); DetectRulePacketAppendAlert(de_ctx, det_ctx, s, p, pflow, alert_flags); @@ -2624,7 +2627,7 @@ static void DetectRunTx(ThreadVars *tv, SCLogDebug("s %u r %d", s->id, r); if (r == 1) { /* match */ - DetectRunPostMatch(tv, det_ctx, p, s); + DetectRunPostMatch(tv, det_ctx, p, s, f, flow_flags, alstate, tx.tx_ptr); SCLogDebug( "%p/%" PRIu64 " sig %u (%u) matched", tx.tx_ptr, tx.tx_id, s->id, s->iid); @@ -2840,7 +2843,7 @@ static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngin r = DetectRunFrameInspectRule(tv, det_ctx, s, f, p, frames, frame); if (r) { /* match */ - DetectRunPostMatch(tv, det_ctx, p, s); + DetectRunPostMatch(tv, det_ctx, p, s, NULL, 0, NULL, NULL); det_ctx->frame_id = frame->id; SCLogDebug( "%p/%" PRIi64 " sig %u (%u) matched", frame, frame->id, s->id, s->iid); From 6772a43a9558a487a1730d81b368163ec53ca23d Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 5 Dec 2024 11:42:58 +0100 Subject: [PATCH 3/5] detect: delay_postmatch ability for buffers Buffers can be put at the tail of inspections whatever their progress. This will allow dataset/set to be inspected in the context of the final packet matching or not, and thus use the ephemeral data from DetectThreadCtxGetKeywordThreadCtx --- src/detect-engine.c | 16 +++++++++++----- src/detect.h | 3 +++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/detect-engine.c b/src/detect-engine.c index 437408cc35..69c0a42ce1 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -732,7 +732,8 @@ static void AppendPacketInspectEngine(DetectEngineCtx *de_ctx, static void AppendAppInspectEngine(DetectEngineCtx *de_ctx, const DetectEngineAppInspectionEngine *t, Signature *s, SigMatchData *smd, - const int mpm_list, const int files_id, uint8_t *last_id, bool *head_is_mpm) + const int mpm_list, const int files_id, uint8_t *last_id, bool *head_is_mpm, + bool delay_postmatch) { if (t->alproto == ALPROTO_UNKNOWN) { /* special case, inspect engine applies to all protocols */ @@ -786,6 +787,9 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx, new_engine->match_on_null = smd ? DetectContentInspectionMatchOnAbsentBuffer(smd) : false; new_engine->progress = t->progress; new_engine->sub_state = t->sub_state; + if (delay_postmatch) + new_engine->progress = AppLayerParserGetStateProgressCompletionStatus( + t->alproto, t->dir == 0 ? STREAM_TOSERVER : STREAM_TOCLIENT); new_engine->v2 = t->v2; SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback, new_engine->v2.GetData, new_engine->v2.transforms); @@ -975,7 +979,8 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature .sm_list_base = (uint16_t)sm_list, .dir = dir, }; - AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm); + AppendAppInspectEngine( + de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm, false); SCLogDebug("sid %u: appended pass-tru engine at hook:%u sm_list:%d for " "SIG_FLAG_INIT_HOOK_LTE", s->id, state, sm_list); @@ -1022,8 +1027,8 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature continue; } } - AppendAppInspectEngine( - de_ctx, t, s, smd, mpm_list, files_id, &last_id, &head_is_mpm); + AppendAppInspectEngine(de_ctx, t, s, smd, mpm_list, files_id, &last_id, + &head_is_mpm, s->init_data->buffers[x].delay_postmatch); } } } @@ -1058,7 +1063,8 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature .sm_list_base = (uint16_t)s->init_data->hook.sm_list, .dir = dir, }; - AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm); + AppendAppInspectEngine( + de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm, false); } if ((s->init_data->init_flags & SIG_FLAG_INIT_STATE_MATCH) && diff --git a/src/detect.h b/src/detect.h index c712680236..68d9d7d68d 100644 --- a/src/detect.h +++ b/src/detect.h @@ -532,6 +532,9 @@ typedef struct SignatureInitDataBuffer_ { http.uri. */ bool only_tc; /**< true if we can only used toclient. */ bool only_ts; /**< true if we can only used toserver. */ + /** delay use of this buffer beyond its normal progress + * to be just in time for postmatch */ + bool delay_postmatch; /* sig match list */ SigMatch *head; SigMatch *tail; From e66eaec6c93e220df913c375874627d8949ca9e8 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 5 Dec 2024 11:49:36 +0100 Subject: [PATCH 4/5] detect: allow inspection of multi-buffers beyond first match DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF is the new return status to allow dataset/set to pick all matching and only the matching buffers in case of a multi-buffer --- src/detect-engine-content-inspection.c | 56 +++++++++++++++++++------- src/detect-engine-content-inspection.h | 4 ++ src/detect-engine-state.h | 4 ++ src/detect-engine.c | 15 ++++--- 4 files changed, 60 insertions(+), 19 deletions(-) diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index cb80f3bd7e..6cabecaf08 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -44,6 +44,7 @@ #include "detect-entropy.h" #include "detect-replace.h" #include "detect-engine-content-inspection.h" +#include "detect-engine-state.h" #include "detect-uricontent.h" #include "detect-urilen.h" #include "detect-engine-uint.h" @@ -108,7 +109,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, struct DetectEngineContentInspectionCtx *ctx, const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const uint8_t *buffer, const uint32_t buffer_len, const uint64_t stream_start_offset, const uint8_t flags, - const enum DetectContentInspectionType inspection_mode) + const enum DetectContentInspectionType inspection_mode, uint32_t local_id) { SCEnter(); KEYWORD_PROFILING_START; @@ -368,9 +369,9 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, * search for another occurrence of this content and see * if the others match then until we run out of matches */ int r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, smd + 1, p, f, - buffer, buffer_len, stream_start_offset, flags, inspection_mode); - if (r == 1) { - SCReturnInt(1); + buffer, buffer_len, stream_start_offset, flags, inspection_mode, local_id); + if (r == 1 || r == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF) { + SCReturnInt(r); } else if (r == -1) { SCLogDebug("'next sm' said to discontinue this right now"); SCReturnInt(-1); @@ -480,9 +481,9 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, * search for another occurrence of this pcre and see * if the others match, until we run out of matches */ r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, smd + 1, p, f, buffer, - buffer_len, stream_start_offset, flags, inspection_mode); - if (r == 1) { - SCReturnInt(1); + buffer_len, stream_start_offset, flags, inspection_mode, local_id); + if (r == 1 || r == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF) { + SCReturnInt(r); } else if (r == -1) { SCReturnInt(-1); } @@ -648,10 +649,22 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, //PrintRawDataFp(stdout, buffer, buffer_len); const DetectDatasetData *sd = (const DetectDatasetData *) smd->ctx; - int r = DetectDatasetBufferMatch(det_ctx, sd, buffer, buffer_len); //TODO buffer offset? - if (r == 1) { + int r = DetectDatasetBufferMatch(det_ctx, sd, buffer, buffer_len); // TODO buffer offset? + if (r == DETECT_ENGINE_INSPECT_SIG_MATCH) { goto match; } + if (r == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF) { + if (!smd->is_last) { + KEYWORD_PROFILING_END(det_ctx, smd->type, 1); + r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, smd + 1, p, f, buffer, + buffer_len, stream_start_offset, flags, inspection_mode, local_id); + if (r == 1) + SCReturnInt(DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF); + SCReturnInt(r); + } + KEYWORD_PROFILING_END(det_ctx, smd->type, 1); + SCReturnInt(DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF); + } goto no_match_discontinue; } else if (smd->type == DETECT_DATAREP) { @@ -700,7 +713,8 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, int r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, s->sm_arrays[DETECT_SM_LIST_BASE64_DATA], NULL, f, det_ctx->base64_decoded, det_ctx->base64_decoded_len, 0, - DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE); + DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, + local_id); if (r == 1) { /* Base64 is a terminal list. */ goto final_match; @@ -736,7 +750,7 @@ match: if (!smd->is_last) { KEYWORD_PROFILING_END(det_ctx, smd->type, 1); int r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, smd + 1, p, f, buffer, - buffer_len, stream_start_offset, flags, inspection_mode); + buffer_len, stream_start_offset, flags, inspection_mode, local_id); SCReturnInt(r); } final_match: @@ -758,11 +772,11 @@ bool DetectEngineContentInspection(DetectEngineCtx *de_ctx, DetectEngineThreadCt det_ctx->buffer_offset = 0; int r = DetectEngineContentInspectionInternal(det_ctx, &ctx, s, smd, p, f, buffer, buffer_len, - stream_start_offset, flags, inspection_mode); + stream_start_offset, flags, inspection_mode, 0); #ifdef UNITTESTS ut_inspection_recursion_counter = ctx.recursion.count; #endif - return (r == 1); + return (r == 1 || r == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF); } /** \brief wrapper around DetectEngineContentInspectionInternal to return true/false only @@ -779,7 +793,7 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh det_ctx->buffer_offset = 0; int r = DetectEngineContentInspectionInternal(det_ctx, &ctx, s, smd, p, f, b->inspect, - b->inspect_len, b->inspect_offset, b->flags, inspection_mode); + b->inspect_len, b->inspect_offset, b->flags, inspection_mode, 0); #ifdef UNITTESTS ut_inspection_recursion_counter = ctx.recursion.count; #endif @@ -804,6 +818,20 @@ bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd) return absent_data; } +int DetectEngineContentInspectionBufferMulti(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, const Signature *s, const SigMatchData *smd, Flow *f, + const InspectionBuffer *b, uint32_t local_id) +{ + struct DetectEngineContentInspectionCtx ctx = { .recursion.count = 0, + .recursion.limit = de_ctx->inspection_recursion_limit }; + + det_ctx->buffer_offset = 0; + + return DetectEngineContentInspectionInternal(det_ctx, &ctx, s, smd, NULL, f, b->inspect, + b->inspect_len, b->inspect_offset, b->flags, + DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, local_id); +} + #ifdef UNITTESTS #include "tests/detect-engine-content-inspection.c" #endif diff --git a/src/detect-engine-content-inspection.h b/src/detect-engine-content-inspection.h index 3c12477265..cf75fd6b61 100644 --- a/src/detect-engine-content-inspection.h +++ b/src/detect-engine-content-inspection.h @@ -75,6 +75,10 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh * \retval bool true to match on absent buffer, false otherwise */ bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd); +int DetectEngineContentInspectionBufferMulti(DetectEngineCtx *de_ctx, + DetectEngineThreadCtx *det_ctx, const Signature *s, const SigMatchData *smd, Flow *f, + const InspectionBuffer *b, uint32_t local_id); + void DetectEngineContentInspectionRegisterTests(void); #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */ diff --git a/src/detect-engine-state.h b/src/detect-engine-state.h index 5952776293..ccc34460a5 100644 --- a/src/detect-engine-state.h +++ b/src/detect-engine-state.h @@ -50,6 +50,10 @@ * indicate that one of the files matched, but that there are still * more files that have ongoing inspection. */ #define DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES 4 +/** Indicates that we matched on an occurence of a multi-buffer + * but we want to try all occurences, for example, + * to see which should go into a dataset */ +#define DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF 5 /** number of DeStateStoreItem's in one DeStateStore object */ #define DE_STATE_CHUNK_SIZE 15 diff --git a/src/detect-engine.c b/src/detect-engine.c index 69c0a42ce1..28b8a244df 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -2352,6 +2352,7 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx, transforms = engine->v2.transforms; } + uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; do { InspectionBuffer *buffer = DetectGetMultiData(det_ctx, transforms, f, flags, txv, engine->sm_list, local_id, engine->v2.GetMultiData); @@ -2361,10 +2362,14 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx, // The GetData functions set buffer->flags to DETECT_CI_FLAGS_SINGLE // This is not meant for streaming buffers - const bool match = DetectEngineContentInspectionBuffer(de_ctx, det_ctx, s, engine->smd, - NULL, f, buffer, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE); - if (match) { - return DETECT_ENGINE_INSPECT_SIG_MATCH; + int match = DetectEngineContentInspectionBufferMulti( + de_ctx, det_ctx, s, engine->smd, f, buffer, local_id); + switch (match) { + case DETECT_ENGINE_INSPECT_SIG_MATCH: + return DETECT_ENGINE_INSPECT_SIG_MATCH; + case DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF: + r = DETECT_ENGINE_INSPECT_SIG_MATCH; + break; } local_id++; } while (1); @@ -2376,7 +2381,7 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx, return DETECT_ENGINE_INSPECT_SIG_MATCH; } } - return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + return r; } /** From 23d5cad872b78aed6a7d574098d6e9984138bea7 Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 5 Dec 2024 11:54:03 +0100 Subject: [PATCH 5/5] detect/dataset: delay set operation after signature full match The set operation of dataset keyword was done even if signature did not fully match, which is not the expected behavior. We want dataset to behave like flowbits for instance. This patch changes the behavior of the dataset keyword to do a match and a post match for the set operation. The postmatch retrieves the data, using the list identifier associated to the buffer for this signature. This avoids to store the buffer(s), when we do not have a dedicated storage (per signature and per tx) that can own and clean arbitrary buffers over multiple packets, in the case the transaction spans over multiple packets with different tx progresses for instance. Thanks to delay_postmatch, the callback DetectDatasetTxMatch should be called on the last packet taking the final match decision. At this point, the inspection buffers are cached and easy to get. The most complex case is for multi-buffers, where we need to know which buffers/local_ids matched. We use the ephemeral storage (lives with one packet detection) provided by DetectThreadCtxGetKeywordThreadCtx For multi-buffers, we also return DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF to indicate that we want to know all the instances which match. Ticket: #5576 --- src/detect-dataset.c | 208 +++++++++++++++++++++---- src/detect-dataset.h | 8 +- src/detect-engine-content-inspection.c | 3 +- 3 files changed, 189 insertions(+), 30 deletions(-) diff --git a/src/detect-dataset.c b/src/detect-dataset.c index 4dabbd6c2c..977674d115 100644 --- a/src/detect-dataset.c +++ b/src/detect-dataset.c @@ -36,6 +36,7 @@ #include "detect-engine-buffer.h" #include "detect-engine-mpm.h" #include "detect-engine-state.h" +#include "detect-engine-content-inspection.h" #include "util-debug.h" #include "util-print.h" @@ -50,10 +51,94 @@ #define DETECT_DATASET_CMD_ISSET 3 #define DATASET_SUBDOMAIN_MAX_LOOKUPS 126 +#define DMD_CAP_STEP 16 +typedef struct DetectDatasetMatchData_ { + uint32_t nb; + uint32_t *local_ids; +} DetectDatasetMatchData; static int DetectDatasetSetup (DetectEngineCtx *, Signature *, const char *); void DetectDatasetFree (DetectEngineCtx *, void *); +static int DetectDatasetTxMatch(DetectEngineThreadCtx *det_ctx, Flow *f, uint8_t flags, void *state, + void *txv, const Signature *s, const SigMatchCtx *ctx) +{ + const DetectDatasetData *sd = (DetectDatasetData *)ctx; + // This is only run for DETECT_SM_LIST_POSTMATCH + DEBUG_VALIDATE_BUG_ON(sd->cmd != DETECT_DATASET_CMD_SET && sd->cmd != DETECT_DATASET_CMD_UNSET); + + // retrieve the app inspection engine associated to the list + DetectEngineAppInspectionEngine *a = s->app_inspect; + while (a != NULL) { + // also check alproto as http.uri as 2 engines : http1 and http2 + if (a->sm_list == sd->list && a->alproto == f->alproto) { + if (a->v2.Callback == DetectEngineInspectBufferGeneric || + a->v2.Callback == DetectEngineInspectBufferSingle) { + // simple buffer, get cached data again + const InspectionBuffer *buffer = SCInspectionBufferGet(det_ctx, sd->list); + if (buffer != NULL && buffer->inspect != NULL) { + if (sd->cmd == DETECT_DATASET_CMD_SET) { + SCDatasetAdd(sd->set, buffer->inspect, buffer->inspect_len); + } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) { + DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len); + } + } + } else if (a->v2.Callback == DetectEngineInspectMultiBufferGeneric) { + DetectDatasetMatchData *dmd = + (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx( + det_ctx, sd->thread_ctx_id); + DEBUG_VALIDATE_BUG_ON(dmd == NULL); + uint32_t local_id = 0; + for (uint32_t i = 0; i < dmd->nb; i++) { + local_id = dmd->local_ids[i]; + InspectionBuffer *buffer = + InspectionBufferMultipleForListGet(det_ctx, sd->list, local_id); + DEBUG_VALIDATE_BUG_ON(buffer == NULL || buffer->inspect == NULL); + if (sd->cmd == DETECT_DATASET_CMD_SET) { + SCDatasetAdd(sd->set, buffer->inspect, buffer->inspect_len); + } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) { + DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len); + } + } + } + return 0; + } + a = a->next; + } + return 0; +} + +static int DetectDatasetMatch( + DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx) +{ + const DetectDatasetData *sd = (DetectDatasetData *)ctx; + // This is only run for DETECT_SM_LIST_POSTMATCH + DEBUG_VALIDATE_BUG_ON(sd->cmd != DETECT_DATASET_CMD_SET && sd->cmd != DETECT_DATASET_CMD_UNSET); + + // retrieve the pkt inspection engine associated to the list if any (ie if list is not a app + // inspection engine) + DetectEnginePktInspectionEngine *e = s->pkt_inspect; + while (e) { + if (e->sm_list == sd->list) { + if (e->v1.Callback == DetectEngineInspectPktBufferGeneric) { + const InspectionBuffer *buffer = SCInspectionBufferGet(det_ctx, sd->list); + // get simple data again and add it + if (buffer != NULL && buffer->inspect != NULL) { + if (sd->cmd == DETECT_DATASET_CMD_SET) { + SCDatasetAdd(sd->set, buffer->inspect, buffer->inspect_len); + } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) { + DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len); + } + } + } + return 0; + } + e = e->next; + } + // return value is unused for postmatch functions + return 0; +} + void DetectDatasetRegister (void) { sigmatch_table[DETECT_DATASET].name = "dataset"; @@ -62,6 +147,9 @@ void DetectDatasetRegister (void) sigmatch_table[DETECT_DATASET].Setup = DetectDatasetSetup; sigmatch_table[DETECT_DATASET].Free = DetectDatasetFree; sigmatch_table[DETECT_DATASET].flags = SIGMATCH_SUPPORT_FIREWALL; + // callbacks for postmatch + sigmatch_table[DETECT_DATASET].AppLayerTxMatch = DetectDatasetTxMatch; + sigmatch_table[DETECT_DATASET].Match = DetectDatasetMatch; } /** \brief walk up the domain hierarchy looking for a match in a JSON dataset */ @@ -88,8 +176,8 @@ static DataJsonResultType DatajsonLookupSubdomain( 1 match 0 no match */ -static int DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd, - const uint8_t *data, const uint32_t data_len) +static uint8_t DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx, + const DetectDatasetData *sd, const uint8_t *data, const uint32_t data_len) { if (data == NULL || data_len == 0) return 0; @@ -162,57 +250,85 @@ static int DatasetLookupSubdomain(Dataset *set, const uint8_t *data, const uint3 1 match 0 no match */ -int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, - const DetectDatasetData *sd, - const uint8_t *data, const uint32_t data_len) +uint8_t DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd, + const uint8_t *data, const uint32_t data_len, uint32_t local_id) { if (data == NULL || data_len == 0) - return 0; + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; if ((sd->format == DATASET_FORMAT_JSON) || (sd->format == DATASET_FORMAT_NDJSON)) { return DetectDatajsonBufferMatch(det_ctx, sd, data, data_len); } + int r = DatasetLookup(sd->set, data, data_len); + SCLogDebug("r %d", r); switch (sd->cmd) { case DETECT_DATASET_CMD_ISSET: { - //PrintRawDataFp(stdout, data, data_len); - int r = DatasetLookup(sd->set, data, data_len); if (r != 1 && sd->match_subdomain) { r = DatasetLookupSubdomain(sd->set, data, data_len); } - SCLogDebug("r %d", r); if (r == 1) - return 1; + return DETECT_ENGINE_INSPECT_SIG_MATCH; break; } case DETECT_DATASET_CMD_ISNOTSET: { - //PrintRawDataFp(stdout, data, data_len); - int r = DatasetLookup(sd->set, data, data_len); if (r != 1 && sd->match_subdomain) { r = DatasetLookupSubdomain(sd->set, data, data_len); } - SCLogDebug("r %d", r); if (r < 1) - return 1; + return DETECT_ENGINE_INSPECT_SIG_MATCH; break; } case DETECT_DATASET_CMD_SET: { - //PrintRawDataFp(stdout, data, data_len); - int r = SCDatasetAdd(sd->set, data, data_len); - if (r == 1) - return 1; - break; + if (r != 0) { + /* Do not match if data is already in set or if error */ + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } + // SCDatasetAdd will be performed postmatch if the rest of the sig completely matched + DetectDatasetMatchData *dmd = + (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx( + det_ctx, sd->thread_ctx_id); + if (dmd == NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } + if (dmd->nb % DMD_CAP_STEP == 0) { + void *tmp = SCRealloc(dmd->local_ids, sizeof(uint32_t) * (dmd->nb + DMD_CAP_STEP)); + if (tmp == NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } + dmd->local_ids = tmp; + } + dmd->local_ids[dmd->nb] = local_id; + dmd->nb++; + return DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF; } case DETECT_DATASET_CMD_UNSET: { - int r = DatasetRemove(sd->set, data, data_len); - if (r == 1) - return 1; - break; + if (r != 1) { + /* Do not match if data is not in set */ + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } + // DatasetRemove will be performed postmatch if the rest of the sig completely matched + DetectDatasetMatchData *dmd = + (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx( + det_ctx, sd->thread_ctx_id); + if (dmd == NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } + if (dmd->nb % DMD_CAP_STEP == 0) { + void *tmp = SCRealloc(dmd->local_ids, sizeof(uint32_t) * (dmd->nb + DMD_CAP_STEP)); + if (tmp == NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } + dmd->local_ids = tmp; + } + dmd->local_ids[dmd->nb] = local_id; + dmd->nb++; + return DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF; } default: DEBUG_VALIDATE_BUG_ON("unknown dataset command"); } - return 0; + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *name, int name_len, @@ -516,6 +632,24 @@ static int SetupSavePath(const DetectEngineCtx *de_ctx, return 0; } +static void *DetectDatasetMatchDataThreadInit(void *data) +{ + DetectDatasetMatchData *scmd = SCCalloc(1, sizeof(DetectDatasetMatchData)); + // make cocci happy + if (unlikely(scmd == NULL)) + return NULL; + return scmd; +} + +static void DetectDatasetMatchDataThreadFree(void *ctx) +{ + if (ctx) { + DetectDatasetMatchData *scmd = (DetectDatasetMatchData *)ctx; + SCFree(scmd->local_ids); + SCFree(scmd); + } +} + int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawstr) { DetectDatasetData *cd = NULL; @@ -657,8 +791,30 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst SCLogDebug("cmd %s, name %s", cmd_str, strlen(name) ? name : "(none)"); - /* Okay so far so good, lets get this into a SigMatch - * and put it in the Signature. */ + if (cmd == DETECT_DATASET_CMD_SET || cmd == DETECT_DATASET_CMD_UNSET) { + if (s->init_data->curbuf) + s->init_data->curbuf->delay_postmatch = true; + cd->thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "dataset", + DetectDatasetMatchDataThreadInit, (void *)cd, DetectDatasetMatchDataThreadFree, 0); + if (cd->thread_ctx_id == -1) + goto error; + + // for set operation, we need one match, and one postmatch + DetectDatasetData *scd = SCCalloc(1, sizeof(DetectDatasetData)); + if (unlikely(scd == NULL)) + goto error; + + scd->set = set; + scd->cmd = cmd; + // remember the list used by match to retrieve the buffer in postmatch + scd->list = list; + scd->thread_ctx_id = cd->thread_ctx_id; + if (SCSigMatchAppendSMToList(de_ctx, s, DETECT_DATASET, (SigMatchCtx *)scd, + DETECT_SM_LIST_POSTMATCH) == NULL) { + SCFree(scd); + goto error; + } + } if (SCSigMatchAppendSMToList(de_ctx, s, DETECT_DATASET, (SigMatchCtx *)cd, list) == NULL) { goto error; diff --git a/src/detect-dataset.h b/src/detect-dataset.h index 7022b6084f..19eda531f9 100644 --- a/src/detect-dataset.h +++ b/src/detect-dataset.h @@ -35,11 +35,13 @@ typedef struct DetectDatasetData_ { DataJsonType json; char json_key[SIG_JSON_CONTENT_KEY_LEN]; void *id; /* pointer to the triggering signature */ + // for postmatch to retrieve the buffer(s) + int list; + int thread_ctx_id; } DetectDatasetData; -int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, - const DetectDatasetData *sd, - const uint8_t *data, const uint32_t data_len); +uint8_t DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd, + const uint8_t *data, const uint32_t data_len, uint32_t local_id); /* prototypes */ void DetectDatasetRegister (void); diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index 6cabecaf08..fa0585e210 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -649,7 +649,8 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, //PrintRawDataFp(stdout, buffer, buffer_len); const DetectDatasetData *sd = (const DetectDatasetData *) smd->ctx; - int r = DetectDatasetBufferMatch(det_ctx, sd, buffer, buffer_len); // TODO buffer offset? + int r = DetectDatasetBufferMatch( + det_ctx, sd, buffer, buffer_len, local_id); // TODO buffer offset? if (r == DETECT_ENGINE_INSPECT_SIG_MATCH) { goto match; }