Merge 23d5cad872 into 9a37bb659b

qa/live/afp-ids.sh

@@ -70,8 +70,8 @@ if [ $STATSCHECK = false ]; then
     echo "ERROR no packets captured"
     RES=1
 fi
-SID1CHECK=$(jq -c 'select(.event_type == "alert")' ./eve.json | tail -n1 | jq '.alert.signature_id == 1')
-if [ $SID1CHECK = false ]; then
+SID1CHECK=$(jq -c 'select(.alert.signature_id == 1)' ./eve.json | wc -l)
+if [ $SID1CHECK -eq 0 ]; then
     echo "ERROR no alerts for sid 1"
     RES=1
 fi

qa/live/pcap.sh

@@ -61,8 +61,8 @@ if [ $STATSCHECK = false ]; then
     echo "ERROR no packets captured"
     RES=1
 fi
-SID1CHECK=$(jq -c 'select(.event_type == "alert")' ./eve.json | tail -n1 | jq '.alert.signature_id == 1')
-if [ $SID1CHECK = false ]; then
+SID1CHECK=$(jq -c 'select(.alert.signature_id == 1)' ./eve.json | wc -l)
+if [ $SID1CHECK -eq 0 ]; then
     echo "ERROR no alerts for sid 1"
     RES=1
 fi

src/detect-dataset.c

@@ -36,6 +36,7 @@
 #include "detect-engine-buffer.h"
 #include "detect-engine-mpm.h"
 #include "detect-engine-state.h"
+#include "detect-engine-content-inspection.h"
 
 #include "util-debug.h"
 #include "util-print.h"
@@ -50,10 +51,94 @@
 #define DETECT_DATASET_CMD_ISSET    3
 
 #define DATASET_SUBDOMAIN_MAX_LOOKUPS 126
+#define DMD_CAP_STEP                  16
+typedef struct DetectDatasetMatchData_ {
+    uint32_t nb;
+    uint32_t *local_ids;
+} DetectDatasetMatchData;
 
 static int DetectDatasetSetup (DetectEngineCtx *, Signature *, const char *);
 void DetectDatasetFree (DetectEngineCtx *, void *);
 
+static int DetectDatasetTxMatch(DetectEngineThreadCtx *det_ctx, Flow *f, uint8_t flags, void *state,
+        void *txv, const Signature *s, const SigMatchCtx *ctx)
+{
+    const DetectDatasetData *sd = (DetectDatasetData *)ctx;
+    // This is only run for DETECT_SM_LIST_POSTMATCH
+    DEBUG_VALIDATE_BUG_ON(sd->cmd != DETECT_DATASET_CMD_SET && sd->cmd != DETECT_DATASET_CMD_UNSET);
+
+    // retrieve the app inspection engine associated to the list
+    DetectEngineAppInspectionEngine *a = s->app_inspect;
+    while (a != NULL) {
+        // also check alproto as http.uri as 2 engines : http1 and http2
+        if (a->sm_list == sd->list && a->alproto == f->alproto) {
+            if (a->v2.Callback == DetectEngineInspectBufferGeneric ||
+                    a->v2.Callback == DetectEngineInspectBufferSingle) {
+                // simple buffer, get cached data again
+                const InspectionBuffer *buffer = SCInspectionBufferGet(det_ctx, sd->list);
+                if (buffer != NULL && buffer->inspect != NULL) {
+                    if (sd->cmd == DETECT_DATASET_CMD_SET) {
+                        SCDatasetAdd(sd->set, buffer->inspect, buffer->inspect_len);
+                    } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) {
+                        DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len);
+                    }
+                }
+            } else if (a->v2.Callback == DetectEngineInspectMultiBufferGeneric) {
+                DetectDatasetMatchData *dmd =
+                        (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx(
+                                det_ctx, sd->thread_ctx_id);
+                DEBUG_VALIDATE_BUG_ON(dmd == NULL);
+                uint32_t local_id = 0;
+                for (uint32_t i = 0; i < dmd->nb; i++) {
+                    local_id = dmd->local_ids[i];
+                    InspectionBuffer *buffer =
+                            InspectionBufferMultipleForListGet(det_ctx, sd->list, local_id);
+                    DEBUG_VALIDATE_BUG_ON(buffer == NULL || buffer->inspect == NULL);
+                    if (sd->cmd == DETECT_DATASET_CMD_SET) {
+                        SCDatasetAdd(sd->set, buffer->inspect, buffer->inspect_len);
+                    } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) {
+                        DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len);
+                    }
+                }
+            }
+            return 0;
+        }
+        a = a->next;
+    }
+    return 0;
+}
+
+static int DetectDatasetMatch(
+        DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx)
+{
+    const DetectDatasetData *sd = (DetectDatasetData *)ctx;
+    // This is only run for DETECT_SM_LIST_POSTMATCH
+    DEBUG_VALIDATE_BUG_ON(sd->cmd != DETECT_DATASET_CMD_SET && sd->cmd != DETECT_DATASET_CMD_UNSET);
+
+    // retrieve the pkt inspection engine associated to the list if any (ie if list is not a app
+    // inspection engine)
+    DetectEnginePktInspectionEngine *e = s->pkt_inspect;
+    while (e) {
+        if (e->sm_list == sd->list) {
+            if (e->v1.Callback == DetectEngineInspectPktBufferGeneric) {
+                const InspectionBuffer *buffer = SCInspectionBufferGet(det_ctx, sd->list);
+                // get simple data again and add it
+                if (buffer != NULL && buffer->inspect != NULL) {
+                    if (sd->cmd == DETECT_DATASET_CMD_SET) {
+                        SCDatasetAdd(sd->set, buffer->inspect, buffer->inspect_len);
+                    } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) {
+                        DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len);
+                    }
+                }
+            }
+            return 0;
+        }
+        e = e->next;
+    }
+    // return value is unused for postmatch functions
+    return 0;
+}
+
 void DetectDatasetRegister (void)
 {
     sigmatch_table[DETECT_DATASET].name = "dataset";
@@ -62,6 +147,9 @@ void DetectDatasetRegister (void)
     sigmatch_table[DETECT_DATASET].Setup = DetectDatasetSetup;
     sigmatch_table[DETECT_DATASET].Free  = DetectDatasetFree;
     sigmatch_table[DETECT_DATASET].flags = SIGMATCH_SUPPORT_FIREWALL;
+    // callbacks for postmatch
+    sigmatch_table[DETECT_DATASET].AppLayerTxMatch = DetectDatasetTxMatch;
+    sigmatch_table[DETECT_DATASET].Match = DetectDatasetMatch;
 }
 
 /** \brief walk up the domain hierarchy looking for a match in a JSON dataset */
@@ -88,8 +176,8 @@ static DataJsonResultType DatajsonLookupSubdomain(
     1 match
     0 no match
  */
-static int DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd,
-        const uint8_t *data, const uint32_t data_len)
+static uint8_t DetectDatajsonBufferMatch(DetectEngineThreadCtx *det_ctx,
+        const DetectDatasetData *sd, const uint8_t *data, const uint32_t data_len)
 {
     if (data == NULL || data_len == 0)
         return 0;
@@ -162,57 +250,85 @@ static int DatasetLookupSubdomain(Dataset *set, const uint8_t *data, const uint3
     1 match
     0 no match
  */
-int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx,
-    const DetectDatasetData *sd,
-    const uint8_t *data, const uint32_t data_len)
+uint8_t DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd,
+        const uint8_t *data, const uint32_t data_len, uint32_t local_id)
 {
     if (data == NULL || data_len == 0)
-        return 0;
+        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 
     if ((sd->format == DATASET_FORMAT_JSON) || (sd->format == DATASET_FORMAT_NDJSON)) {
         return DetectDatajsonBufferMatch(det_ctx, sd, data, data_len);
     }
 
+    int r = DatasetLookup(sd->set, data, data_len);
+    SCLogDebug("r %d", r);
     switch (sd->cmd) {
         case DETECT_DATASET_CMD_ISSET: {
-            //PrintRawDataFp(stdout, data, data_len);
-            int r = DatasetLookup(sd->set, data, data_len);
             if (r != 1 && sd->match_subdomain) {
                 r = DatasetLookupSubdomain(sd->set, data, data_len);
             }
-            SCLogDebug("r %d", r);
             if (r == 1)
-                return 1;
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
             break;
         }
         case DETECT_DATASET_CMD_ISNOTSET: {
-            //PrintRawDataFp(stdout, data, data_len);
-            int r = DatasetLookup(sd->set, data, data_len);
             if (r != 1 && sd->match_subdomain) {
                 r = DatasetLookupSubdomain(sd->set, data, data_len);
             }
-            SCLogDebug("r %d", r);
             if (r < 1)
-                return 1;
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
             break;
         }
         case DETECT_DATASET_CMD_SET: {
-            //PrintRawDataFp(stdout, data, data_len);
-            int r = SCDatasetAdd(sd->set, data, data_len);
-            if (r == 1)
-                return 1;
-            break;
+            if (r != 0) {
+                /* Do not match if data is already in set or if error */
+                return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+            }
+            // SCDatasetAdd will be performed postmatch if the rest of the sig completely matched
+            DetectDatasetMatchData *dmd =
+                    (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx(
+                            det_ctx, sd->thread_ctx_id);
+            if (dmd == NULL) {
+                return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+            }
+            if (dmd->nb % DMD_CAP_STEP == 0) {
+                void *tmp = SCRealloc(dmd->local_ids, sizeof(uint32_t) * (dmd->nb + DMD_CAP_STEP));
+                if (tmp == NULL) {
+                    return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+                }
+                dmd->local_ids = tmp;
+            }
+            dmd->local_ids[dmd->nb] = local_id;
+            dmd->nb++;
+            return DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF;
         }
         case DETECT_DATASET_CMD_UNSET: {
-            int r = DatasetRemove(sd->set, data, data_len);
-            if (r == 1)
-                return 1;
-            break;
+            if (r != 1) {
+                /* Do not match if data is not in set */
+                return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+            }
+            // DatasetRemove will be performed postmatch if the rest of the sig completely matched
+            DetectDatasetMatchData *dmd =
+                    (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx(
+                            det_ctx, sd->thread_ctx_id);
+            if (dmd == NULL) {
+                return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+            }
+            if (dmd->nb % DMD_CAP_STEP == 0) {
+                void *tmp = SCRealloc(dmd->local_ids, sizeof(uint32_t) * (dmd->nb + DMD_CAP_STEP));
+                if (tmp == NULL) {
+                    return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+                }
+                dmd->local_ids = tmp;
+            }
+            dmd->local_ids[dmd->nb] = local_id;
+            dmd->nb++;
+            return DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF;
         }
         default:
             DEBUG_VALIDATE_BUG_ON("unknown dataset command");
     }
-    return 0;
+    return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 }
 
 static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *name, int name_len,
@@ -516,6 +632,24 @@ static int SetupSavePath(const DetectEngineCtx *de_ctx,
     return 0;
 }
 
+static void *DetectDatasetMatchDataThreadInit(void *data)
+{
+    DetectDatasetMatchData *scmd = SCCalloc(1, sizeof(DetectDatasetMatchData));
+    // make cocci happy
+    if (unlikely(scmd == NULL))
+        return NULL;
+    return scmd;
+}
+
+static void DetectDatasetMatchDataThreadFree(void *ctx)
+{
+    if (ctx) {
+        DetectDatasetMatchData *scmd = (DetectDatasetMatchData *)ctx;
+        SCFree(scmd->local_ids);
+        SCFree(scmd);
+    }
+}
+
 int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawstr)
 {
     DetectDatasetData *cd = NULL;
@@ -657,8 +791,30 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     SCLogDebug("cmd %s, name %s",
         cmd_str, strlen(name) ? name : "(none)");
 
-    /* Okay so far so good, lets get this into a SigMatch
-     * and put it in the Signature. */
+    if (cmd == DETECT_DATASET_CMD_SET || cmd == DETECT_DATASET_CMD_UNSET) {
+        if (s->init_data->curbuf)
+            s->init_data->curbuf->delay_postmatch = true;
+        cd->thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "dataset",
+                DetectDatasetMatchDataThreadInit, (void *)cd, DetectDatasetMatchDataThreadFree, 0);
+        if (cd->thread_ctx_id == -1)
+            goto error;
+
+        // for set operation, we need one match, and one postmatch
+        DetectDatasetData *scd = SCCalloc(1, sizeof(DetectDatasetData));
+        if (unlikely(scd == NULL))
+            goto error;
+
+        scd->set = set;
+        scd->cmd = cmd;
+        // remember the list used by match to retrieve the buffer in postmatch
+        scd->list = list;
+        scd->thread_ctx_id = cd->thread_ctx_id;
+        if (SCSigMatchAppendSMToList(de_ctx, s, DETECT_DATASET, (SigMatchCtx *)scd,
+                    DETECT_SM_LIST_POSTMATCH) == NULL) {
+            SCFree(scd);
+            goto error;
+        }
+    }
 
     if (SCSigMatchAppendSMToList(de_ctx, s, DETECT_DATASET, (SigMatchCtx *)cd, list) == NULL) {
         goto error;

src/detect-dataset.h

@@ -35,11 +35,13 @@ typedef struct DetectDatasetData_ {
     DataJsonType json;
     char json_key[SIG_JSON_CONTENT_KEY_LEN];
     void *id; /* pointer to the triggering signature */
+    // for postmatch to retrieve the buffer(s)
+    int list;
+    int thread_ctx_id;
 } DetectDatasetData;
 
-int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx,
-    const DetectDatasetData *sd,
-    const uint8_t *data, const uint32_t data_len);
+uint8_t DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd,
+        const uint8_t *data, const uint32_t data_len, uint32_t local_id);
 
 /* prototypes */
 void DetectDatasetRegister (void);

src/detect-engine-content-inspection.c

@@ -44,6 +44,7 @@
 #include "detect-entropy.h"
 #include "detect-replace.h"
 #include "detect-engine-content-inspection.h"
+#include "detect-engine-state.h"
 #include "detect-uricontent.h"
 #include "detect-urilen.h"
 #include "detect-engine-uint.h"
@@ -107,7 +108,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
         struct DetectEngineContentInspectionCtx *ctx, const Signature *s, const SigMatchData *smd,
         Packet *p, Flow *f, const uint8_t *buffer, const uint32_t buffer_len,
         const uint64_t stream_start_offset, const uint8_t flags,
-        const enum DetectContentInspectionType inspection_mode)
+        const enum DetectContentInspectionType inspection_mode, uint32_t local_id)
 {
     SCEnter();
     KEYWORD_PROFILING_START;
@@ -367,9 +368,9 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                  * search for another occurrence of this content and see
                  * if the others match then until we run out of matches */
                 int r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, smd + 1, p, f,
-                        buffer, buffer_len, stream_start_offset, flags, inspection_mode);
-                if (r == 1) {
-                    SCReturnInt(1);
+                        buffer, buffer_len, stream_start_offset, flags, inspection_mode, local_id);
+                if (r == 1 || r == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF) {
+                    SCReturnInt(r);
                 } else if (r == -1) {
                     SCLogDebug("'next sm' said to discontinue this right now");
                     SCReturnInt(-1);
@@ -479,9 +480,9 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
              * search for another occurrence of this pcre and see
              * if the others match, until we run out of matches */
             r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, smd + 1, p, f, buffer,
-                    buffer_len, stream_start_offset, flags, inspection_mode);
-            if (r == 1) {
-                SCReturnInt(1);
+                    buffer_len, stream_start_offset, flags, inspection_mode, local_id);
+            if (r == 1 || r == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF) {
+                SCReturnInt(r);
             } else if (r == -1) {
                 SCReturnInt(-1);
             }
@@ -647,10 +648,23 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
 
         //PrintRawDataFp(stdout, buffer, buffer_len);
         const DetectDatasetData *sd = (const DetectDatasetData *) smd->ctx;
-        int r = DetectDatasetBufferMatch(det_ctx, sd, buffer, buffer_len); //TODO buffer offset?
-        if (r == 1) {
+        int r = DetectDatasetBufferMatch(
+                det_ctx, sd, buffer, buffer_len, local_id); // TODO buffer offset?
+        if (r == DETECT_ENGINE_INSPECT_SIG_MATCH) {
             goto match;
         }
+        if (r == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF) {
+            if (!smd->is_last) {
+                KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
+                r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, smd + 1, p, f, buffer,
+                        buffer_len, stream_start_offset, flags, inspection_mode, local_id);
+                if (r == 1)
+                    SCReturnInt(DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF);
+                SCReturnInt(r);
+            }
+            KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
+            SCReturnInt(DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF);
+        }
         goto no_match_discontinue;
 
     } else if (smd->type == DETECT_DATAREP) {
@@ -699,7 +713,8 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                     int r = DetectEngineContentInspectionInternal(det_ctx, ctx, s,
                             s->sm_arrays[DETECT_SM_LIST_BASE64_DATA], NULL, f,
                             det_ctx->base64_decoded, det_ctx->base64_decoded_len, 0,
-                            DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
+                            DETECT_CI_FLAGS_SINGLE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE,
+                            local_id);
                     if (r == 1) {
                         /* Base64 is a terminal list. */
                         goto final_match;
@@ -735,7 +750,7 @@ match:
     if (!smd->is_last) {
         KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
         int r = DetectEngineContentInspectionInternal(det_ctx, ctx, s, smd + 1, p, f, buffer,
-                buffer_len, stream_start_offset, flags, inspection_mode);
+                buffer_len, stream_start_offset, flags, inspection_mode, local_id);
         SCReturnInt(r);
     }
 final_match:
@@ -757,11 +772,11 @@ bool DetectEngineContentInspection(DetectEngineCtx *de_ctx, DetectEngineThreadCt
     det_ctx->buffer_offset = 0;
 
     int r = DetectEngineContentInspectionInternal(det_ctx, &ctx, s, smd, p, f, buffer, buffer_len,
-            stream_start_offset, flags, inspection_mode);
+            stream_start_offset, flags, inspection_mode, 0);
 #ifdef UNITTESTS
     ut_inspection_recursion_counter = ctx.recursion.count;
 #endif
-    return (r == 1);
+    return (r == 1 || r == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF);
 }
 
 /** \brief wrapper around DetectEngineContentInspectionInternal to return true/false only
@@ -778,7 +793,7 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
     det_ctx->buffer_offset = 0;
 
     int r = DetectEngineContentInspectionInternal(det_ctx, &ctx, s, smd, p, f, b->inspect,
-            b->inspect_len, b->inspect_offset, b->flags, inspection_mode);
+            b->inspect_len, b->inspect_offset, b->flags, inspection_mode, 0);
 #ifdef UNITTESTS
     ut_inspection_recursion_counter = ctx.recursion.count;
 #endif
@@ -803,6 +818,20 @@ bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd)
     return absent_data;
 }
 
+int DetectEngineContentInspectionBufferMulti(DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, const Signature *s, const SigMatchData *smd, Flow *f,
+        const InspectionBuffer *b, uint32_t local_id)
+{
+    struct DetectEngineContentInspectionCtx ctx = { .recursion.count = 0,
+        .recursion.limit = de_ctx->inspection_recursion_limit };
+
+    det_ctx->buffer_offset = 0;
+
+    return DetectEngineContentInspectionInternal(det_ctx, &ctx, s, smd, NULL, f, b->inspect,
+            b->inspect_len, b->inspect_offset, b->flags,
+            DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, local_id);
+}
+
 #ifdef UNITTESTS
 #include "tests/detect-engine-content-inspection.c"
 #endif

src/detect-engine-content-inspection.h

@@ -75,6 +75,10 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh
  *  \retval bool true to match on absent buffer, false otherwise */
 bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd);
 
+int DetectEngineContentInspectionBufferMulti(DetectEngineCtx *de_ctx,
+        DetectEngineThreadCtx *det_ctx, const Signature *s, const SigMatchData *smd, Flow *f,
+        const InspectionBuffer *b, uint32_t local_id);
+
 void DetectEngineContentInspectionRegisterTests(void);
 
 #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */

src/detect-engine-state.h

@@ -50,6 +50,10 @@
  *  indicate that one of the files matched, but that there are still
  *  more files that have ongoing inspection. */
 #define DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES 4
+/** Indicates that we matched on an occurence of a multi-buffer
+ *  but we want to try all occurences, for example,
+ *  to see which should go into a dataset */
+#define DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF 5
 
 /** number of DeStateStoreItem's in one DeStateStore object */
 #define DE_STATE_CHUNK_SIZE             15

src/detect-engine.c

@@ -731,7 +731,8 @@ static void AppendPacketInspectEngine(DetectEngineCtx *de_ctx,
 
 static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
         const DetectEngineAppInspectionEngine *t, Signature *s, SigMatchData *smd,
-        const int mpm_list, const int files_id, uint8_t *last_id, bool *head_is_mpm)
+        const int mpm_list, const int files_id, uint8_t *last_id, bool *head_is_mpm,
+        bool delay_postmatch)
 {
     if (t->alproto == ALPROTO_UNKNOWN) {
         /* special case, inspect engine applies to all protocols */
@@ -785,6 +786,9 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx,
     new_engine->match_on_null = smd ? DetectContentInspectionMatchOnAbsentBuffer(smd) : false;
     new_engine->progress = t->progress;
     new_engine->sub_state = t->sub_state;
+    if (delay_postmatch)
+        new_engine->progress = AppLayerParserGetStateProgressCompletionStatus(
+                t->alproto, t->dir == 0 ? STREAM_TOSERVER : STREAM_TOCLIENT);
     new_engine->v2 = t->v2;
     SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback,
             new_engine->v2.GetData, new_engine->v2.transforms);
@@ -972,7 +976,8 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
                 .sm_list_base = (uint16_t)sm_list,
                 .dir = dir,
             };
-            AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm);
+            AppendAppInspectEngine(
+                    de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm, false);
             SCLogDebug("sid %u: appended pass-tru engine at hook:%u sm_list:%d for "
                        "SIG_FLAG_INIT_HOOK_LTE",
                     s->id, state, sm_list);
@@ -1019,8 +1024,8 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
                             continue;
                         }
                     }
-                    AppendAppInspectEngine(
-                            de_ctx, t, s, smd, mpm_list, files_id, &last_id, &head_is_mpm);
+                    AppendAppInspectEngine(de_ctx, t, s, smd, mpm_list, files_id, &last_id,
+                            &head_is_mpm, s->init_data->buffers[x].delay_postmatch);
                 }
             }
         }
@@ -1055,7 +1060,8 @@ int DetectEngineAppInspectionEngine2Signature(DetectEngineCtx *de_ctx, Signature
             .sm_list_base = (uint16_t)s->init_data->hook.sm_list,
             .dir = dir,
         };
-        AppendAppInspectEngine(de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm);
+        AppendAppInspectEngine(
+                de_ctx, &t, s, NULL, mpm_list, files_id, &last_id, &head_is_mpm, false);
     }
 
     if ((s->init_data->init_flags & SIG_FLAG_INIT_STATE_MATCH) &&
@@ -2387,6 +2393,7 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx,
         transforms = engine->v2.transforms;
     }
 
+    uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
     do {
         InspectionBuffer *buffer = DetectGetMultiData(det_ctx, transforms, f, flags, txv,
                 engine->sm_list, local_id, engine->v2.GetMultiData);
@@ -2396,10 +2403,14 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx,
 
         // The GetData functions set buffer->flags to DETECT_CI_FLAGS_SINGLE
         // This is not meant for streaming buffers
-        const bool match = DetectEngineContentInspectionBuffer(de_ctx, det_ctx, s, engine->smd,
-                NULL, f, buffer, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE);
-        if (match) {
-            return DETECT_ENGINE_INSPECT_SIG_MATCH;
+        int match = DetectEngineContentInspectionBufferMulti(
+                de_ctx, det_ctx, s, engine->smd, f, buffer, local_id);
+        switch (match) {
+            case DETECT_ENGINE_INSPECT_SIG_MATCH:
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
+            case DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF:
+                r = DETECT_ENGINE_INSPECT_SIG_MATCH;
+                break;
         }
         local_id++;
     } while (1);
@@ -2410,7 +2421,7 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx,
             return DETECT_ENGINE_INSPECT_SIG_MATCH;
         }
     }
-    return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    return r;
 }
 
 /**

src/detect.c

@@ -260,9 +260,8 @@ end:
     SCReturn;
 }
 
-static void DetectRunPostMatch(ThreadVars *tv,
-                               DetectEngineThreadCtx *det_ctx, Packet *p,
-                               const Signature *s)
+static void DetectRunPostMatch(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Packet *p,
+        const Signature *s, Flow *f, uint8_t flags, void *alstate, void *txv)
 {
     /* run the packet match functions */
     const SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_POSTMATCH];
@@ -273,6 +272,10 @@ static void DetectRunPostMatch(ThreadVars *tv,
 
         while (1) {
             KEYWORD_PROFILING_START;
+            if (txv && sigmatch_table[smd->type].AppLayerTxMatch != NULL) {
+                sigmatch_table[smd->type].AppLayerTxMatch(
+                        det_ctx, f, flags, alstate, txv, s, smd->ctx);
+            }
             (void)sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx);
             KEYWORD_PROFILING_END(det_ctx, smd->type, 1);
             if (smd->is_last)
@@ -888,7 +891,7 @@ static inline uint8_t DetectRulePacketRules(ThreadVars *const tv,
 #ifdef PROFILE_RULES
         smatch = true;
 #endif
-        DetectRunPostMatch(tv, det_ctx, p, s);
+        DetectRunPostMatch(tv, det_ctx, p, s, NULL, 0, NULL, NULL);
 
         DetectRulePacketAppendAlert(de_ctx, det_ctx, s, p, pflow, alert_flags);
 
@@ -2850,7 +2853,7 @@ static void DetectRunTx(ThreadVars *tv,
             }
             if (r == 1) {
                 /* match */
-                DetectRunPostMatch(tv, det_ctx, p, s);
+                DetectRunPostMatch(tv, det_ctx, p, s, f, flow_flags, alstate, tx.tx_ptr);
 
                 SCLogDebug(
                         "%p/%" PRIu64 " sig %u (%u) matched", tx.tx_ptr, tx.tx_id, s->id, s->iid);
@@ -3087,7 +3090,7 @@ static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngin
                 r = DetectRunFrameInspectRule(tv, det_ctx, s, f, p, frames, frame);
                 if (r) {
                     /* match */
-                    DetectRunPostMatch(tv, det_ctx, p, s);
+                    DetectRunPostMatch(tv, det_ctx, p, s, NULL, 0, NULL, NULL);
                     det_ctx->frame_id = frame->id;
                     SCLogDebug(
                             "%p/%" PRIi64 " sig %u (%u) matched", frame, frame->id, s->id, s->iid);

src/detect.h

@@ -539,6 +539,9 @@ typedef struct SignatureInitDataBuffer_ {
                            http.uri. */
     bool only_tc;       /**< true if we can only used toclient. */
     bool only_ts;       /**< true if we can only used toserver. */
+    /** delay use of this buffer beyond its normal progress
+     * to be just in time for postmatch */
+    bool delay_postmatch;
     /* sig match list */
     SigMatch *head;
     SigMatch *tail;