detect: improve inspect buffer handling

Fix and Optimize cleanup. For the simple single inspect buffer optimize
the cleanup by keeping track of the actually used buffers. This avoid
looping over unused buffers.

Fix the case of cleaning not being done after a tx if the next tx is
also inspected in the context of the same packet.

Fix cleanup of the multi-inspect buffers. Optimize in 2 ways. First
like with single keep track of which multi-inspect buffers have been
used. Second, keep a max of the buffers within a multi-inspect buffer.
Use this max to limit (nested) looping.

src/detect-dns-query.c

@@ -77,41 +77,14 @@ struct DnsQueryGetDataArgs {
 #endif
 };
 
-/** \brief get an InspectionBuffer. Make space if we have to. */
-static InspectionBuffer *GetBuffer(InspectionBufferMultipleForList *fb, uint32_t id)
-{
-    if (id >= fb->size) {
-        uint32_t old_size = fb->size;
-        uint32_t new_size = id + 1;
-        uint32_t grow_by = new_size - old_size;
-        SCLogDebug("size is %u, need %u, so growing by %u",
-                old_size, new_size, grow_by);
-
-        void *ptr = SCRealloc(fb->inspection_buffers, (id + 1) * sizeof(InspectionBuffer));
-        if (ptr == NULL)
-            return NULL;
-
-        InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size;
-        SCLogDebug("fb->inspection_buffers %p ptr %p to_zero %p",
-                fb->inspection_buffers, ptr, to_zero);
-        memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer)));
-        fb->inspection_buffers = ptr;
-        fb->size = new_size;
-    }
-
-    InspectionBuffer *buffer = &fb->inspection_buffers[id];
-    SCLogDebug("using file_data buffer %p", buffer);
-    return buffer;
-}
-
 static InspectionBuffer *DnsQueryGetData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms,
         Flow *f, struct DnsQueryGetDataArgs *cbdata, int list_id, bool first)
 {
     SCEnter();
 
-    InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[list_id];
-    InspectionBuffer *buffer = GetBuffer(fb, cbdata->local_id);
+    InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id);
+    InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, cbdata->local_id);
     if (buffer == NULL)
         return NULL;
     if (!first && buffer->inspect != NULL)

src/detect-engine-filedata.c

@@ -36,31 +36,6 @@
 
 #include "app-layer-parser.h"
 
-static InspectionBuffer *GetBuffer(InspectionBufferMultipleForList *fb, uint32_t id)
-{
-    if (id >= fb->size) {
-        uint32_t old_size = fb->size;
-        uint32_t new_size = id + 1;
-        uint32_t grow_by = new_size - old_size;
-        SCLogDebug("size is %u, need %u, so growing by %u", old_size, new_size, grow_by);
-
-        SCLogDebug("fb->inspection_buffers %p", fb->inspection_buffers);
-        void *ptr = SCRealloc(fb->inspection_buffers, (id + 1) * sizeof(InspectionBuffer));
-        if (ptr == NULL)
-            return NULL;
-
-        InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size;
-        SCLogDebug("ptr %p to_zero %p", ptr, to_zero);
-        memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer)));
-        fb->inspection_buffers = ptr;
-        fb->size = new_size;
-    }
-
-    InspectionBuffer *buffer = &fb->inspection_buffers[id];
-    SCLogDebug("using file_data buffer %p", buffer);
-    return buffer;
-}
-
 static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms,
         Flow *f, uint8_t flow_flags, File *cur_file,
@@ -68,8 +43,8 @@ static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
 {
     SCEnter();
 
-    InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[list_id];
-    InspectionBuffer *buffer = GetBuffer(fb, local_file_id);
+    InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id);
+    InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, local_file_id);
     if (buffer == NULL)
         return NULL;
     if (!first && buffer->inspect != NULL)

src/detect-engine-hsbd.c

@@ -85,7 +85,7 @@ InspectionBuffer *HttpServerBodyGetDataCallback(DetectEngineThreadCtx *det_ctx,
 {
     SCEnter();
 
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect != NULL)
         return buffer;
 

src/detect-engine.c

@@ -844,6 +844,81 @@ int DetectBufferGetActiveList(DetectEngineCtx *de_ctx, Signature *s)
     return 0;
 }
 
+void InspectionBufferClean(DetectEngineThreadCtx *det_ctx)
+{
+    /* single buffers */
+    for (uint32_t i = 0; i < det_ctx->inspect.to_clear_idx; i++)
+    {
+        const uint32_t idx = det_ctx->inspect.to_clear_queue[i];
+        InspectionBuffer *buffer = &det_ctx->inspect.buffers[idx];
+        buffer->inspect = NULL;
+    }
+    det_ctx->inspect.to_clear_idx = 0;
+
+    /* multi buffers */
+    for (uint32_t i = 0; i < det_ctx->multi_inspect.to_clear_idx; i++)
+    {
+        const uint32_t idx = det_ctx->multi_inspect.to_clear_queue[i];
+        InspectionBufferMultipleForList *mbuffer = &det_ctx->multi_inspect.buffers[idx];
+        for (uint32_t x = 0; x <= mbuffer->max; x++) {
+            InspectionBuffer *buffer = &mbuffer->inspection_buffers[x];
+            buffer->inspect = NULL;
+        }
+        mbuffer->init = 0;
+        mbuffer->max = 0;
+    }
+    det_ctx->multi_inspect.to_clear_idx = 0;
+}
+
+InspectionBuffer *InspectionBufferGet(DetectEngineThreadCtx *det_ctx, const int list_id)
+{
+    InspectionBuffer *buffer = &det_ctx->inspect.buffers[list_id];
+    if (buffer->inspect == NULL) {
+        det_ctx->inspect.to_clear_queue[det_ctx->inspect.to_clear_idx++] = list_id;
+    }
+    return buffer;
+}
+
+/** \brief for a InspectionBufferMultipleForList get a InspectionBuffer
+ *  \param fb the multiple buffer array
+ *  \param local_id the index to get a buffer
+ *  \param buffer the inspect buffer or NULL in case of error */
+InspectionBuffer *InspectionBufferMultipleForListGet(InspectionBufferMultipleForList *fb, uint32_t local_id)
+{
+    if (local_id >= fb->size) {
+        uint32_t old_size = fb->size;
+        uint32_t new_size = local_id + 1;
+        uint32_t grow_by = new_size - old_size;
+        SCLogDebug("size is %u, need %u, so growing by %u", old_size, new_size, grow_by);
+
+        SCLogDebug("fb->inspection_buffers %p", fb->inspection_buffers);
+        void *ptr = SCRealloc(fb->inspection_buffers, (local_id + 1) * sizeof(InspectionBuffer));
+        if (ptr == NULL)
+            return NULL;
+
+        InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size;
+        SCLogDebug("ptr %p to_zero %p", ptr, to_zero);
+        memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer)));
+        fb->inspection_buffers = ptr;
+        fb->size = new_size;
+    }
+
+    fb->max = MAX(fb->max, local_id);
+    InspectionBuffer *buffer = &fb->inspection_buffers[local_id];
+    SCLogDebug("using file_data buffer %p", buffer);
+    return buffer;
+}
+
+InspectionBufferMultipleForList *InspectionBufferGetMulti(DetectEngineThreadCtx *det_ctx, const int list_id)
+{
+    InspectionBufferMultipleForList *buffer = &det_ctx->multi_inspect.buffers[list_id];
+    if (!buffer->init) {
+        det_ctx->multi_inspect.to_clear_queue[det_ctx->multi_inspect.to_clear_idx++] = list_id;
+        buffer->init = 1;
+    }
+    return buffer;
+}
+
 void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size)
 {
     memset(buffer, 0, sizeof(*buffer));
@@ -2253,16 +2328,28 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *
         det_ctx->base64_decoded_len = 0;
     }
 
-    det_ctx->inspect_buffers_size = de_ctx->buffer_type_id;
-    det_ctx->inspect_buffers = SCCalloc(det_ctx->inspect_buffers_size, sizeof(InspectionBuffer));
-    if (det_ctx->inspect_buffers == NULL) {
+    det_ctx->inspect.buffers_size = de_ctx->buffer_type_id;
+    det_ctx->inspect.buffers = SCCalloc(det_ctx->inspect.buffers_size, sizeof(InspectionBuffer));
+    if (det_ctx->inspect.buffers == NULL) {
         return TM_ECODE_FAILED;
     }
-    det_ctx->multi_inspect_buffers_size = de_ctx->buffer_type_id;
-    det_ctx->multi_inspect_buffers = SCCalloc(det_ctx->multi_inspect_buffers_size, sizeof(InspectionBufferMultipleForList));
-    if (det_ctx->multi_inspect_buffers == NULL) {
+    det_ctx->inspect.to_clear_queue = SCCalloc(det_ctx->inspect.buffers_size, sizeof(uint32_t));
+    if (det_ctx->inspect.to_clear_queue == NULL) {
         return TM_ECODE_FAILED;
     }
+    det_ctx->inspect.to_clear_idx = 0;
+
+    det_ctx->multi_inspect.buffers_size = de_ctx->buffer_type_id;
+    det_ctx->multi_inspect.buffers = SCCalloc(det_ctx->multi_inspect.buffers_size, sizeof(InspectionBufferMultipleForList));
+    if (det_ctx->multi_inspect.buffers == NULL) {
+        return TM_ECODE_FAILED;
+    }
+    det_ctx->multi_inspect.to_clear_queue = SCCalloc(det_ctx->multi_inspect.buffers_size, sizeof(uint32_t));
+    if (det_ctx->multi_inspect.to_clear_queue == NULL) {
+        return TM_ECODE_FAILED;
+    }
+    det_ctx->multi_inspect.to_clear_idx = 0;
+
 
     DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx);
     DetectEngineThreadCtxInitGlobalKeywords(det_ctx);
@@ -2469,23 +2556,28 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
         SCFree(det_ctx->base64_decoded);
     }
 
-    if (det_ctx->inspect_buffers) {
-        for (uint32_t i = 0; i < det_ctx->inspect_buffers_size; i++) {
-            InspectionBufferFree(&det_ctx->inspect_buffers[i]);
+    if (det_ctx->inspect.buffers) {
+        for (uint32_t i = 0; i < det_ctx->inspect.buffers_size; i++) {
+            InspectionBufferFree(&det_ctx->inspect.buffers[i]);
         }
-        SCFree(det_ctx->inspect_buffers);
+        SCFree(det_ctx->inspect.buffers);
+    }
+    if (det_ctx->inspect.to_clear_queue) {
+        SCFree(det_ctx->inspect.to_clear_queue);
     }
-    if (det_ctx->multi_inspect_buffers) {
-        for (uint32_t i = 0; i < det_ctx->multi_inspect_buffers_size; i++) {
-            InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[i];
+    if (det_ctx->multi_inspect.buffers) {
+        for (uint32_t i = 0; i < det_ctx->multi_inspect.buffers_size; i++) {
+            InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect.buffers[i];
             for (uint32_t x = 0; x < fb->size; x++) {
                 InspectionBufferFree(&fb->inspection_buffers[x]);
             }
             SCFree(fb->inspection_buffers);
         }
-        SCFree(det_ctx->multi_inspect_buffers);
+        SCFree(det_ctx->multi_inspect.buffers);
+    }
+    if (det_ctx->multi_inspect.to_clear_queue) {
+        SCFree(det_ctx->multi_inspect.to_clear_queue);
     }
-
 
     DetectEngineThreadCtxDeinitGlobalKeywords(det_ctx);
     if (det_ctx->de_ctx != NULL) {

src/detect-engine.h

@@ -35,6 +35,10 @@ void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
 void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len);
 void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
         const DetectEngineTransforms *transforms);
+void InspectionBufferClean(DetectEngineThreadCtx *det_ctx);
+InspectionBuffer *InspectionBufferGet(DetectEngineThreadCtx *det_ctx, const int list_id);
+InspectionBuffer *InspectionBufferMultipleForListGet(InspectionBufferMultipleForList *fb, uint32_t local_id);
+InspectionBufferMultipleForList *InspectionBufferGetMulti(DetectEngineThreadCtx *det_ctx, const int list_id);
 
 int DetectBufferTypeRegister(const char *name);
 int DetectBufferTypeGetByName(const char *name);

src/detect-http-request-line.c

@@ -122,9 +122,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         Flow *_f, const uint8_t _flow_flags,
         void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         htp_tx_t *tx = (htp_tx_t *)txv;
         if (unlikely(tx->request_line == NULL)) {

src/detect-krb5-cname.c

@@ -55,33 +55,6 @@ static int DetectKrb5CNameSetup(DetectEngineCtx *de_ctx, Signature *s, const cha
     return 0;
 }
 
-/** \brief get an InspectionBuffer. Make space if we have to. */
-static InspectionBuffer *GetBuffer(InspectionBufferMultipleForList *fb, uint32_t id)
-{
-    if (id >= fb->size) {
-        uint32_t old_size = fb->size;
-        uint32_t new_size = id + 1;
-        uint32_t grow_by = new_size - old_size;
-        SCLogDebug("size is %u, need %u, so growing by %u",
-                old_size, new_size, grow_by);
-
-        void *ptr = SCRealloc(fb->inspection_buffers, (id + 1) * sizeof(InspectionBuffer));
-        if (ptr == NULL)
-            return NULL;
-
-        InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size;
-        SCLogDebug("fb->inspection_buffers %p ptr %p to_zero %p",
-                fb->inspection_buffers, ptr, to_zero);
-        memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer)));
-        fb->inspection_buffers = ptr;
-        fb->size = new_size;
-    }
-
-    InspectionBuffer *buffer = &fb->inspection_buffers[id];
-    SCLogDebug("using file_data buffer %p", buffer);
-    return buffer;
-}
-
 static InspectionBuffer *GetKrb5CNameData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms,
         Flow *_f, const struct Krb5PrincipalNameDataArgs *cbdata,
@@ -89,8 +62,8 @@ static InspectionBuffer *GetKrb5CNameData(DetectEngineThreadCtx *det_ctx,
 {
     SCEnter();
 
-    InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[list_id];
-    InspectionBuffer *buffer = GetBuffer(fb, cbdata->local_id);
+    InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id);
+    InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, cbdata->local_id);
     if (buffer == NULL)
         return NULL;
     if (!first && buffer->inspect != NULL)

src/detect-krb5-sname.c

@@ -55,33 +55,6 @@ static int DetectKrb5SNameSetup(DetectEngineCtx *de_ctx, Signature *s, const cha
     return 0;
 }
 
-/** \brief get an InspectionBuffer. Make space if we have to. */
-static InspectionBuffer *GetBuffer(InspectionBufferMultipleForList *fb, uint32_t id)
-{
-    if (id >= fb->size) {
-        uint32_t old_size = fb->size;
-        uint32_t new_size = id + 1;
-        uint32_t grow_by = new_size - old_size;
-        SCLogDebug("size is %u, need %u, so growing by %u",
-                old_size, new_size, grow_by);
-
-        void *ptr = SCRealloc(fb->inspection_buffers, (id + 1) * sizeof(InspectionBuffer));
-        if (ptr == NULL)
-            return NULL;
-
-        InspectionBuffer *to_zero = (InspectionBuffer *)ptr + old_size;
-        SCLogDebug("fb->inspection_buffers %p ptr %p to_zero %p",
-                fb->inspection_buffers, ptr, to_zero);
-        memset((uint8_t *)to_zero, 0, (grow_by * sizeof(InspectionBuffer)));
-        fb->inspection_buffers = ptr;
-        fb->size = new_size;
-    }
-
-    InspectionBuffer *buffer = &fb->inspection_buffers[id];
-    SCLogDebug("using file_data buffer %p", buffer);
-    return buffer;
-}
-
 static InspectionBuffer *GetKrb5SNameData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms,
         Flow *_f, const struct Krb5PrincipalNameDataArgs *cbdata,
@@ -89,8 +62,8 @@ static InspectionBuffer *GetKrb5SNameData(DetectEngineThreadCtx *det_ctx,
 {
     SCEnter();
 
-    InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[list_id];
-    InspectionBuffer *buffer = GetBuffer(fb, cbdata->local_id);
+    InspectionBufferMultipleForList *fb = InspectionBufferGetMulti(det_ctx, list_id);
+    InspectionBuffer *buffer = InspectionBufferMultipleForListGet(fb, cbdata->local_id);
     if (buffer == NULL)
         return NULL;
     if (!first && buffer->inspect != NULL)

src/detect-smb-share.c

@@ -56,9 +56,7 @@ static InspectionBuffer *GetNamedPipeData(DetectEngineThreadCtx *det_ctx,
         Flow *_f, const uint8_t _flow_flags,
         void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         uint32_t b_len = 0;
         uint8_t *b = NULL;
@@ -119,9 +117,7 @@ static InspectionBuffer *GetShareData(DetectEngineThreadCtx *det_ctx,
         Flow *_f, const uint8_t _flow_flags,
         void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         uint32_t b_len = 0;
         uint8_t *b = NULL;

src/detect-template-buffer.c

@@ -117,14 +117,11 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         Flow *_f, const uint8_t flow_flags,
         void *txv, const int list_id)
 {
-    const uint8_t *data = NULL;
-    uint32_t data_len = 0;
-
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         const TemplateTransaction  *tx = (TemplateTransaction *)txv;
+        const uint8_t *data = NULL;
+        uint32_t data_len = 0;
 
         if (flow_flags & STREAM_TOSERVER) {
             data = tx->request_buffer;

src/detect-tls-cert-fingerprint.c

@@ -125,9 +125,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Flow *_f,
         const uint8_t _flow_flags, void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         SSLState *ssl_state = (SSLState *)_f->alstate;
 

src/detect-tls-cert-issuer.c

@@ -112,9 +112,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Flow *_f,
         const uint8_t _flow_flags, void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         SSLState *ssl_state = (SSLState *)_f->alstate;
 

src/detect-tls-cert-serial.c

@@ -124,9 +124,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Flow *_f,
         const uint8_t _flow_flags, void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         SSLState *ssl_state = (SSLState *)_f->alstate;
 

src/detect-tls-cert-subject.c

@@ -111,9 +111,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Flow *_f,
         const uint8_t _flow_flags, void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         SSLState *ssl_state = (SSLState *)_f->alstate;
 

src/detect-tls-ja3-hash.c

@@ -130,9 +130,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Flow *_f,
         const uint8_t _flow_flags, void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         SSLState *ssl_state = (SSLState *)_f->alstate;
 

src/detect-tls-ja3-string.c

@@ -120,9 +120,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Flow *_f,
         const uint8_t _flow_flags, void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         SSLState *ssl_state = (SSLState *)_f->alstate;
 

src/detect-tls-sni.c

@@ -110,9 +110,7 @@ static InspectionBuffer *GetData(DetectEngineThreadCtx *det_ctx,
         const DetectEngineTransforms *transforms, Flow *_f,
         const uint8_t _flow_flags, void *txv, const int list_id)
 {
-    BUG_ON(det_ctx->inspect_buffers == NULL);
-    InspectionBuffer *buffer = &det_ctx->inspect_buffers[list_id];
-
+    InspectionBuffer *buffer = InspectionBufferGet(det_ctx, list_id);
     if (buffer->inspect == NULL) {
         SSLState *ssl_state = (SSLState *)_f->alstate;
 

src/detect.c

@@ -1038,13 +1038,6 @@ static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
     PacketPatternCleanup(det_ctx);
 
     if (pflow != NULL) {
-        // TODO clean this up
-        const int nlists = det_ctx->de_ctx->buffer_type_id;
-        for (int i = 0; i < nlists; i++) {
-            InspectionBuffer *buffer = &det_ctx->inspect_buffers[i];
-            buffer->inspect = NULL;
-        }
-
         /* update inspected tracker for raw reassembly */
         if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL) {
             StreamReassembleRawUpdateProgress(pflow->protoctx, p,
@@ -1600,6 +1593,8 @@ static void DetectRunTx(ThreadVars *tv,
                     flow_flags, new_detect_flags);
         }
 next:
+        InspectionBufferClean(det_ctx);
+
         if (!ires.has_next)
             break;
     }

src/detect.h

@@ -366,7 +366,9 @@ typedef struct InspectionBuffer {
 
 typedef struct InspectionBufferMultipleForList {
     InspectionBuffer *inspection_buffers;
-    uint32_t size;  /**< size in number of elements */
+    uint32_t size;      /**< size in number of elements */
+    uint32_t max:31;    /**< max id in use in this run */
+    uint32_t init:1;    /**< first time used this run. Used for clean logic */
 } InspectionBufferMultipleForList;
 
 typedef struct DetectEngineTransforms {
@@ -1008,14 +1010,22 @@ typedef struct DetectEngineThreadCtx_ {
 #endif
 
     int inspect_list; /**< list we're currently inspecting, DETECT_SM_LIST_* */
-    InspectionBuffer *inspect_buffers;
 
-    uint32_t inspect_buffers_size;          /**< in number of elements */
-    uint32_t multi_inspect_buffers_size;    /**< in number of elements */
+    struct {
+        InspectionBuffer *buffers;
+        uint32_t buffers_size;          /**< in number of elements */
+        uint32_t to_clear_idx;
+        uint32_t *to_clear_queue;
+    } inspect;
 
-    /* inspection buffers for more complete case. As we can inspect multiple
-     * buffers in parallel, we need this extra wrapper struct */
-    InspectionBufferMultipleForList *multi_inspect_buffers;
+    struct {
+        /** inspection buffers for more complex case. As we can inspect multiple
+         *  buffers in parallel, we need this extra wrapper struct */
+        InspectionBufferMultipleForList *buffers;
+        uint32_t buffers_size;                      /**< in number of elements */
+        uint32_t to_clear_idx;
+        uint32_t *to_clear_queue;
+    } multi_inspect;
 
     /* used to discontinue any more matching */
     uint16_t discontinue_matching;