http body handling: use streaming buffer API

Convert HTTP body handling to use the Streaming Buffer API. This means
the HtpBodyChunks no longer maintain their own data segments, but
instead add their data to the StreamingBuffer instance in the HtpBody
structure.

In case the HtpBodyChunk needs to access it's data it can do so still
through the Streaming Buffer API.

Updates & simplifies the various users of the reassembled bodies:
multipart parsing and the detection engine.

src/app-layer-htp-body.c

@@ -62,6 +62,9 @@
 
 #include "util-memcmp.h"
 
+static StreamingBufferConfig default_cfg = {
+    0, 0, 3072, HTPMalloc, HTPCalloc, HTPRealloc, HTPFree };
+
 /**
  * \brief Append a chunk of body to the HtpBody struct
  *
@@ -82,20 +85,19 @@ int HtpBodyAppendChunk(HtpBody *body, const uint8_t *data, uint32_t len)
         SCReturnInt(0);
     }
 
+    if (body->sb == NULL) {
+        body->sb = StreamingBufferInit(&default_cfg);
+        if (body->sb == NULL)
+            SCReturnInt(-1);
+    }
+
     if (body->first == NULL) {
         /* New chunk */
         bd = (HtpBodyChunk *)HTPCalloc(1, sizeof(HtpBodyChunk));
         if (bd == NULL)
             goto error;
 
-        bd->len = len;
-        bd->stream_offset = 0;
-
-        bd->data = HTPCalloc(1, len);
-        if (bd->data == NULL) {
-            goto error;
-        }
-        memcpy(bd->data, data, len);
+        StreamingBufferAppend(body->sb, &bd->sbseg, data, len);
 
         body->first = body->last = bd;
 
@@ -105,29 +107,19 @@ int HtpBodyAppendChunk(HtpBody *body, const uint8_t *data, uint32_t len)
         if (bd == NULL)
             goto error;
 
-        bd->len = len;
-        bd->stream_offset = body->content_len_so_far;
-
-        bd->data = HTPCalloc(1, len);
-        if (bd->data == NULL) {
-            goto error;
-        }
-        memcpy(bd->data, data, len);
+        StreamingBufferAppend(body->sb, &bd->sbseg, data, len);
 
         body->last->next = bd;
         body->last = bd;
 
         body->content_len_so_far += len;
     }
-    SCLogDebug("Body %p; data %p, len %"PRIu32, body, bd->data, (uint32_t)bd->len);
+    SCLogDebug("body %p", body);
 
     SCReturnInt(0);
 
 error:
     if (bd != NULL) {
-        if (bd->data != NULL) {
-            HTPFree(bd->data, bd->len);
-        }
         HTPFree(bd, sizeof(HtpBodyChunk));
     }
     SCReturnInt(-1);
@@ -150,9 +142,12 @@ void HtpBodyPrint(HtpBody *body)
         SCLogDebug("--- Start body chunks at %p ---", body);
         printf("--- Start body chunks at %p ---\n", body);
         for (cur = body->first; cur != NULL; cur = cur->next) {
-            SCLogDebug("Body %p; data %p, len %"PRIu32, body, cur->data, (uint32_t)cur->len);
-            printf("Body %p; data %p, len %"PRIu32"\n", body, cur->data, (uint32_t)cur->len);
-            PrintRawDataFp(stdout, (uint8_t*)cur->data, cur->len);
+            const uint8_t *data = NULL;
+            uint32_t data_len = 0;
+            StreamingBufferSegmentGetData(body->sb, &cur->sbseg, &data, &data_len);
+            SCLogDebug("Body %p; data %p, len %"PRIu32, body, data, data_len);
+            printf("Body %p; data %p, len %"PRIu32"\n", body, data, data_len);
+            PrintRawDataFp(stdout, data, data_len);
         }
         SCLogDebug("--- End body chunks at %p ---", body);
     }
@@ -170,8 +165,7 @@ void HtpBodyFree(HtpBody *body)
     if (body->first == NULL)
         return;
 
-    SCLogDebug("Removing chunks of Body %p; data %p, len %"PRIu32, body,
-            body->last->data, (uint32_t)body->last->len);
+    SCLogDebug("removing chunks of body %p", body);
 
     HtpBodyChunk *cur = NULL;
     HtpBodyChunk *prev = NULL;
@@ -179,12 +173,12 @@ void HtpBodyFree(HtpBody *body)
     prev = body->first;
     while (prev != NULL) {
         cur = prev->next;
-        if (prev->data != NULL)
-            HTPFree(prev->data, prev->len);
         HTPFree(prev, sizeof(HtpBodyChunk));
         prev = cur;
     }
     body->first = body->last = NULL;
+
+    StreamingBufferFree(body->sb);
 }
 
 /**
@@ -230,24 +224,26 @@ void HtpBodyPrune(HtpState *state, HtpBody *body, int direction)
         SCReturn;
     }
 
-    SCLogDebug("Pruning chunks of Body %p; data %p, len %"PRIu32, body,
-            body->last->data, (uint32_t)body->last->len);
+    uint64_t left_edge = body->body_inspected;
+    if (left_edge <= min_size || left_edge <= window)
+        left_edge = 0;
+    if (left_edge)
+        left_edge -= window;
+
+    if (left_edge) {
+        SCLogDebug("sliding body to offset %"PRIu64, left_edge);
+        StreamingBufferSlideToOffset(body->sb, left_edge);
+    }
+
+    SCLogDebug("pruning chunks of body %p", body);
 
     HtpBodyChunk *cur = body->first;
     while (cur != NULL) {
         HtpBodyChunk *next = cur->next;
+        SCLogDebug("cur %p", cur);
 
-        SCLogDebug("cur->stream_offset %"PRIu64" + cur->len %u = %"PRIu64", "
-                "body->body_parsed %"PRIu64, cur->stream_offset, cur->len,
-                cur->stream_offset + cur->len, body->body_parsed);
-
-        uint64_t left_edge = body->body_inspected;
-        if (left_edge <= min_size || left_edge <= window)
-            left_edge = 0;
-        if (left_edge)
-            left_edge -= window;
-
-        if (cur->stream_offset + cur->len > left_edge) {
+        if (!StreamingBufferSegmentIsBeforeWindow(body->sb, &cur->sbseg)) {
+            SCLogDebug("not removed");
             break;
         }
 
@@ -256,12 +252,10 @@ void HtpBodyPrune(HtpState *state, HtpBody *body, int direction)
             body->last = next;
         }
 
-        if (cur->data != NULL) {
-            HTPFree(cur->data, cur->len);
-        }
         HTPFree(cur, sizeof(HtpBodyChunk));
 
         cur = next;
+        SCLogDebug("removed");
     }
 
     SCReturn;

src/app-layer-htp-file.c

@@ -629,8 +629,11 @@ static int HTPFileParserTest03(void)
         goto end;
     }
 
-    if (http_state->files_ts->head->chunks_head->len != 11) {
-        printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
+    if (http_state->files_ts->head->chunks_head == NULL ||
+        http_state->files_ts->head->chunks_head->len != 11)
+    {
+        if (http_state->files_ts->head->chunks_head != NULL)
+            printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
         goto end;
     }
 

src/app-layer-htp.c

@@ -1170,60 +1170,11 @@ static void HtpRequestBodyMultipartParseHeader(HtpState *hstate,
  *  \param chunks_buffer_len pointer to pass back the buffer length to the caller
  */
 static void HtpRequestBodyReassemble(HtpTxUserData *htud,
-        uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
+        const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len)
 {
-    uint8_t *buf = NULL;
-    uint8_t *pbuf = NULL;
-    uint32_t buf_len = 0;
-    HtpBodyChunk *cur = htud->request_body.first;
-
-    for ( ; cur != NULL; cur = cur->next) {
-        SCLogDebug("chunk %p", cur);
-
-        /* skip body chunks entirely before what we parsed already */
-        if ((uint64_t )cur->stream_offset + cur->len <= htud->request_body.body_parsed) {
-            SCLogDebug("skipping chunk");
-            continue;
-        }
-
-        SCLogDebug("cur->stream_offset %"PRIu64", cur->len %"PRIu32", body_parsed %"PRIu64,
-            cur->stream_offset, cur->len, htud->request_body.body_parsed);
-
-        if (cur->stream_offset < htud->request_body.body_parsed &&
-                cur->stream_offset + cur->len >= htud->request_body.body_parsed) {
-            SCLogDebug("use part");
-
-            uint32_t toff = htud->request_body.body_parsed - cur->stream_offset;
-            uint32_t tlen = (cur->stream_offset + cur->len) - htud->request_body.body_parsed;
-            uint8_t *pbuf = NULL;
-
-            buf_len += tlen;
-            if ((pbuf = HTPRealloc(buf, buf_len - tlen, buf_len)) == NULL) {
-                HTPFree(buf, buf_len - tlen);
-                buf = NULL;
-                buf_len = 0;
-                break;
-            }
-            buf = pbuf;
-            memcpy(buf + buf_len - tlen, cur->data + toff, tlen);
-
-        } else {
-            SCLogDebug("use entire chunk");
-
-            buf_len += cur->len;
-            if ((pbuf = HTPRealloc(buf, buf_len - cur->len, buf_len)) == NULL) {
-                HTPFree(buf, buf_len - cur->len);
-                buf = NULL;
-                buf_len = 0;
-                break;
-            }
-            buf = pbuf;
-            memcpy(buf + buf_len - cur->len, cur->data, cur->len);
-        }
-    }
-
-    *chunks_buffer = buf;
-    *chunks_buffer_len = buf_len;
+    StreamingBufferGetDataAtOffset(htud->request_body.sb,
+            chunks_buffer, chunks_buffer_len,
+            htud->request_body.body_parsed);
 }
 
 static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir)
@@ -1249,8 +1200,8 @@ static void HtpRequestBodySetupBoundary(HtpTxUserData *htud,
     memcpy(boundary + 2, htud->boundary, htud->boundary_len);
 }
 
-int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
-                                  void *tx, uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
+int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx,
+        const uint8_t *chunks_buffer, uint32_t chunks_buffer_len)
 {
     int result = 0;
     uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */
@@ -1288,7 +1239,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
         if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) {
             SCLogDebug("reached the end of the file");
 
-            uint8_t *filedata = chunks_buffer;
+            const uint8_t *filedata = chunks_buffer;
             uint32_t filedata_len = 0;
             uint8_t flags = 0;
 
@@ -1328,7 +1279,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
             SCLogDebug("not yet at the end of the file");
 
             if (chunks_buffer_len > expected_boundary_end_len) {
-                uint8_t *filedata = chunks_buffer;
+                const uint8_t *filedata = chunks_buffer;
                 uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
 #ifdef PRINT
                 printf("FILEDATA (part) START: \n");
@@ -1816,7 +1767,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
         HtpBodyAppendChunk(&tx_ud->request_body, d->data, len);
 
-        uint8_t *chunks_buffer = NULL;
+        const uint8_t *chunks_buffer = NULL;
         uint32_t chunks_buffer_len = 0;
 
         if (tx_ud->request_body_type == HTP_BODY_REQUEST_MULTIPART) {
@@ -1837,9 +1788,6 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
             HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len);
 
-            if (chunks_buffer != NULL) {
-                HTPFree(chunks_buffer, chunks_buffer_len);
-            }
         } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) {
             HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len);
         } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) {
@@ -2189,6 +2137,10 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
 #endif
     cfg_prec->randomize_range = HTP_CONFIG_DEFAULT_RANDOMIZE_RANGE;
 
+    cfg_prec->sbcfg.flags = 0;
+    cfg_prec->sbcfg.buf_size = cfg_prec->request_inspect_window;
+    cfg_prec->sbcfg.buf_slide = 0;
+
     htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
     htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData);
     htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData);
@@ -5666,7 +5618,7 @@ static int HTPBodyReassemblyTest01(void)
     r = HtpBodyAppendChunk(&htud.request_body, chunk2, sizeof(chunk2)-1);
     BUG_ON(r != 0);
 
-    uint8_t *chunks_buffer = NULL;
+    const uint8_t *chunks_buffer = NULL;
     uint32_t chunks_buffer_len = 0;
 
     HtpRequestBodyReassemble(&htud, &chunks_buffer, &chunks_buffer_len);

src/app-layer-htp.h

@@ -37,6 +37,7 @@
 #include "util-file.h"
 #include "app-layer-htp-mem.h"
 #include "detect-engine-state.h"
+#include "util-streaming-buffer.h"
 
 #include <htp/htp.h>
 
@@ -158,14 +159,14 @@ typedef struct HTPCfgRec_ {
     int                 randomize;
     int                 randomize_range;
     int                 http_body_inline;
+
+    StreamingBufferConfig sbcfg;
 } HTPCfgRec;
 
 /** Struct used to hold chunks of a body on a request */
 struct HtpBodyChunk_ {
-    uint8_t *data;              /**< Pointer to the data of the chunk */
     struct HtpBodyChunk_ *next; /**< Pointer to the next chunk */
-    uint64_t stream_offset;
-    uint32_t len;               /**< Length of the chunk */
+    StreamingBufferSegment sbseg;
     int logged;
 } __attribute__((__packed__));
 typedef struct HtpBodyChunk_ HtpBodyChunk;
@@ -175,6 +176,8 @@ typedef struct HtpBody_ {
     HtpBodyChunk *first; /**< Pointer to the first chunk */
     HtpBodyChunk *last;  /**< Pointer to the last chunk */
 
+    StreamingBuffer *sb;
+
     /* Holds the length of the htp request body seen so far */
     uint64_t content_len_so_far;
     /* parser tracker */

src/detect-engine-hcbd.c

@@ -97,7 +97,7 @@ static inline int HCBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size)
 
 /**
  */
-static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
+static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
@@ -106,7 +106,7 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                uint32_t *stream_start_offset)
 {
     int index = 0;
-    uint8_t *buffer = NULL;
+    const uint8_t *buffer = NULL;
     *buffer_len = 0;
     *stream_start_offset = 0;
 
@@ -170,49 +170,12 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
         goto end;
     }
 
-    int first = 1;
-    while (cur != NULL) {
-        /* see if we can filter out chunks */
-        if (htud->request_body.body_inspected > 0) {
-            if (cur->stream_offset < htud->request_body.body_inspected) {
-                if ((htud->request_body.body_inspected - cur->stream_offset) > htp_state->cfg->request_inspect_min_size) {
-                    cur = cur->next;
-                    continue;
-                } else {
-                    /* include this one */
-                }
-            } else {
-                /* include this one */
-            }
-        }
-
-        if (first) {
-            det_ctx->hcbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* see if we need to grow the buffer */
-        if (det_ctx->hcbd[index].buffer == NULL || (det_ctx->hcbd[index].buffer_len + cur->len) > det_ctx->hcbd[index].buffer_size) {
-            void *ptmp;
-            det_ctx->hcbd[index].buffer_size += cur->len * 2;
-
-            if ((ptmp = SCRealloc(det_ctx->hcbd[index].buffer, det_ctx->hcbd[index].buffer_size)) == NULL) {
-                SCFree(det_ctx->hcbd[index].buffer);
-                det_ctx->hcbd[index].buffer = NULL;
-                det_ctx->hcbd[index].buffer_size = 0;
-                det_ctx->hcbd[index].buffer_len = 0;
-                goto end;
-            }
-            det_ctx->hcbd[index].buffer = ptmp;
-        }
-        memcpy(det_ctx->hcbd[index].buffer + det_ctx->hcbd[index].buffer_len, cur->data, cur->len);
-        det_ctx->hcbd[index].buffer_len += cur->len;
-
-        cur = cur->next;
-    }
+    StreamingBufferGetData(htud->request_body.sb,
+            &det_ctx->hcbd[index].buffer, &det_ctx->hcbd[index].buffer_len,
+            &det_ctx->hcbd[index].offset);
 
     /* update inspected tracker */
-    htud->request_body.body_inspected = htud->request_body.last->stream_offset + htud->request_body.last->len;
+    htud->request_body.body_inspected = htud->request_body.last->sbseg.stream_offset + htud->request_body.last->sbseg.segment_len;
 
     buffer = det_ctx->hcbd[index].buffer;
     *buffer_len = det_ctx->hcbd[index].buffer_len;
@@ -258,7 +221,7 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx,
     uint32_t cnt = 0;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx,
+    const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -267,7 +230,7 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx,
     if (buffer_len == 0)
         goto end;
 
-    cnt = HttpClientBodyPatternSearch(det_ctx, buffer, buffer_len, flags);
+    cnt = HttpClientBodyPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags);
 
  end:
     return cnt;
@@ -282,7 +245,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv,
     HtpState *htp_state = (HtpState *)alstate;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id,
+    const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -296,7 +259,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv,
     det_ctx->inspection_recursion_counter = 0;
     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HCBDMATCH],
                                           f,
-                                          buffer,
+                                          (uint8_t *)buffer,
                                           buffer_len,
                                           stream_start_offset,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HCBD, NULL);

src/detect-engine-hsbd.c

@@ -96,134 +96,7 @@ static inline int HSBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size)
     return 0;
 }
 
-static void HSBDGetBufferForTXInIDSMode(DetectEngineThreadCtx *det_ctx,
-                                        HtpState *htp_state, HtpBodyChunk *cur,
-                                        HtpTxUserData *htud, int index)
-{
-    int first = 1;
-    while (cur != NULL) {
-        /* see if we can filter out chunks */
-        if (htud->response_body.body_inspected > 0) {
-            if (cur->stream_offset < htud->response_body.body_inspected) {
-                if ((htud->response_body.body_inspected - cur->stream_offset) > htp_state->cfg->response_inspect_window) {
-                    cur = cur->next;
-                    continue;
-                } else {
-                    /* include this one */
-                }
-            } else {
-                /* include this one */
-            }
-        }
-
-        if (first) {
-            det_ctx->hsbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* see if we need to grow the buffer */
-        if (det_ctx->hsbd[index].buffer == NULL || (det_ctx->hsbd[index].buffer_len + cur->len) > det_ctx->hsbd[index].buffer_size) {
-            void *ptmp;
-            uint32_t newsize = det_ctx->hsbd[index].buffer_size + (cur->len * 2);
-
-            if ((ptmp = HTPRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size, newsize)) == NULL) {
-                HTPFree(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size);
-                det_ctx->hsbd[index].buffer = NULL;
-                det_ctx->hsbd[index].buffer_size = 0;
-                det_ctx->hsbd[index].buffer_len = 0;
-                return;
-            }
-            det_ctx->hsbd[index].buffer = ptmp;
-            det_ctx->hsbd[index].buffer_size = newsize;
-        }
-        memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data, cur->len);
-        det_ctx->hsbd[index].buffer_len += cur->len;
-
-        cur = cur->next;
-    }
-
-    /* update inspected tracker */
-    htud->response_body.body_inspected = htud->response_body.last->stream_offset + htud->response_body.last->len;
-}
-
-#define MAX_WINDOW 10*1024*1024
-static void HSBDGetBufferForTXInIPSMode(DetectEngineThreadCtx *det_ctx,
-                                        HtpState *htp_state, HtpBodyChunk *cur,
-                                        HtpTxUserData *htud, int index)
-{
-    uint32_t window_size = 0;
-
-    /* how much from before body_inspected will we consider? */
-    uint32_t cfg_win =
-        htud->response_body.body_inspected >= htp_state->cfg->response_inspect_min_size ?
-            htp_state->cfg->response_inspect_window :
-            htp_state->cfg->response_inspect_min_size;
-
-    /* but less if we don't have that much before body_inspected */
-    if ((htud->response_body.body_inspected - htud->response_body.first->stream_offset) < cfg_win) {
-        cfg_win = htud->response_body.body_inspected - htud->response_body.first->stream_offset;
-    }
-    window_size = (htud->response_body.content_len_so_far - htud->response_body.body_inspected) + cfg_win;
-    if (window_size > MAX_WINDOW) {
-        SCLogDebug("weird: body size is %uk", window_size/1024);
-        window_size = MAX_WINDOW;
-    }
-
-    if (det_ctx->hsbd[index].buffer == NULL || window_size > det_ctx->hsbd[index].buffer_size) {
-        void *ptmp;
-
-        if ((ptmp = HTPRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size, window_size)) == NULL) {
-            HTPFree(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size);
-            det_ctx->hsbd[index].buffer = NULL;
-            det_ctx->hsbd[index].buffer_size = 0;
-            det_ctx->hsbd[index].buffer_len = 0;
-            return;
-        }
-        det_ctx->hsbd[index].buffer = ptmp;
-        det_ctx->hsbd[index].buffer_size = window_size;
-    }
-
-    uint32_t left_edge = htud->response_body.body_inspected - cfg_win;
-
-    int first = 1;
-    while (cur != NULL) {
-        if (first) {
-            det_ctx->hsbd[index].offset = cur->stream_offset;
-            first = 0;
-        }
-
-        /* entirely before our window */
-        if ((cur->stream_offset + cur->len) <= left_edge) {
-            cur = cur->next;
-            continue;
-        } else {
-            uint32_t offset = 0;
-            if (cur->stream_offset < left_edge && (cur->stream_offset + cur->len) > left_edge) {
-                offset = left_edge - cur->stream_offset;
-                BUG_ON(offset > cur->len);
-            }
-
-            /* unusual: if window isn't big enough, we just give up */
-            if (det_ctx->hsbd[index].buffer_len + (cur->len - offset) > window_size) {
-                htud->response_body.body_inspected = cur->stream_offset;
-                SCReturn;
-            }
-
-            BUG_ON(det_ctx->hsbd[index].buffer_len + (cur->len - offset) > window_size);
-
-            memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data + offset, cur->len - offset);
-            det_ctx->hsbd[index].buffer_len += (cur->len - offset);
-            det_ctx->hsbd[index].offset -= offset;
-        }
-
-        cur = cur->next;
-    }
-
-    /* update inspected tracker to point before the current window */
-    htud->response_body.body_inspected = htud->response_body.content_len_so_far;
-}
-
-static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
+static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                DetectEngineCtx *de_ctx,
                                                DetectEngineThreadCtx *det_ctx,
                                                Flow *f, HtpState *htp_state,
@@ -232,7 +105,7 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                                                uint32_t *stream_start_offset)
 {
     int index = 0;
-    uint8_t *buffer = NULL;
+    const uint8_t *buffer = NULL;
     *buffer_len = 0;
     *stream_start_offset = 0;
 
@@ -303,14 +176,25 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id,
                        "entire body.");
             goto end;
         }
-        HSBDGetBufferForTXInIDSMode(det_ctx, htp_state, cur, htud, index);
+    }
+
+    StreamingBufferGetData(htud->response_body.sb,
+            &det_ctx->hsbd[index].buffer, &det_ctx->hsbd[index].buffer_len,
+            &det_ctx->hsbd[index].offset);
+
+    /* update inspected tracker */
+    if (!htp_state->cfg->http_body_inline) {
+        htud->response_body.body_inspected = htud->response_body.last->sbseg.stream_offset +
+                                             htud->response_body.last->sbseg.segment_len;
     } else {
-        HSBDGetBufferForTXInIPSMode(det_ctx, htp_state, cur, htud, index);
+        htud->response_body.body_inspected = htud->response_body.content_len_so_far;
     }
+    SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected);
 
     buffer = det_ctx->hsbd[index].buffer;
     *buffer_len = det_ctx->hsbd[index].buffer_len;
     *stream_start_offset = det_ctx->hsbd[index].offset;
+
  end:
     return buffer;
 }
@@ -361,7 +245,7 @@ int DetectEngineRunHttpServerBodyMpm(DetectEngineCtx *de_ctx,
     if (buffer_len == 0)
         goto end;
 
-    cnt = HttpServerBodyPatternSearch(det_ctx, buffer, buffer_len, flags);
+    cnt = HttpServerBodyPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags);
 
  end:
     return cnt;
@@ -377,7 +261,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv,
     HtpState *htp_state = (HtpState *)alstate;
     uint32_t buffer_len = 0;
     uint32_t stream_start_offset = 0;
-    uint8_t *buffer = DetectEngineHSBDGetBufferForTX(tx, tx_id,
+    const uint8_t *buffer = DetectEngineHSBDGetBufferForTX(tx, tx_id,
                                                      de_ctx, det_ctx,
                                                      f, htp_state,
                                                      flags,
@@ -391,7 +275,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv,
     det_ctx->inspection_recursion_counter = 0;
     int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_FILEDATA],
                                           f,
-                                          buffer,
+                                          (uint8_t *)buffer,
                                           buffer_len,
                                           stream_start_offset,
                                           DETECT_ENGINE_CONTENT_INSPECTION_MODE_HSBD, NULL);

src/detect-engine.c

@@ -1687,22 +1687,12 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
     /* HSBD */
     if (det_ctx->hsbd != NULL) {
         SCLogDebug("det_ctx hsbd %u", det_ctx->hsbd_buffers_size);
-        for (i = 0; i < det_ctx->hsbd_buffers_size; i++) {
-            if (det_ctx->hsbd[i].buffer != NULL) {
-                HTPFree(det_ctx->hsbd[i].buffer, det_ctx->hsbd[i].buffer_size);
-            }
-        }
         SCFree(det_ctx->hsbd);
     }
 
     /* HSCB */
     if (det_ctx->hcbd != NULL) {
         SCLogDebug("det_ctx hcbd %u", det_ctx->hcbd_buffers_size);
-        for (i = 0; i < det_ctx->hcbd_buffers_size; i++) {
-            if (det_ctx->hcbd[i].buffer != NULL)
-                SCFree(det_ctx->hcbd[i].buffer);
-            SCLogDebug("det_ctx->hcbd[i].buffer_size %u", det_ctx->hcbd[i].buffer_size);
-        }
         SCFree(det_ctx->hcbd);
     }
 

src/detect-http-client-body.c

@@ -1641,7 +1641,9 @@ static int DetectHttpClientBodyTest15(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg,
+                (uint8_t *)"Body one!!", 10) != 1)
+    {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -1654,7 +1656,9 @@ static int DetectHttpClientBodyTest15(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg,
+                (uint8_t *)"Body two!!", 10) != 1)
+    {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }

src/detect-pcre.c

@@ -3116,7 +3116,7 @@ static int DetectPcreTxBodyChunksTest01(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body one!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3129,7 +3129,7 @@ static int DetectPcreTxBodyChunksTest01(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body two!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3358,7 +3358,7 @@ static int DetectPcreTxBodyChunksTest02(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body one!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }
@@ -3371,7 +3371,7 @@ static int DetectPcreTxBodyChunksTest02(void)
         goto end;
     }
 
-    if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) {
+    if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body two!!", 10) != 1) {
         SCLogDebug("Body data in t1 is not correctly set: ");
         goto end;
     }

src/detect.h

@@ -697,7 +697,7 @@ enum {
 };
 
 typedef struct HttpReassembledBody_ {
-    uint8_t *buffer;
+    const uint8_t *buffer;
     uint32_t buffer_size;   /**< size of the buffer itself */
     uint32_t buffer_len;    /**< data len in the buffer */
     uint64_t offset;        /**< data offset */

src/output-streaming.c

@@ -205,7 +205,7 @@ int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags)
                         }
 
                         uint8_t flags = iflags | OUTPUT_STREAMING_FLAG_TRANSACTION;
-                        if (chunk->stream_offset == 0)
+                        if (chunk->sbseg.stream_offset == 0)
                             flags |= OUTPUT_STREAMING_FLAG_OPEN;
                         /* if we need to close and we're at the last segment in the list
                          * we add the 'close' flag so the logger can close up. */
@@ -213,9 +213,13 @@ int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags)
                             flags |= OUTPUT_STREAMING_FLAG_CLOSE;
                         }
 
+                        const uint8_t *data = NULL;
+                        uint32_t data_len = 0;
+                        StreamingBufferSegmentGetData(body->sb, &chunk->sbseg, &data, &data_len);
+
                         // invoke Streamer
-                        Streamer(cbdata, f, chunk->data, (uint32_t)chunk->len, tx_id, flags);
-                        //PrintRawDataFp(stdout, chunk->data, chunk->len);
+                        Streamer(cbdata, f, data, data_len, tx_id, flags);
+                        //PrintRawDataFp(stdout, data, data_len);
                         chunk->logged = 1;
                         tx_logged = 1;
                     }

src/util-lua-http.c

@@ -285,7 +285,12 @@ static int HttpGetBody(lua_State *luastate, int dir)
     lua_newtable(luastate);
     while (chunk != NULL) {
         lua_pushinteger(luastate, index);
-        LuaPushStringBuffer(luastate, chunk->data, chunk->len);
+
+        const uint8_t *data = NULL;
+        uint32_t data_len = 0;
+        StreamingBufferSegmentGetData(body->sb, &chunk->sbseg, &data, &data_len);
+        LuaPushStringBuffer(luastate, data, data_len);
+
         lua_settable(luastate, -3);
 
         chunk = chunk->next;
@@ -293,8 +298,8 @@ static int HttpGetBody(lua_State *luastate, int dir)
     }
 
     if (body->first && body->last) {
-        lua_pushinteger(luastate, body->first->stream_offset);
-        lua_pushinteger(luastate, body->last->stream_offset + body->last->len);
+        lua_pushinteger(luastate, body->first->sbseg.stream_offset);
+        lua_pushinteger(luastate, body->last->sbseg.stream_offset + body->last->sbseg.segment_len);
         return 3;
     } else {
         return 1;