diff --git a/src/app-layer-htp-body.c b/src/app-layer-htp-body.c index d3720f8d39..53194307b4 100644 --- a/src/app-layer-htp-body.c +++ b/src/app-layer-htp-body.c @@ -62,6 +62,9 @@ #include "util-memcmp.h" +static StreamingBufferConfig default_cfg = { + 0, 0, 3072, HTPMalloc, HTPCalloc, HTPRealloc, HTPFree }; + /** * \brief Append a chunk of body to the HtpBody struct * @@ -82,20 +85,19 @@ int HtpBodyAppendChunk(HtpBody *body, const uint8_t *data, uint32_t len) SCReturnInt(0); } + if (body->sb == NULL) { + body->sb = StreamingBufferInit(&default_cfg); + if (body->sb == NULL) + SCReturnInt(-1); + } + if (body->first == NULL) { /* New chunk */ bd = (HtpBodyChunk *)HTPCalloc(1, sizeof(HtpBodyChunk)); if (bd == NULL) goto error; - bd->len = len; - bd->stream_offset = 0; - - bd->data = HTPCalloc(1, len); - if (bd->data == NULL) { - goto error; - } - memcpy(bd->data, data, len); + StreamingBufferAppend(body->sb, &bd->sbseg, data, len); body->first = body->last = bd; @@ -105,29 +107,19 @@ int HtpBodyAppendChunk(HtpBody *body, const uint8_t *data, uint32_t len) if (bd == NULL) goto error; - bd->len = len; - bd->stream_offset = body->content_len_so_far; - - bd->data = HTPCalloc(1, len); - if (bd->data == NULL) { - goto error; - } - memcpy(bd->data, data, len); + StreamingBufferAppend(body->sb, &bd->sbseg, data, len); body->last->next = bd; body->last = bd; body->content_len_so_far += len; } - SCLogDebug("Body %p; data %p, len %"PRIu32, body, bd->data, (uint32_t)bd->len); + SCLogDebug("body %p", body); SCReturnInt(0); error: if (bd != NULL) { - if (bd->data != NULL) { - HTPFree(bd->data, bd->len); - } HTPFree(bd, sizeof(HtpBodyChunk)); } SCReturnInt(-1); @@ -150,9 +142,12 @@ void HtpBodyPrint(HtpBody *body) SCLogDebug("--- Start body chunks at %p ---", body); printf("--- Start body chunks at %p ---\n", body); for (cur = body->first; cur != NULL; cur = cur->next) { - SCLogDebug("Body %p; data %p, len %"PRIu32, body, cur->data, (uint32_t)cur->len); - printf("Body %p; data %p, len %"PRIu32"\n", body, cur->data, (uint32_t)cur->len); - PrintRawDataFp(stdout, (uint8_t*)cur->data, cur->len); + const uint8_t *data = NULL; + uint32_t data_len = 0; + StreamingBufferSegmentGetData(body->sb, &cur->sbseg, &data, &data_len); + SCLogDebug("Body %p; data %p, len %"PRIu32, body, data, data_len); + printf("Body %p; data %p, len %"PRIu32"\n", body, data, data_len); + PrintRawDataFp(stdout, data, data_len); } SCLogDebug("--- End body chunks at %p ---", body); } @@ -170,8 +165,7 @@ void HtpBodyFree(HtpBody *body) if (body->first == NULL) return; - SCLogDebug("Removing chunks of Body %p; data %p, len %"PRIu32, body, - body->last->data, (uint32_t)body->last->len); + SCLogDebug("removing chunks of body %p", body); HtpBodyChunk *cur = NULL; HtpBodyChunk *prev = NULL; @@ -179,12 +173,12 @@ void HtpBodyFree(HtpBody *body) prev = body->first; while (prev != NULL) { cur = prev->next; - if (prev->data != NULL) - HTPFree(prev->data, prev->len); HTPFree(prev, sizeof(HtpBodyChunk)); prev = cur; } body->first = body->last = NULL; + + StreamingBufferFree(body->sb); } /** @@ -230,24 +224,26 @@ void HtpBodyPrune(HtpState *state, HtpBody *body, int direction) SCReturn; } - SCLogDebug("Pruning chunks of Body %p; data %p, len %"PRIu32, body, - body->last->data, (uint32_t)body->last->len); + uint64_t left_edge = body->body_inspected; + if (left_edge <= min_size || left_edge <= window) + left_edge = 0; + if (left_edge) + left_edge -= window; + + if (left_edge) { + SCLogDebug("sliding body to offset %"PRIu64, left_edge); + StreamingBufferSlideToOffset(body->sb, left_edge); + } + + SCLogDebug("pruning chunks of body %p", body); HtpBodyChunk *cur = body->first; while (cur != NULL) { HtpBodyChunk *next = cur->next; + SCLogDebug("cur %p", cur); - SCLogDebug("cur->stream_offset %"PRIu64" + cur->len %u = %"PRIu64", " - "body->body_parsed %"PRIu64, cur->stream_offset, cur->len, - cur->stream_offset + cur->len, body->body_parsed); - - uint64_t left_edge = body->body_inspected; - if (left_edge <= min_size || left_edge <= window) - left_edge = 0; - if (left_edge) - left_edge -= window; - - if (cur->stream_offset + cur->len > left_edge) { + if (!StreamingBufferSegmentIsBeforeWindow(body->sb, &cur->sbseg)) { + SCLogDebug("not removed"); break; } @@ -256,12 +252,10 @@ void HtpBodyPrune(HtpState *state, HtpBody *body, int direction) body->last = next; } - if (cur->data != NULL) { - HTPFree(cur->data, cur->len); - } HTPFree(cur, sizeof(HtpBodyChunk)); cur = next; + SCLogDebug("removed"); } SCReturn; diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index 9e68f09708..33b747f13e 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -629,8 +629,11 @@ static int HTPFileParserTest03(void) goto end; } - if (http_state->files_ts->head->chunks_head->len != 11) { - printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len); + if (http_state->files_ts->head->chunks_head == NULL || + http_state->files_ts->head->chunks_head->len != 11) + { + if (http_state->files_ts->head->chunks_head != NULL) + printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len); goto end; } diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 45c14d7d81..f8069c757b 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1170,60 +1170,11 @@ static void HtpRequestBodyMultipartParseHeader(HtpState *hstate, * \param chunks_buffer_len pointer to pass back the buffer length to the caller */ static void HtpRequestBodyReassemble(HtpTxUserData *htud, - uint8_t **chunks_buffer, uint32_t *chunks_buffer_len) + const uint8_t **chunks_buffer, uint32_t *chunks_buffer_len) { - uint8_t *buf = NULL; - uint8_t *pbuf = NULL; - uint32_t buf_len = 0; - HtpBodyChunk *cur = htud->request_body.first; - - for ( ; cur != NULL; cur = cur->next) { - SCLogDebug("chunk %p", cur); - - /* skip body chunks entirely before what we parsed already */ - if ((uint64_t )cur->stream_offset + cur->len <= htud->request_body.body_parsed) { - SCLogDebug("skipping chunk"); - continue; - } - - SCLogDebug("cur->stream_offset %"PRIu64", cur->len %"PRIu32", body_parsed %"PRIu64, - cur->stream_offset, cur->len, htud->request_body.body_parsed); - - if (cur->stream_offset < htud->request_body.body_parsed && - cur->stream_offset + cur->len >= htud->request_body.body_parsed) { - SCLogDebug("use part"); - - uint32_t toff = htud->request_body.body_parsed - cur->stream_offset; - uint32_t tlen = (cur->stream_offset + cur->len) - htud->request_body.body_parsed; - uint8_t *pbuf = NULL; - - buf_len += tlen; - if ((pbuf = HTPRealloc(buf, buf_len - tlen, buf_len)) == NULL) { - HTPFree(buf, buf_len - tlen); - buf = NULL; - buf_len = 0; - break; - } - buf = pbuf; - memcpy(buf + buf_len - tlen, cur->data + toff, tlen); - - } else { - SCLogDebug("use entire chunk"); - - buf_len += cur->len; - if ((pbuf = HTPRealloc(buf, buf_len - cur->len, buf_len)) == NULL) { - HTPFree(buf, buf_len - cur->len); - buf = NULL; - buf_len = 0; - break; - } - buf = pbuf; - memcpy(buf + buf_len - cur->len, cur->data, cur->len); - } - } - - *chunks_buffer = buf; - *chunks_buffer_len = buf_len; + StreamingBufferGetDataAtOffset(htud->request_body.sb, + chunks_buffer, chunks_buffer_len, + htud->request_body.body_parsed); } static void FlagDetectStateNewFile(HtpTxUserData *tx, int dir) @@ -1249,8 +1200,8 @@ static void HtpRequestBodySetupBoundary(HtpTxUserData *htud, memcpy(boundary + 2, htud->boundary, htud->boundary_len); } -int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, - void *tx, uint8_t *chunks_buffer, uint32_t chunks_buffer_len) +int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, void *tx, + const uint8_t *chunks_buffer, uint32_t chunks_buffer_len) { int result = 0; uint8_t boundary[htud->boundary_len + 4]; /**< size limited to HTP_BOUNDARY_MAX + 4 */ @@ -1288,7 +1239,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, if (header_start != NULL || form_end != NULL || (tx_progress > HTP_REQUEST_BODY)) { SCLogDebug("reached the end of the file"); - uint8_t *filedata = chunks_buffer; + const uint8_t *filedata = chunks_buffer; uint32_t filedata_len = 0; uint8_t flags = 0; @@ -1328,7 +1279,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, SCLogDebug("not yet at the end of the file"); if (chunks_buffer_len > expected_boundary_end_len) { - uint8_t *filedata = chunks_buffer; + const uint8_t *filedata = chunks_buffer; uint32_t filedata_len = chunks_buffer_len - expected_boundary_len; #ifdef PRINT printf("FILEDATA (part) START: \n"); @@ -1816,7 +1767,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) HtpBodyAppendChunk(&tx_ud->request_body, d->data, len); - uint8_t *chunks_buffer = NULL; + const uint8_t *chunks_buffer = NULL; uint32_t chunks_buffer_len = 0; if (tx_ud->request_body_type == HTP_BODY_REQUEST_MULTIPART) { @@ -1837,9 +1788,6 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) HtpRequestBodyHandleMultipart(hstate, tx_ud, d->tx, chunks_buffer, chunks_buffer_len); - if (chunks_buffer != NULL) { - HTPFree(chunks_buffer, chunks_buffer_len); - } } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_POST) { HtpRequestBodyHandlePOST(hstate, tx_ud, d->tx, (uint8_t *)d->data, (uint32_t)d->len); } else if (tx_ud->request_body_type == HTP_BODY_REQUEST_PUT) { @@ -2189,6 +2137,10 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec) #endif cfg_prec->randomize_range = HTP_CONFIG_DEFAULT_RANDOMIZE_RANGE; + cfg_prec->sbcfg.flags = 0; + cfg_prec->sbcfg.buf_size = cfg_prec->request_inspect_window; + cfg_prec->sbcfg.buf_slide = 0; + htp_config_register_request_header_data(cfg_prec->cfg, HTPCallbackRequestHeaderData); htp_config_register_request_trailer_data(cfg_prec->cfg, HTPCallbackRequestHeaderData); htp_config_register_response_header_data(cfg_prec->cfg, HTPCallbackResponseHeaderData); @@ -5666,7 +5618,7 @@ static int HTPBodyReassemblyTest01(void) r = HtpBodyAppendChunk(&htud.request_body, chunk2, sizeof(chunk2)-1); BUG_ON(r != 0); - uint8_t *chunks_buffer = NULL; + const uint8_t *chunks_buffer = NULL; uint32_t chunks_buffer_len = 0; HtpRequestBodyReassemble(&htud, &chunks_buffer, &chunks_buffer_len); diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index d7aab93993..3c7ae66cbf 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -37,6 +37,7 @@ #include "util-file.h" #include "app-layer-htp-mem.h" #include "detect-engine-state.h" +#include "util-streaming-buffer.h" #include @@ -158,14 +159,14 @@ typedef struct HTPCfgRec_ { int randomize; int randomize_range; int http_body_inline; + + StreamingBufferConfig sbcfg; } HTPCfgRec; /** Struct used to hold chunks of a body on a request */ struct HtpBodyChunk_ { - uint8_t *data; /**< Pointer to the data of the chunk */ struct HtpBodyChunk_ *next; /**< Pointer to the next chunk */ - uint64_t stream_offset; - uint32_t len; /**< Length of the chunk */ + StreamingBufferSegment sbseg; int logged; } __attribute__((__packed__)); typedef struct HtpBodyChunk_ HtpBodyChunk; @@ -175,6 +176,8 @@ typedef struct HtpBody_ { HtpBodyChunk *first; /**< Pointer to the first chunk */ HtpBodyChunk *last; /**< Pointer to the last chunk */ + StreamingBuffer *sb; + /* Holds the length of the htp request body seen so far */ uint64_t content_len_so_far; /* parser tracker */ diff --git a/src/detect-engine-hcbd.c b/src/detect-engine-hcbd.c index 5fc48c5fc0..1b4b1534a3 100644 --- a/src/detect-engine-hcbd.c +++ b/src/detect-engine-hcbd.c @@ -97,7 +97,7 @@ static inline int HCBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size) /** */ -static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, +static const uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Flow *f, HtpState *htp_state, @@ -106,7 +106,7 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, uint32_t *stream_start_offset) { int index = 0; - uint8_t *buffer = NULL; + const uint8_t *buffer = NULL; *buffer_len = 0; *stream_start_offset = 0; @@ -170,49 +170,12 @@ static uint8_t *DetectEngineHCBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, goto end; } - int first = 1; - while (cur != NULL) { - /* see if we can filter out chunks */ - if (htud->request_body.body_inspected > 0) { - if (cur->stream_offset < htud->request_body.body_inspected) { - if ((htud->request_body.body_inspected - cur->stream_offset) > htp_state->cfg->request_inspect_min_size) { - cur = cur->next; - continue; - } else { - /* include this one */ - } - } else { - /* include this one */ - } - } - - if (first) { - det_ctx->hcbd[index].offset = cur->stream_offset; - first = 0; - } - - /* see if we need to grow the buffer */ - if (det_ctx->hcbd[index].buffer == NULL || (det_ctx->hcbd[index].buffer_len + cur->len) > det_ctx->hcbd[index].buffer_size) { - void *ptmp; - det_ctx->hcbd[index].buffer_size += cur->len * 2; - - if ((ptmp = SCRealloc(det_ctx->hcbd[index].buffer, det_ctx->hcbd[index].buffer_size)) == NULL) { - SCFree(det_ctx->hcbd[index].buffer); - det_ctx->hcbd[index].buffer = NULL; - det_ctx->hcbd[index].buffer_size = 0; - det_ctx->hcbd[index].buffer_len = 0; - goto end; - } - det_ctx->hcbd[index].buffer = ptmp; - } - memcpy(det_ctx->hcbd[index].buffer + det_ctx->hcbd[index].buffer_len, cur->data, cur->len); - det_ctx->hcbd[index].buffer_len += cur->len; - - cur = cur->next; - } + StreamingBufferGetData(htud->request_body.sb, + &det_ctx->hcbd[index].buffer, &det_ctx->hcbd[index].buffer_len, + &det_ctx->hcbd[index].offset); /* update inspected tracker */ - htud->request_body.body_inspected = htud->request_body.last->stream_offset + htud->request_body.last->len; + htud->request_body.body_inspected = htud->request_body.last->sbseg.stream_offset + htud->request_body.last->sbseg.segment_len; buffer = det_ctx->hcbd[index].buffer; *buffer_len = det_ctx->hcbd[index].buffer_len; @@ -258,7 +221,7 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx, uint32_t cnt = 0; uint32_t buffer_len = 0; uint32_t stream_start_offset = 0; - uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx, + const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, idx, de_ctx, det_ctx, f, htp_state, flags, @@ -267,7 +230,7 @@ int DetectEngineRunHttpClientBodyMpm(DetectEngineCtx *de_ctx, if (buffer_len == 0) goto end; - cnt = HttpClientBodyPatternSearch(det_ctx, buffer, buffer_len, flags); + cnt = HttpClientBodyPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags); end: return cnt; @@ -282,7 +245,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv, HtpState *htp_state = (HtpState *)alstate; uint32_t buffer_len = 0; uint32_t stream_start_offset = 0; - uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id, + const uint8_t *buffer = DetectEngineHCBDGetBufferForTX(tx, tx_id, de_ctx, det_ctx, f, htp_state, flags, @@ -296,7 +259,7 @@ int DetectEngineInspectHttpClientBody(ThreadVars *tv, det_ctx->inspection_recursion_counter = 0; int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_HCBDMATCH], f, - buffer, + (uint8_t *)buffer, buffer_len, stream_start_offset, DETECT_ENGINE_CONTENT_INSPECTION_MODE_HCBD, NULL); diff --git a/src/detect-engine-hsbd.c b/src/detect-engine-hsbd.c index 48648e3f4e..3918c6e094 100644 --- a/src/detect-engine-hsbd.c +++ b/src/detect-engine-hsbd.c @@ -96,134 +96,7 @@ static inline int HSBDCreateSpace(DetectEngineThreadCtx *det_ctx, uint64_t size) return 0; } -static void HSBDGetBufferForTXInIDSMode(DetectEngineThreadCtx *det_ctx, - HtpState *htp_state, HtpBodyChunk *cur, - HtpTxUserData *htud, int index) -{ - int first = 1; - while (cur != NULL) { - /* see if we can filter out chunks */ - if (htud->response_body.body_inspected > 0) { - if (cur->stream_offset < htud->response_body.body_inspected) { - if ((htud->response_body.body_inspected - cur->stream_offset) > htp_state->cfg->response_inspect_window) { - cur = cur->next; - continue; - } else { - /* include this one */ - } - } else { - /* include this one */ - } - } - - if (first) { - det_ctx->hsbd[index].offset = cur->stream_offset; - first = 0; - } - - /* see if we need to grow the buffer */ - if (det_ctx->hsbd[index].buffer == NULL || (det_ctx->hsbd[index].buffer_len + cur->len) > det_ctx->hsbd[index].buffer_size) { - void *ptmp; - uint32_t newsize = det_ctx->hsbd[index].buffer_size + (cur->len * 2); - - if ((ptmp = HTPRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size, newsize)) == NULL) { - HTPFree(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size); - det_ctx->hsbd[index].buffer = NULL; - det_ctx->hsbd[index].buffer_size = 0; - det_ctx->hsbd[index].buffer_len = 0; - return; - } - det_ctx->hsbd[index].buffer = ptmp; - det_ctx->hsbd[index].buffer_size = newsize; - } - memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data, cur->len); - det_ctx->hsbd[index].buffer_len += cur->len; - - cur = cur->next; - } - - /* update inspected tracker */ - htud->response_body.body_inspected = htud->response_body.last->stream_offset + htud->response_body.last->len; -} - -#define MAX_WINDOW 10*1024*1024 -static void HSBDGetBufferForTXInIPSMode(DetectEngineThreadCtx *det_ctx, - HtpState *htp_state, HtpBodyChunk *cur, - HtpTxUserData *htud, int index) -{ - uint32_t window_size = 0; - - /* how much from before body_inspected will we consider? */ - uint32_t cfg_win = - htud->response_body.body_inspected >= htp_state->cfg->response_inspect_min_size ? - htp_state->cfg->response_inspect_window : - htp_state->cfg->response_inspect_min_size; - - /* but less if we don't have that much before body_inspected */ - if ((htud->response_body.body_inspected - htud->response_body.first->stream_offset) < cfg_win) { - cfg_win = htud->response_body.body_inspected - htud->response_body.first->stream_offset; - } - window_size = (htud->response_body.content_len_so_far - htud->response_body.body_inspected) + cfg_win; - if (window_size > MAX_WINDOW) { - SCLogDebug("weird: body size is %uk", window_size/1024); - window_size = MAX_WINDOW; - } - - if (det_ctx->hsbd[index].buffer == NULL || window_size > det_ctx->hsbd[index].buffer_size) { - void *ptmp; - - if ((ptmp = HTPRealloc(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size, window_size)) == NULL) { - HTPFree(det_ctx->hsbd[index].buffer, det_ctx->hsbd[index].buffer_size); - det_ctx->hsbd[index].buffer = NULL; - det_ctx->hsbd[index].buffer_size = 0; - det_ctx->hsbd[index].buffer_len = 0; - return; - } - det_ctx->hsbd[index].buffer = ptmp; - det_ctx->hsbd[index].buffer_size = window_size; - } - - uint32_t left_edge = htud->response_body.body_inspected - cfg_win; - - int first = 1; - while (cur != NULL) { - if (first) { - det_ctx->hsbd[index].offset = cur->stream_offset; - first = 0; - } - - /* entirely before our window */ - if ((cur->stream_offset + cur->len) <= left_edge) { - cur = cur->next; - continue; - } else { - uint32_t offset = 0; - if (cur->stream_offset < left_edge && (cur->stream_offset + cur->len) > left_edge) { - offset = left_edge - cur->stream_offset; - BUG_ON(offset > cur->len); - } - - /* unusual: if window isn't big enough, we just give up */ - if (det_ctx->hsbd[index].buffer_len + (cur->len - offset) > window_size) { - htud->response_body.body_inspected = cur->stream_offset; - SCReturn; - } - - BUG_ON(det_ctx->hsbd[index].buffer_len + (cur->len - offset) > window_size); - - memcpy(det_ctx->hsbd[index].buffer + det_ctx->hsbd[index].buffer_len, cur->data + offset, cur->len - offset); - det_ctx->hsbd[index].buffer_len += (cur->len - offset); - det_ctx->hsbd[index].offset -= offset; - } - - cur = cur->next; - } - - /* update inspected tracker to point before the current window */ - htud->response_body.body_inspected = htud->response_body.content_len_so_far; -} - -static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, +static const uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Flow *f, HtpState *htp_state, @@ -232,7 +105,7 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, uint32_t *stream_start_offset) { int index = 0; - uint8_t *buffer = NULL; + const uint8_t *buffer = NULL; *buffer_len = 0; *stream_start_offset = 0; @@ -303,14 +176,25 @@ static uint8_t *DetectEngineHSBDGetBufferForTX(htp_tx_t *tx, uint64_t tx_id, "entire body."); goto end; } - HSBDGetBufferForTXInIDSMode(det_ctx, htp_state, cur, htud, index); + } + + StreamingBufferGetData(htud->response_body.sb, + &det_ctx->hsbd[index].buffer, &det_ctx->hsbd[index].buffer_len, + &det_ctx->hsbd[index].offset); + + /* update inspected tracker */ + if (!htp_state->cfg->http_body_inline) { + htud->response_body.body_inspected = htud->response_body.last->sbseg.stream_offset + + htud->response_body.last->sbseg.segment_len; } else { - HSBDGetBufferForTXInIPSMode(det_ctx, htp_state, cur, htud, index); + htud->response_body.body_inspected = htud->response_body.content_len_so_far; } + SCLogDebug("htud->response_body.body_inspected now: %"PRIu64, htud->response_body.body_inspected); buffer = det_ctx->hsbd[index].buffer; *buffer_len = det_ctx->hsbd[index].buffer_len; *stream_start_offset = det_ctx->hsbd[index].offset; + end: return buffer; } @@ -361,7 +245,7 @@ int DetectEngineRunHttpServerBodyMpm(DetectEngineCtx *de_ctx, if (buffer_len == 0) goto end; - cnt = HttpServerBodyPatternSearch(det_ctx, buffer, buffer_len, flags); + cnt = HttpServerBodyPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags); end: return cnt; @@ -377,7 +261,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv, HtpState *htp_state = (HtpState *)alstate; uint32_t buffer_len = 0; uint32_t stream_start_offset = 0; - uint8_t *buffer = DetectEngineHSBDGetBufferForTX(tx, tx_id, + const uint8_t *buffer = DetectEngineHSBDGetBufferForTX(tx, tx_id, de_ctx, det_ctx, f, htp_state, flags, @@ -391,7 +275,7 @@ int DetectEngineInspectHttpServerBody(ThreadVars *tv, det_ctx->inspection_recursion_counter = 0; int r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_FILEDATA], f, - buffer, + (uint8_t *)buffer, buffer_len, stream_start_offset, DETECT_ENGINE_CONTENT_INSPECTION_MODE_HSBD, NULL); diff --git a/src/detect-engine.c b/src/detect-engine.c index 34f1565202..f591056848 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -1687,22 +1687,12 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) /* HSBD */ if (det_ctx->hsbd != NULL) { SCLogDebug("det_ctx hsbd %u", det_ctx->hsbd_buffers_size); - for (i = 0; i < det_ctx->hsbd_buffers_size; i++) { - if (det_ctx->hsbd[i].buffer != NULL) { - HTPFree(det_ctx->hsbd[i].buffer, det_ctx->hsbd[i].buffer_size); - } - } SCFree(det_ctx->hsbd); } /* HSCB */ if (det_ctx->hcbd != NULL) { SCLogDebug("det_ctx hcbd %u", det_ctx->hcbd_buffers_size); - for (i = 0; i < det_ctx->hcbd_buffers_size; i++) { - if (det_ctx->hcbd[i].buffer != NULL) - SCFree(det_ctx->hcbd[i].buffer); - SCLogDebug("det_ctx->hcbd[i].buffer_size %u", det_ctx->hcbd[i].buffer_size); - } SCFree(det_ctx->hcbd); } diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c index 729629a098..2813d4ca99 100644 --- a/src/detect-http-client-body.c +++ b/src/detect-http-client-body.c @@ -1641,7 +1641,9 @@ static int DetectHttpClientBodyTest15(void) goto end; } - if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) { + if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, + (uint8_t *)"Body one!!", 10) != 1) + { SCLogDebug("Body data in t1 is not correctly set: "); goto end; } @@ -1654,7 +1656,9 @@ static int DetectHttpClientBodyTest15(void) goto end; } - if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) { + if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, + (uint8_t *)"Body two!!", 10) != 1) + { SCLogDebug("Body data in t1 is not correctly set: "); goto end; } diff --git a/src/detect-pcre.c b/src/detect-pcre.c index b4303744f4..72692ee073 100644 --- a/src/detect-pcre.c +++ b/src/detect-pcre.c @@ -3116,7 +3116,7 @@ static int DetectPcreTxBodyChunksTest01(void) goto end; } - if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) { + if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body one!!", 10) != 1) { SCLogDebug("Body data in t1 is not correctly set: "); goto end; } @@ -3129,7 +3129,7 @@ static int DetectPcreTxBodyChunksTest01(void) goto end; } - if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) { + if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body two!!", 10) != 1) { SCLogDebug("Body data in t1 is not correctly set: "); goto end; } @@ -3358,7 +3358,7 @@ static int DetectPcreTxBodyChunksTest02(void) goto end; } - if (memcmp(cur->data, "Body one!!", strlen("Body one!!")) != 0) { + if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body one!!", 10) != 1) { SCLogDebug("Body data in t1 is not correctly set: "); goto end; } @@ -3371,7 +3371,7 @@ static int DetectPcreTxBodyChunksTest02(void) goto end; } - if (memcmp(cur->data, "Body two!!", strlen("Body two!!")) != 0) { + if (StreamingBufferSegmentCompareRawData(htud->request_body.sb, &cur->sbseg, (uint8_t *)"Body two!!", 10) != 1) { SCLogDebug("Body data in t1 is not correctly set: "); goto end; } diff --git a/src/detect.h b/src/detect.h index 90c6502984..8a57b08161 100644 --- a/src/detect.h +++ b/src/detect.h @@ -697,7 +697,7 @@ enum { }; typedef struct HttpReassembledBody_ { - uint8_t *buffer; + const uint8_t *buffer; uint32_t buffer_size; /**< size of the buffer itself */ uint32_t buffer_len; /**< data len in the buffer */ uint64_t offset; /**< data offset */ diff --git a/src/output-streaming.c b/src/output-streaming.c index 4ae7c58c2e..234fbee70e 100644 --- a/src/output-streaming.c +++ b/src/output-streaming.c @@ -205,7 +205,7 @@ int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags) } uint8_t flags = iflags | OUTPUT_STREAMING_FLAG_TRANSACTION; - if (chunk->stream_offset == 0) + if (chunk->sbseg.stream_offset == 0) flags |= OUTPUT_STREAMING_FLAG_OPEN; /* if we need to close and we're at the last segment in the list * we add the 'close' flag so the logger can close up. */ @@ -213,9 +213,13 @@ int HttpBodyIterator(Flow *f, int close, void *cbdata, uint8_t iflags) flags |= OUTPUT_STREAMING_FLAG_CLOSE; } + const uint8_t *data = NULL; + uint32_t data_len = 0; + StreamingBufferSegmentGetData(body->sb, &chunk->sbseg, &data, &data_len); + // invoke Streamer - Streamer(cbdata, f, chunk->data, (uint32_t)chunk->len, tx_id, flags); - //PrintRawDataFp(stdout, chunk->data, chunk->len); + Streamer(cbdata, f, data, data_len, tx_id, flags); + //PrintRawDataFp(stdout, data, data_len); chunk->logged = 1; tx_logged = 1; } diff --git a/src/util-lua-http.c b/src/util-lua-http.c index 3d97b0f640..1207f7f9ef 100644 --- a/src/util-lua-http.c +++ b/src/util-lua-http.c @@ -285,7 +285,12 @@ static int HttpGetBody(lua_State *luastate, int dir) lua_newtable(luastate); while (chunk != NULL) { lua_pushinteger(luastate, index); - LuaPushStringBuffer(luastate, chunk->data, chunk->len); + + const uint8_t *data = NULL; + uint32_t data_len = 0; + StreamingBufferSegmentGetData(body->sb, &chunk->sbseg, &data, &data_len); + LuaPushStringBuffer(luastate, data, data_len); + lua_settable(luastate, -3); chunk = chunk->next; @@ -293,8 +298,8 @@ static int HttpGetBody(lua_State *luastate, int dir) } if (body->first && body->last) { - lua_pushinteger(luastate, body->first->stream_offset); - lua_pushinteger(luastate, body->last->stream_offset + body->last->len); + lua_pushinteger(luastate, body->first->sbseg.stream_offset); + lua_pushinteger(luastate, body->last->sbseg.stream_offset + body->last->sbseg.segment_len); return 3; } else { return 1;