diff --git a/src/app-layer-frames.c b/src/app-layer-frames.c index 71f6247e5c..97ef816e79 100644 --- a/src/app-layer-frames.c +++ b/src/app-layer-frames.c @@ -224,23 +224,6 @@ static inline uint64_t FrameLeftEdge( } } } -#if 0 -static inline uint64_t FramesLeftEdge(const TcpStream *stream, const Frames *frames) -{ - uint64_t le = STREAM_APP_PROGRESS(stream); - for (uint16_t i = 0; i < frames->cnt; i++) { - if (i < FRAMES_STATIC_CNT) { - const Frame *frame = &frames->sframes[i]; - le = MIN(le, FrameLeftEdge(stream, frame)); - } else { - const uint16_t o = i - FRAMES_STATIC_CNT; - const Frame *frame = &frames->dframes[o]; - le = MIN(le, FrameLeftEdge(stream, frame)); - } - } - return le; -} -#endif /** Stream buffer slides forward, we need to update and age out * frame offsets/frames. Aging out means we move existing frames @@ -274,12 +257,6 @@ static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, c BUG_ON(frames == NULL); SCLogDebug("%s frames %p: sliding %u bytes", ds, frames, slide); uint64_t le = STREAM_APP_PROGRESS(stream); - - if (slide >= frames->progress_rel) - frames->progress_rel = 0; - else - frames->progress_rel -= slide; - const uint64_t next_base = STREAM_BASE_OFFSET(stream) + slide; const uint16_t start = frames->cnt; uint16_t removed = 0; @@ -348,24 +325,6 @@ static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, c return 0; } -void AppLayerFramesUpdateProgress( - Flow *f, TcpStream *stream, const uint64_t progress, const uint8_t direction) -{ - FramesContainer *frames_container = AppLayerFramesGetContainer(f); - if (frames_container == NULL) - return; - - Frames *frames; - if (direction == STREAM_TOSERVER) { - frames = &frames_container->toserver; - } else { - frames = &frames_container->toclient; - } - - const uint32_t slide = progress - STREAM_APP_PROGRESS(stream); - frames->progress_rel += slide; -} - void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction) { FramesContainer *frames_container = AppLayerFramesGetContainer(f); @@ -711,7 +670,7 @@ static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof) SCLogDebug("start: left edge %" PRIu64 ", left_edge_rel %u, stream base %" PRIu64, (uint64_t)frames->left_edge_rel + STREAM_BASE_OFFSET(stream), frames->left_edge_rel, STREAM_BASE_OFFSET(stream)); - const uint64_t abs_offset = STREAM_BASE_OFFSET(stream) + (uint64_t)frames->progress_rel; + const uint64_t abs_offset = STREAM_BASE_OFFSET(stream); const uint64_t acked = StreamTcpGetUsable(stream, eof); uint64_t le = STREAM_APP_PROGRESS(stream); diff --git a/src/app-layer-frames.h b/src/app-layer-frames.h index be2900224e..746c729692 100644 --- a/src/app-layer-frames.h +++ b/src/app-layer-frames.h @@ -61,9 +61,8 @@ typedef struct Frame { typedef struct Frames { uint16_t cnt; uint16_t dyn_size; /**< size in elements of `dframes` */ - uint32_t progress_rel; /**< processing depth relative to STREAM_BASE_OFFSET */ - uint64_t base_id; uint32_t left_edge_rel; + uint64_t base_id; Frame sframes[FRAMES_STATIC_CNT]; /**< static frames */ Frame *dframes; /**< dynamically allocated space for more frames */ #ifdef DEBUG @@ -102,8 +101,6 @@ void AppLayerFrameSetLengthById(Flow *f, const int dir, const FrameId id, int64_ void AppLayerFrameSetTxId(Frame *r, uint64_t tx_id); void AppLayerFrameSetTxIdById(Flow *f, const int dir, const FrameId id, uint64_t tx_id); -void AppLayerFramesUpdateProgress( - Flow *f, TcpStream *stream, const uint64_t progress, const uint8_t direction); void AppLayerFramesSlide(Flow *f, const uint32_t slide, const uint8_t direction); FramesContainer *AppLayerFramesGetContainer(Flow *f); diff --git a/src/detect-engine-frame.c b/src/detect-engine-frame.c index 6aeda8048f..19624fd0d5 100644 --- a/src/detect-engine-frame.c +++ b/src/detect-engine-frame.c @@ -89,6 +89,7 @@ static void PrefilterMpmFrame(DetectEngineThreadCtx *det_ctx, const void *pectx, det_ctx, ctx->transforms, p, frames, frame, ctx->list_id, idx, true); if (buffer == NULL) return; + BUG_ON(buffer->orig_len > frame->len); const uint32_t data_len = buffer->inspect_len; const uint8_t *data = buffer->inspect; @@ -192,6 +193,86 @@ static InspectionBuffer *DetectFrame2InspectBufferUdp(DetectEngineThreadCtx *det return buffer; } +struct FrameStreamData { + DetectEngineThreadCtx *det_ctx; + const DetectEngineTransforms *transforms; + const Frame *frame; + int list_id; + uint32_t idx; + uint64_t frame_data_offset_abs; + uint64_t frame_start_offset_abs; +}; + +static int FrameStreamDataFunc( + void *cb_data, const uint8_t *input, const uint32_t input_len, const uint64_t offset) +{ + struct FrameStreamData *fsd = cb_data; + SCLogDebug("fsd %p { det_ct:%p, transforms:%p, frame:%p, list_id:%d, idx:%u, " + "frame_data_offset_abs:%" PRIu64 ", frame_start_offset_abs:%" PRIu64 + " }, input: %p, input_len:%u, offset:%" PRIu64, + fsd, fsd->det_ctx, fsd->transforms, fsd->frame, fsd->list_id, fsd->idx, + fsd->frame_data_offset_abs, fsd->frame_start_offset_abs, input, input_len, offset); + + InspectionBuffer *buffer = + InspectionBufferMultipleForListGet(fsd->det_ctx, fsd->list_id, fsd->idx); + BUG_ON(buffer == NULL); + SCLogDebug("buffer %p", buffer); + + const Frame *frame = fsd->frame; + SCLogDebug("frame rel_offset:%" PRIi64, frame->rel_offset); + const uint8_t *data = input; + uint8_t ci_flags = 0; + uint32_t data_len; + if (fsd->frame_start_offset_abs == offset) { + ci_flags |= DETECT_CI_FLAGS_START; + SCLogDebug("have frame data start"); + + if (frame->len >= 0) { + data_len = MIN(input_len, frame->len); + } else { + data_len = input_len; + } + + if (data_len == frame->len) { + ci_flags |= DETECT_CI_FLAGS_END; + SCLogDebug("have frame data end"); + } + } else { + BUG_ON(offset < fsd->frame_data_offset_abs); + + uint64_t frame_delta = offset - fsd->frame_start_offset_abs; + uint64_t request_delta = + offset - + fsd->frame_data_offset_abs; // diff between what we requested and what we got + BUG_ON(request_delta > frame_delta); + + if (frame->len >= 0) { + if (frame_delta >= (uint64_t)frame->len) { + SCLogDebug("data entirely past frame"); + return 1; + } + uint32_t adjusted_frame_len = (uint32_t)((uint64_t)frame->len - frame_delta); + SCLogDebug("frame len after applying offset %" PRIu64 ": %u", frame_delta, + adjusted_frame_len); + + data_len = MIN(adjusted_frame_len, input_len); + SCLogDebug("usable data len for frame: %u", data_len); + + if ((uint64_t)data_len + frame_delta == (uint64_t)frame->len) { + ci_flags |= DETECT_CI_FLAGS_END; + SCLogDebug("have frame data end"); + } + } else { + data_len = input_len; + } + } + // PrintRawDataFp(stdout, data, data_len); + InspectionBufferSetupMulti(buffer, fsd->transforms, data, data_len); + buffer->inspect_offset = frame->rel_offset < 0 ? -1 * frame->rel_offset : 0; // TODO review/test + buffer->flags = ci_flags; + return 1; // for now only the first chunk +} + InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx, const DetectEngineTransforms *transforms, Packet *p, const Frames *frames, const Frame *frame, const int list_id, const uint32_t idx, const bool first) @@ -244,21 +325,15 @@ InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx, avail: 50 (complete) */ - SCLogDebug("frame %" PRIi64 ", len %" PRIi64, frame->id, frame->len); - - uint32_t data_len = 0; - const uint8_t *data = NULL; + SCLogDebug("frame %" PRIi64 ", len %" PRIi64 ", rel_offset %" PRIi64, frame->id, frame->len, + frame->rel_offset); uint64_t offset = STREAM_BASE_OFFSET(stream); - if (frame->rel_offset > 0 || frames->progress_rel) { - uint64_t frame_offset = 0; - if (frame->rel_offset >= 0) { - frame_offset = MAX((uint64_t)frame->rel_offset, (uint64_t)frames->progress_rel); - } else { - frame_offset = (uint64_t)frames->progress_rel; - } - offset += frame_offset; + if (frame->rel_offset > 0) { + offset += (uint64_t)frame->rel_offset; } + const int64_t frame_start_abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream); + BUG_ON(frame_start_abs_offset < 0); const bool eof = ssn->state == TCP_CLOSED || PKT_IS_PSEUDOPKT(p); @@ -266,68 +341,14 @@ InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx, if (usable <= offset) return NULL; - // TODO GAP handling - if (StreamingBufferGetDataAtOffset(&stream->sb, &data, &data_len, offset) == 0) { - return NULL; - } + struct FrameStreamData fsd = { det_ctx, transforms, frame, list_id, idx, offset, + (uint64_t)frame_start_abs_offset }; + StreamReassembleForFrame(ssn, stream, FrameStreamDataFunc, &fsd, offset, eof); + SCLogDebug("offset %" PRIu64, offset); - const uint64_t data_right_edge = offset + data_len; - if (data_right_edge > usable) - data_len = usable - offset; - - const int64_t frame_start_abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream); - const uint64_t usable_right_edge = MIN(data_right_edge, usable); - - bool have_end = false; - - if (frame->len > 0) { - const int64_t frame_avail_data_abs = (int64_t)usable_right_edge; - const int64_t frame_end_abs_offset = frame_start_abs_offset + frame->len; - have_end = (int64_t)usable_right_edge >= frame_end_abs_offset; - - SCLogDebug("frame_end_abs_offset %" PRIi64 ", usable_right_edge %" PRIu64, - frame_end_abs_offset, usable_right_edge); - - const int64_t avail_from_frame = MIN(frame_end_abs_offset, frame_avail_data_abs) - offset; - if (avail_from_frame < (int64_t)data_len) { - SCLogDebug("adjusted data len from %u to %" PRIi64, data_len, avail_from_frame); - data_len = (uint32_t)avail_from_frame; - } - } - const bool have_start = frame_start_abs_offset == (int64_t)offset; - - if (data == NULL || data_len == 0) { - return NULL; - } - - // TODO use eof too? - SCLogDebug("stream->min_inspect_depth %u", stream->min_inspect_depth); - if (data_len < frame->len && data_len < stream->min_inspect_depth) { - if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED || ssn->state == TCP_CLOSED || - stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) { - SCLogDebug("EOF use available data: %u bytes", data_len); - } else { - SCLogDebug("not enough data to inspect now: have %u, want %u", data_len, - stream->min_inspect_depth); - return NULL; - } - } - - const uint8_t ci_flags = - (have_start ? DETECT_CI_FLAGS_START : 0) | (have_end ? DETECT_CI_FLAGS_END : 0); - SCLogDebug("packet %" PRIu64 " -> frame %p/%" PRIi64 "/%s rel_offset %" PRIi64 - " type %u len %" PRIi64 " ci_flags %02x (start:%s, end:%s)", - p->pcap_cnt, frame, frame->id, - AppLayerParserGetFrameNameById(p->flow->proto, p->flow->alproto, frame->type), - frame->rel_offset, frame->type, frame->len, ci_flags, - (ci_flags & DETECT_CI_FLAGS_START) ? "true" : "false", - (ci_flags & DETECT_CI_FLAGS_END) ? "true" : "false"); - // PrintRawDataFp(stdout, data, MIN(32,data_len)); - - InspectionBufferSetupMulti(buffer, transforms, data, data_len); - buffer->inspect_offset = frame->rel_offset < 0 ? -1 * frame->rel_offset : 0; // TODO review/test - buffer->flags = ci_flags; - return buffer; + InspectionBuffer *ret = InspectionBufferMultipleForListGet(det_ctx, list_id, idx); + SCLogDebug("ret %p", ret); + return ret; } /** diff --git a/src/output-json-frame.c b/src/output-json-frame.c index e0caf1de03..00e438f00d 100644 --- a/src/output-json-frame.c +++ b/src/output-json-frame.c @@ -330,9 +330,9 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p) int64_t abs_offset = (int64_t)frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream); int64_t win = STREAM_APP_PROGRESS(stream) - abs_offset; - SCLogDebug("abs_offset %" PRIi64 ", frame->rel_offset %" PRIi64 - ", frames->progress_rel %d win %" PRIi64, - abs_offset, frame->rel_offset, frames->progress_rel, win); + // SCLogDebug("abs_offset %" PRIi64 ", frame->rel_offset %" PRIi64 + // ", frames->progress_rel %d win %" PRIi64, + // abs_offset, frame->rel_offset, frames->progress_rel, win); if (!eof && win < frame->len && win < 2500) { SCLogDebug("frame id %" PRIi64 " len %" PRIi64 ", win %" PRIi64 diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index 31cfaf49f7..5956e1e5d5 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -1389,14 +1389,6 @@ void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t prog stream = &ssn->server; } - /* Record updates */ - if (!(ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED)) { - if (progress > STREAM_APP_PROGRESS(stream)) { - AppLayerFramesUpdateProgress(p->flow, stream, progress, - PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT); - } - } - if (progress > STREAM_RAW_PROGRESS(stream)) { uint32_t slide = progress - STREAM_RAW_PROGRESS(stream); stream->raw_progress_rel += slide; @@ -1745,8 +1737,13 @@ end: int StreamReassembleForFrame(TcpSession *ssn, TcpStream *stream, StreamReassembleRawFunc Callback, void *cb_data, const uint64_t offset, const bool eof) { + /* take app progress as the right edge of used data. */ + const uint64_t app_progress = STREAM_APP_PROGRESS(stream); + SCLogDebug("app_progress %" PRIu64, app_progress); + uint64_t unused = 0; - return StreamReassembleRawDo(ssn, stream, Callback, cb_data, offset, &unused, eof, false); + return StreamReassembleRawDo( + ssn, stream, Callback, cb_data, offset, app_progress, &unused, eof, false); } int StreamReassembleRaw(TcpSession *ssn, const Packet *p,