aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

app-layer/frames: use absolute frame offset

Browse Source 
Frame offset was already a 64 bit integer, so simplify things by
making it an absolute offset from the start of the stream.
pull/8426/head
Victor Julien 4 years ago
parent 39d9b3adbe
commit b43dc5a64a
5 changed files with 95 additions and 176 deletions
Show Stats Download Patch File Download Diff File

112
src/app-layer-frames.c
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2007-2021 Open Information Security Foundation /* Copyright (C) 2007-2022 Open Information Security Foundation
* *
* You can copy, redistribute or modify this Program under the terms of * You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free * the GNU General Public License version 2 as published by the Free
@ -39,11 +39,11 @@ static void FrameDebug(const char *prefix, const Frames *frames, const Frame *fr
} else if (frames != NULL) { } else if (frames != NULL) {
type_name = AppLayerParserGetFrameNameById(frames->ipproto, frames->alproto, frame->type); type_name = AppLayerParserGetFrameNameById(frames->ipproto, frames->alproto, frame->type);
} }
SCLogDebug("[%s] %p: frame: %p type %u/%s id %" PRIi64 " flags %02x rel_offset:%" PRIi64 SCLogDebug("[%s] %p: frame:%p type:%u/%s id:%" PRIi64 " flags:%02x offset:%" PRIu64
", len:%" PRIi64 ", events:%u %u/%u/%u/%u", ", len:%" PRIi64 ", events:%u %u/%u/%u/%u",
prefix, frames, frame, frame->type, type_name, frame->id, frame->flags, prefix, frames, frame, frame->type, type_name, frame->id, frame->flags, frame->offset,
frame->rel_offset, frame->len, frame->event_cnt, frame->events[0], frame->events[1], frame->len, frame->event_cnt, frame->events[0], frame->events[1], frame->events[2],
frame->events[2], frame->events[3]); frame->events[3]);
#endif #endif
} }
@ -84,16 +84,13 @@ Frame *FrameGetByIndex(Frames *frames, const uint32_t idx)
} }
} }
// TODO review rel_offset logic. App-layer passes STREAM_APP_PROGRESS as static Frame *FrameNew(Frames *frames, uint64_t offset, int64_t len)
// offset, but I think we're using rel_offset relative to BASE_PROGRESS
// here which changes only on slide.
static Frame *FrameNew(Frames *frames, int64_t rel_offset, int64_t len)
{ {
BUG_ON(frames == NULL); BUG_ON(frames == NULL);
if (frames->cnt < FRAMES_STATIC_CNT) { if (frames->cnt < FRAMES_STATIC_CNT) {
Frame *frame = &frames->sframes[frames->cnt]; Frame *frame = &frames->sframes[frames->cnt];
frames->sframes[frames->cnt].rel_offset = rel_offset; frames->sframes[frames->cnt].offset = offset;
frames->sframes[frames->cnt].len = len; frames->sframes[frames->cnt].len = len;
frames->sframes[frames->cnt].id = ++frames->base_id; frames->sframes[frames->cnt].id = ++frames->base_id;
frames->cnt++; frames->cnt++;
@ -110,7 +107,7 @@ static Frame *FrameNew(Frames *frames, int64_t rel_offset, int64_t len)
BUG_ON(frames->cnt != FRAMES_STATIC_CNT + 1); BUG_ON(frames->cnt != FRAMES_STATIC_CNT + 1);
frames->dyn_size = 8; frames->dyn_size = 8;
frames->dframes[0].rel_offset = rel_offset; frames->dframes[0].offset = offset;
frames->dframes[0].len = len; frames->dframes[0].len = len;
frames->dframes[0].id = ++frames->base_id; frames->dframes[0].id = ++frames->base_id;
return &frames->dframes[0]; return &frames->dframes[0];
@ -147,7 +144,7 @@ static Frame *FrameNew(Frames *frames, int64_t rel_offset, int64_t len)
} }
frames->cnt++; frames->cnt++;
frames->dframes[dyn_cnt].rel_offset = rel_offset; frames->dframes[dyn_cnt].offset = offset;
frames->dframes[dyn_cnt].len = len; frames->dframes[dyn_cnt].len = len;
frames->dframes[dyn_cnt].id = ++frames->base_id; frames->dframes[dyn_cnt].id = ++frames->base_id;
return &frames->dframes[dyn_cnt]; return &frames->dframes[dyn_cnt];
@ -180,19 +177,15 @@ static void AppLayerFrameDumpForFrames(const char *prefix, const Frames *frames)
SCLogDebug("prefix: %s", prefix); SCLogDebug("prefix: %s", prefix);
} }
static inline uint64_t FrameLeftEdge( static inline uint64_t FrameLeftEdge(const TcpStream *stream, const Frame *frame)
const TcpStream *stream, const Frame *frame, const int64_t base_offset)
{ {
const int64_t app_progress = STREAM_APP_PROGRESS(stream); const int64_t app_progress = STREAM_APP_PROGRESS(stream);
BUG_ON(base_offset > app_progress);
const int64_t frame_offset = base_offset + frame->rel_offset; const int64_t frame_offset = frame->offset;
const int64_t frame_data = app_progress - frame_offset; const int64_t frame_data = app_progress - frame_offset;
SCLogDebug("base_offset %" PRIi64 ", app_progress %" PRIi64, base_offset, app_progress);
SCLogDebug("frame_offset %" PRIi64 ", frame_data %" PRIi64 ", frame->len %" PRIi64, SCLogDebug("frame_offset %" PRIi64 ", frame_data %" PRIi64 ", frame->len %" PRIi64,
frame_offset, frame_data, frame->len); frame_offset, frame_data, frame->len);
BUG_ON(frame_offset < 0);
BUG_ON(frame_offset > app_progress); BUG_ON(frame_offset > app_progress);
/* length unknown, make sure to have at least 2500 */ /* length unknown, make sure to have at least 2500 */
@ -240,19 +233,19 @@ static inline uint64_t FrameLeftEdge(
* *
* [ stream ] * [ stream ]
* [ frame ...........] * [ frame ...........]
* rel_offset: 2 * offset: 2
* len: 19 * len: 19
* *
* Slide: * Slide:
* [ stream ] * [ stream ]
* [ frame .... .] * [ frame .... .]
* rel_offset: -10 * offset: 2
* len: 19 * len: 19
* *
* Slide: * Slide:
* [ stream ] * [ stream ]
* [ frame ........... ] * [ frame ........... ]
* rel_offset: -16 * offset: 2
* len: 19 * len: 19
*/ */
static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, const uint32_t slide) static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, const uint32_t slide)
@ -272,8 +265,7 @@ static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, c
if (i < FRAMES_STATIC_CNT) { if (i < FRAMES_STATIC_CNT) {
Frame *frame = &frames->sframes[i]; Frame *frame = &frames->sframes[i];
FrameDebug("slide(s)", frames, frame); FrameDebug("slide(s)", frames, frame);
if (frame->len >= 0 && if (frame->len >= 0 && frame->offset + frame->len <= next_base) {
frame->rel_offset + frame->len <= (int64_t)slide) { // TODO check seems off
// remove by not incrementing 'x' // remove by not incrementing 'x'
SCLogDebug("removing %p id %" PRIi64, frame, frame->id); SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
FrameClean(frame); FrameClean(frame);
@ -281,18 +273,17 @@ static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, c
} else { } else {
Frame *nframe = &frames->sframes[x]; Frame *nframe = &frames->sframes[x];
FrameCopy(nframe, frame); FrameCopy(nframe, frame);
nframe->rel_offset -= slide; /* turns negative if start if before window */
if (frame != nframe) { if (frame != nframe) {
FrameClean(frame); FrameClean(frame);
} }
le = MIN(le, FrameLeftEdge(stream, nframe, next_base)); le = MIN(le, FrameLeftEdge(stream, nframe));
x++; x++;
} }
} else { } else {
const uint16_t o = i - FRAMES_STATIC_CNT; const uint16_t o = i - FRAMES_STATIC_CNT;
Frame *frame = &frames->dframes[o]; Frame *frame = &frames->dframes[o];
FrameDebug("slide(d)", frames, frame); FrameDebug("slide(d)", frames, frame);
if (frame->len >= 0 && frame->rel_offset + frame->len <= (int64_t)slide) { if (frame->len >= 0 && frame->offset + frame->len <= next_base) {
// remove by not incrementing 'x' // remove by not incrementing 'x'
SCLogDebug("removing %p id %" PRIi64, frame, frame->id); SCLogDebug("removing %p id %" PRIi64, frame, frame->id);
FrameClean(frame); FrameClean(frame);
@ -305,11 +296,10 @@ static int FrameSlide(const char *ds, Frames *frames, const TcpStream *stream, c
nframe = &frames->sframes[x]; nframe = &frames->sframes[x];
} }
FrameCopy(nframe, frame); FrameCopy(nframe, frame);
nframe->rel_offset -= slide; /* turns negative if start is before window */
if (frame != nframe) { if (frame != nframe) {
FrameClean(frame); FrameClean(frame);
} }
le = MIN(le, FrameLeftEdge(stream, nframe, next_base)); le = MIN(le, FrameLeftEdge(stream, nframe));
x++; x++;
} }
} }
@ -390,8 +380,8 @@ void FramesFree(Frames *frames)
Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice, Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
const uint8_t *frame_start, const int64_t len, int dir, uint8_t frame_type) const uint8_t *frame_start, const int64_t len, int dir, uint8_t frame_type)
{ {
SCLogDebug("stream_slice offset %" PRIu64, stream_slice->offset); SCLogDebug("frame_start:%p stream_slice->input:%p stream_slice->offset:%" PRIu64, frame_start,
SCLogDebug("frame_start %p stream_slice->input %p", frame_start, stream_slice->input); stream_slice->input, stream_slice->offset);
/* workarounds for many (unit|fuzz)tests not handling TCP data properly */ /* workarounds for many (unit|fuzz)tests not handling TCP data properly */
#if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) #if defined(UNITTESTS) || defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION)
@ -418,23 +408,19 @@ Frame *AppLayerFrameNewByPointer(Flow *f, const StreamSlice *stream_slice,
if (frames_container == NULL) if (frames_container == NULL)
return NULL; return NULL;
TcpStream *stream;
TcpSession *ssn = f->protoctx;
Frames *frames; Frames *frames;
if (dir == 0) { if (dir == 0) {
frames = &frames_container->toserver; frames = &frames_container->toserver;
stream = &ssn->client;
} else { } else {
frames = &frames_container->toclient; frames = &frames_container->toclient;
stream = &ssn->server;
} }
int64_t abs_frame_offset = stream_slice->offset + (int64_t)ptr_offset; uint64_t abs_frame_offset = stream_slice->offset + ptr_offset;
int64_t rel_offset = abs_frame_offset - STREAM_BASE_OFFSET(stream);
Frame *r = FrameNew(frames, rel_offset, len); Frame *r = FrameNew(frames, abs_frame_offset, len);
if (r != NULL) { if (r != NULL) {
r->type = frame_type; r->type = frame_type;
FrameDebug("new_by_ptr", frames, r);
} }
return r; return r;
} }
@ -487,42 +473,15 @@ Frame *AppLayerFrameNewByRelativeOffset(Flow *f, const StreamSlice *stream_slice
if (frames_container == NULL) if (frames_container == NULL)
return NULL; return NULL;
TcpStream *stream;
TcpSession *ssn = f->protoctx;
Frames *frames; Frames *frames;
if (dir == 0) { if (dir == 0) {
frames = &frames_container->toserver; frames = &frames_container->toserver;
stream = &ssn->client;
} else { } else {
frames = &frames_container->toclient; frames = &frames_container->toclient;
stream = &ssn->server;
} }
const uint64_t base = STREAM_BASE_OFFSET(stream);
#ifdef DEBUG
const uint64_t app = STREAM_APP_PROGRESS(stream);
const uint64_t app_offset = app - base;
const uint64_t slice_offset = stream_slice->offset - base;
SCLogDebug("app %" PRIu64 ", base %" PRIu64 ", slice %" PRIu64, app, base, slice_offset);
SCLogDebug("app_offset %" PRIu64 ", slice_offset %" PRIu64, app_offset, slice_offset);
#endif
const uint64_t frame_abs_offset = (uint64_t)frame_start_rel + stream_slice->offset; const uint64_t frame_abs_offset = (uint64_t)frame_start_rel + stream_slice->offset;
const uint64_t frame_base_offset = frame_abs_offset - base; Frame *r = FrameNew(frames, frame_abs_offset, len);
SCLogDebug("frame_start_rel %u frame_abs_offset %" PRIu64 ", frame_base_offset %" PRIu64,
frame_start_rel, frame_abs_offset, frame_base_offset);
int64_t rel_offset = frame_base_offset;
#ifdef DEBUG
const char *type_name = AppLayerParserGetFrameNameById(f->proto, f->alproto, frame_type);
SCLogDebug("flow %p direction %s frame offset %u rel_offset %" PRIi64 " (abs %" PRIu64
") starting at %" PRIu64 " len %" PRIi64 " (offset %" PRIu64 ") type %u/%s",
f, dir == 0 ? "toserver" : "toclient", frame_start_rel, rel_offset, frame_abs_offset,
frame_abs_offset, len, stream_slice->offset, frame_type, type_name);
#endif
Frame *r = FrameNew(frames, rel_offset, len);
if (r != NULL) { if (r != NULL) {
r->type = frame_type; r->type = frame_type;
} }
@ -563,25 +522,18 @@ Frame *AppLayerFrameNewByAbsoluteOffset(Flow *f, const StreamSlice *stream_slice
if (frames_container == NULL) if (frames_container == NULL)
return NULL; return NULL;
TcpSession *ssn = f->protoctx;
TcpStream *stream;
Frames *frames; Frames *frames;
if (dir == 0) { if (dir == 0) {
stream = &ssn->client;
frames = &frames_container->toserver; frames = &frames_container->toserver;
} else { } else {
stream = &ssn->server;
frames = &frames_container->toclient; frames = &frames_container->toclient;
} }
const uint64_t frame_start_rel = frame_start - STREAM_BASE_OFFSET(stream); SCLogDebug("flow %p direction %s frame type %u offset %" PRIu64 " len %" PRIi64
#ifdef DEBUG " (slice offset %" PRIu64 ")",
SCLogDebug("flow %p direction %s frame type %u offset %" PRIu64 " (abs %" PRIu64 f, dir == 0 ? "toserver" : "toclient", frame_type, frame_start, len,
") starting at %" PRIu64 " len %" PRIi64 " (offset %" PRIu64 ")", stream_slice->offset);
f, dir == 0 ? "toserver" : "toclient", frame_type, frame_start_rel, frame_start, Frame *r = FrameNew(frames, frame_start, len);
frame_start, len, stream_slice->offset);
#endif
Frame *r = FrameNew(frames, (uint32_t)frame_start_rel, len);
if (r != NULL) { if (r != NULL) {
r->type = frame_type; r->type = frame_type;
} }
@ -667,7 +619,7 @@ static inline bool FrameIsDone(
if (frame->len < 0) if (frame->len < 0)
return false; return false;
const int64_t frame_abs_offset = (int64_t)abs_offset + frame->rel_offset; const int64_t frame_abs_offset = frame->offset;
const int64_t frame_right_edge = frame_abs_offset + frame->len; const int64_t frame_right_edge = frame_abs_offset + frame->len;
if ((uint64_t)frame_right_edge <= abs_right_edge) { if ((uint64_t)frame_right_edge <= abs_right_edge) {
SCLogDebug("frame %p id %" PRIi64 " is done", frame, frame->id); SCLogDebug("frame %p id %" PRIi64 " is done", frame, frame->id);
@ -700,7 +652,7 @@ static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof)
FrameClean(frame); FrameClean(frame);
removed++; removed++;
} else { } else {
const uint64_t fle = FrameLeftEdge(stream, frame, STREAM_BASE_OFFSET(stream)); const uint64_t fle = FrameLeftEdge(stream, frame);
le = MIN(le, fle); le = MIN(le, fle);
SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle); SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle);
Frame *nframe = &frames->sframes[x]; Frame *nframe = &frames->sframes[x];
@ -721,7 +673,7 @@ static void FramePrune(Frames *frames, const TcpStream *stream, const bool eof)
FrameClean(frame); FrameClean(frame);
removed++; removed++;
} else { } else {
const uint64_t fle = FrameLeftEdge(stream, frame, STREAM_BASE_OFFSET(stream)); const uint64_t fle = FrameLeftEdge(stream, frame);
le = MIN(le, fle); le = MIN(le, fle);
SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle); SCLogDebug("le %" PRIu64 ", frame fle %" PRIu64, le, fle);
Frame *nframe; Frame *nframe;

3
src/app-layer-frames.h
Unescape Escape View File

@ -47,8 +47,7 @@ typedef struct Frame {
uint8_t event_cnt; uint8_t event_cnt;
// TODO one event per frame enough? // TODO one event per frame enough?
uint8_t events[4]; /**< per frame store for events */ uint8_t events[4]; /**< per frame store for events */
int64_t rel_offset; /**< relative offset in the stream on top of Stream::stream_offset (if uint64_t offset; /**< offset from the start of the stream */
negative the start if before the stream data) */
int64_t len; int64_t len;
int64_t id; int64_t id;
uint64_t tx_id; /**< tx_id to match this frame. UINT64T_MAX if not used. */ uint64_t tx_id; /**< tx_id to match this frame. UINT64T_MAX if not used. */

11
src/app-layer-parser.c
Unescape Escape View File

@ -1293,13 +1293,10 @@ static void HandleStreamFrames(Flow *f, StreamSlice stream_slice, const uint8_t
SCLogDebug("EOF closing: frame %p", frame); SCLogDebug("EOF closing: frame %p", frame);
if (frame) { if (frame) {
/* calculate final frame length */ /* calculate final frame length */
const TcpSession *ssn = f->protoctx; int64_t slice_o = (int64_t)stream_slice.offset - (int64_t)frame->offset;
const TcpStream *stream = (direction == 0) ? &ssn->client : &ssn->server; int64_t frame_len = slice_o + (int64_t)input_len;
int64_t slice_o = SCLogDebug("%s: EOF frame->offset %" PRIu64 " -> %" PRIi64 ": o %" PRIi64,
(int64_t)stream_slice.offset - (int64_t)stream->sb.region.stream_offset; AppProtoToString(f->alproto), frame->offset, frame_len, slice_o);
int64_t frame_len = slice_o + ((int64_t)-1 * frame->rel_offset) + (int64_t)input_len;
SCLogDebug("%s: EOF frame->rel_offset %" PRIi64 " -> %" PRIi64 ": o %" PRIi64,
AppProtoToString(f->alproto), frame->rel_offset, frame_len, slice_o);
frame->len = frame_len; frame->len = frame_len;
} }
} }

111
src/detect-engine-frame.c
Unescape Escape View File

@ -95,7 +95,7 @@ static void PrefilterMpmFrame(DetectEngineThreadCtx *det_ctx, const void *pectx,
const uint8_t *data = buffer->inspect; const uint8_t *data = buffer->inspect;
SCLogDebug("mpm'ing buffer:"); SCLogDebug("mpm'ing buffer:");
// SCLogDebug("frame: %p", frame); SCLogDebug("frame: %p", frame);
// PrintRawDataFp(stdout, data, MIN(32, data_len)); // PrintRawDataFp(stdout, data, MIN(32, data_len));
if (data != NULL && data_len >= mpm_ctx->minlen) { if (data != NULL && data_len >= mpm_ctx->minlen) {
@ -166,26 +166,30 @@ static InspectionBuffer *DetectFrame2InspectBufferUdp(DetectEngineThreadCtx *det
const Frames *frames, const Frame *frame, const int list_id, const uint32_t idx, const Frames *frames, const Frame *frame, const int list_id, const uint32_t idx,
const bool first) const bool first)
{ {
DEBUG_VALIDATE_BUG_ON(frame->rel_offset >= p->payload_len); DEBUG_VALIDATE_BUG_ON(frame->offset >= p->payload_len);
if (frame->rel_offset >= p->payload_len) if (frame->offset >= p->payload_len)
return NULL; return NULL;
int frame_len = frame->len != -1 ? frame->len : p->payload_len - frame->rel_offset;
uint8_t ci_flags = DETECT_CI_FLAGS_START; uint8_t ci_flags = DETECT_CI_FLAGS_START;
uint32_t frame_len;
if (frame->rel_offset + frame_len > p->payload_len) { if (frame->len == -1) {
frame_len = p->payload_len - frame->rel_offset; frame_len = p->payload_len - frame->offset;
} else {
frame_len = (uint32_t)frame->len;
}
if (frame->offset + frame_len > p->payload_len) {
frame_len = p->payload_len - frame->offset;
} else { } else {
ci_flags |= DETECT_CI_FLAGS_END; ci_flags |= DETECT_CI_FLAGS_END;
} }
const uint8_t *data = p->payload + frame->rel_offset; const uint8_t *data = p->payload + frame->offset;
const uint32_t data_len = frame_len; const uint32_t data_len = frame_len;
SCLogDebug("packet %" PRIu64 " -> frame %p/%" PRIi64 "/%s rel_offset %" PRIi64 SCLogDebug("packet %" PRIu64 " -> frame %p/%" PRIi64 "/%s offset %" PRIu64
" type %u len %" PRIi64, " type %u len %" PRIi64,
p->pcap_cnt, frame, frame->id, p->pcap_cnt, frame, frame->id,
AppLayerParserGetFrameNameById(p->flow->proto, p->flow->alproto, frame->type), AppLayerParserGetFrameNameById(p->flow->proto, p->flow->alproto, frame->type),
frame->rel_offset, frame->type, frame->len); frame->offset, frame->type, frame->len);
// PrintRawDataFp(stdout, data, MIN(64,data_len)); // PrintRawDataFp(stdout, data, MIN(64,data_len));
InspectionBufferSetupMulti(buffer, transforms, data, data_len); InspectionBufferSetupMulti(buffer, transforms, data, data_len);
@ -200,8 +204,8 @@ struct FrameStreamData {
const Frame *frame; const Frame *frame;
int list_id; int list_id;
uint32_t idx; uint32_t idx;
uint64_t frame_data_offset_abs; uint64_t data_offset;
uint64_t frame_start_offset_abs; uint64_t frame_offset;
/** buffer is set if callback was successful */ /** buffer is set if callback was successful */
InspectionBuffer *buffer; InspectionBuffer *buffer;
}; };
@ -210,11 +214,11 @@ static int FrameStreamDataFunc(
void *cb_data, const uint8_t *input, const uint32_t input_len, const uint64_t offset) void *cb_data, const uint8_t *input, const uint32_t input_len, const uint64_t offset)
{ {
struct FrameStreamData *fsd = cb_data; struct FrameStreamData *fsd = cb_data;
SCLogDebug("fsd %p { det_ct:%p, transforms:%p, frame:%p, list_id:%d, idx:%u, " SCLogDebug("fsd %p { det_ctx:%p, transforms:%p, frame:%p, list_id:%d, idx:%u, "
"frame_data_offset_abs:%" PRIu64 ", frame_start_offset_abs:%" PRIu64 "data_offset:%" PRIu64 ", frame_offset:%" PRIu64
" }, input: %p, input_len:%u, offset:%" PRIu64, " }, input: %p, input_len:%u, offset:%" PRIu64,
fsd, fsd->det_ctx, fsd->transforms, fsd->frame, fsd->list_id, fsd->idx, fsd, fsd->det_ctx, fsd->transforms, fsd->frame, fsd->list_id, fsd->idx,
fsd->frame_data_offset_abs, fsd->frame_start_offset_abs, input, input_len, offset); fsd->data_offset, fsd->frame_offset, input, input_len, offset);
InspectionBuffer *buffer = InspectionBuffer *buffer =
InspectionBufferMultipleForListGet(fsd->det_ctx, fsd->list_id, fsd->idx); InspectionBufferMultipleForListGet(fsd->det_ctx, fsd->list_id, fsd->idx);
@ -222,11 +226,12 @@ static int FrameStreamDataFunc(
SCLogDebug("buffer %p", buffer); SCLogDebug("buffer %p", buffer);
const Frame *frame = fsd->frame; const Frame *frame = fsd->frame;
SCLogDebug("frame rel_offset:%" PRIi64, frame->rel_offset); SCLogDebug("frame offset:%" PRIu64, frame->offset);
const uint8_t *data = input; const uint8_t *data = input;
uint8_t ci_flags = 0; uint8_t ci_flags = 0;
uint32_t data_len; uint32_t data_len;
if (fsd->frame_start_offset_abs == offset) { uint64_t inspect_offset = 0;
if (fsd->frame_offset == offset) {
ci_flags |= DETECT_CI_FLAGS_START; ci_flags |= DETECT_CI_FLAGS_START;
SCLogDebug("have frame data start"); SCLogDebug("have frame data start");
@ -240,28 +245,24 @@ static int FrameStreamDataFunc(
ci_flags |= DETECT_CI_FLAGS_END; ci_flags |= DETECT_CI_FLAGS_END;
SCLogDebug("have frame data end"); SCLogDebug("have frame data end");
} }
// inspect_offset 0
} else { } else {
BUG_ON(offset < fsd->frame_data_offset_abs); BUG_ON(offset < fsd->data_offset);
inspect_offset = offset - fsd->frame_offset;
uint64_t frame_delta = offset - fsd->frame_start_offset_abs;
uint64_t request_delta =
offset -
fsd->frame_data_offset_abs; // diff between what we requested and what we got
BUG_ON(request_delta > frame_delta);
if (frame->len >= 0) { if (frame->len >= 0) {
if (frame_delta >= (uint64_t)frame->len) { if (inspect_offset >= (uint64_t)frame->len) {
SCLogDebug("data entirely past frame"); SCLogDebug("data entirely past frame");
return 1; return 1;
} }
uint32_t adjusted_frame_len = (uint32_t)((uint64_t)frame->len - frame_delta); uint32_t adjusted_frame_len = (uint32_t)((uint64_t)frame->len - inspect_offset);
SCLogDebug("frame len after applying offset %" PRIu64 ": %u", frame_delta, SCLogDebug("frame len after applying offset %" PRIu64 ": %u", inspect_offset,
adjusted_frame_len); adjusted_frame_len);
data_len = MIN(adjusted_frame_len, input_len); data_len = MIN(adjusted_frame_len, input_len);
SCLogDebug("usable data len for frame: %u", data_len); SCLogDebug("usable data len for frame: %u", data_len);
if ((uint64_t)data_len + frame_delta == (uint64_t)frame->len) { if ((uint64_t)data_len + inspect_offset == (uint64_t)frame->len) {
ci_flags |= DETECT_CI_FLAGS_END; ci_flags |= DETECT_CI_FLAGS_END;
SCLogDebug("have frame data end"); SCLogDebug("have frame data end");
} }
@ -271,7 +272,8 @@ static int FrameStreamDataFunc(
} }
// PrintRawDataFp(stdout, data, data_len); // PrintRawDataFp(stdout, data, data_len);
InspectionBufferSetupMulti(buffer, fsd->transforms, data, data_len); InspectionBufferSetupMulti(buffer, fsd->transforms, data, data_len);
buffer->inspect_offset = frame->rel_offset < 0 ? -1 * frame->rel_offset : 0; // TODO review/test SCLogDebug("inspect_offset %" PRIu64, inspect_offset);
buffer->inspect_offset = inspect_offset;
buffer->flags = ci_flags; buffer->flags = ci_flags;
fsd->buffer = buffer; fsd->buffer = buffer;
return 1; // for now only the first chunk return 1; // for now only the first chunk
@ -304,51 +306,20 @@ InspectionBuffer *DetectFrame2InspectBuffer(DetectEngineThreadCtx *det_ctx,
stream = &ssn->server; stream = &ssn->server;
} }
/* SCLogDebug("frame %" PRIi64 ", len %" PRIi64 ", offset %" PRIu64, frame->id, frame->len,
stream: [s ] frame->offset);
frame: [r ]
progress: |>p
rel_offset: 10, len 100
progress: 20
avail: 90 (complete)
stream: [s ]
frame: [r ]
progress: |>p
stream: 0, len 59
rel_offset: 10, len 100
progress: 20
avail: 30 (incomplete)
stream: [s ]
frame: [r ]
progress: |>p
stream: 0, len 200
rel_offset: -30, len 100
progress: 20
avail: 50 (complete)
*/
SCLogDebug("frame %" PRIi64 ", len %" PRIi64 ", rel_offset %" PRIi64, frame->id, frame->len,
frame->rel_offset);
uint64_t offset = STREAM_BASE_OFFSET(stream);
if (frame->rel_offset > 0) {
offset += (uint64_t)frame->rel_offset;
}
const int64_t frame_start_abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream);
BUG_ON(frame_start_abs_offset < 0);
const uint64_t frame_offset = frame->offset;
const bool eof = ssn->state == TCP_CLOSED || PKT_IS_PSEUDOPKT(p); const bool eof = ssn->state == TCP_CLOSED || PKT_IS_PSEUDOPKT(p);
const uint64_t usable = StreamTcpGetUsable(stream, eof); const uint64_t usable = StreamTcpGetUsable(stream, eof);
if (usable <= offset) if (usable <= frame_offset)
return NULL; return NULL;
const uint64_t offset = MAX(STREAM_BASE_OFFSET(stream), frame_offset);
struct FrameStreamData fsd = { det_ctx, transforms, frame, list_id, idx, offset, struct FrameStreamData fsd = { det_ctx, transforms, frame, list_id, idx,
(uint64_t)frame_start_abs_offset, NULL }; STREAM_BASE_OFFSET(stream), frame_offset, NULL };
StreamReassembleForFrame(ssn, stream, FrameStreamDataFunc, &fsd, offset, eof); StreamReassembleForFrame(ssn, stream, FrameStreamDataFunc, &fsd, offset, eof);
SCLogDebug("offset %" PRIu64, offset); SCLogDebug("offset %" PRIu64, frame_offset);
return fsd.buffer; return fsd.buffer;
} }
@ -394,9 +365,9 @@ int DetectEngineInspectFrameBufferGeneric(DetectEngineThreadCtx *det_ctx,
det_ctx->inspection_recursion_counter = 0; det_ctx->inspection_recursion_counter = 0;
#ifdef DEBUG #ifdef DEBUG
const uint8_t ci_flags = buffer->flags; const uint8_t ci_flags = buffer->flags;
SCLogDebug("frame %p rel_offset %" PRIi64 " type %u len %" PRIi64 SCLogDebug("frame %p offset %" PRIu64 " type %u len %" PRIi64
" ci_flags %02x (start:%s, end:%s)", " ci_flags %02x (start:%s, end:%s)",
frame, frame->rel_offset, frame->type, frame->len, ci_flags, frame, frame->offset, frame->type, frame->len, ci_flags,
(ci_flags & DETECT_CI_FLAGS_START) ? "true" : "false", (ci_flags & DETECT_CI_FLAGS_START) ? "true" : "false",
(ci_flags & DETECT_CI_FLAGS_END) ? "true" : "false"); (ci_flags & DETECT_CI_FLAGS_END) ? "true" : "false");
SCLogDebug("buffer %p offset %" PRIu64 " len %u ci_flags %02x (start:%s, end:%s)", buffer, SCLogDebug("buffer %p offset %" PRIu64 " len %u ci_flags %02x (start:%s, end:%s)", buffer,

34
src/output-json-frame.c
Unescape Escape View File

@ -133,13 +133,13 @@ static void FrameAddPayloadTCP(JsonBuilder *js, const TcpStream *stream, const F
// TODO consider ACK'd // TODO consider ACK'd
if (frame->rel_offset < 0) { if (frame->offset < STREAM_BASE_OFFSET(stream)) {
if (StreamingBufferGetData(&stream->sb, &data, &sb_data_len, &data_offset) == 0) { if (StreamingBufferGetData(&stream->sb, &data, &sb_data_len, &data_offset) == 0) {
SCLogDebug("NO DATA1"); SCLogDebug("NO DATA1");
return; return;
} }
} else { } else {
data_offset = (uint64_t)(frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream)); data_offset = (uint64_t)frame->offset;
SCLogDebug("data_offset %" PRIu64, data_offset); SCLogDebug("data_offset %" PRIu64, data_offset);
if (StreamingBufferGetDataAtOffset( if (StreamingBufferGetDataAtOffset(
&stream->sb, &data, &sb_data_len, (uint64_t)data_offset) == 0) { &stream->sb, &data, &sb_data_len, (uint64_t)data_offset) == 0) {
@ -179,19 +179,23 @@ static void FrameAddPayloadTCP(JsonBuilder *js, const TcpStream *stream, const F
static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *frame) static void FrameAddPayloadUDP(JsonBuilder *js, const Packet *p, const Frame *frame)
{ {
DEBUG_VALIDATE_BUG_ON(frame->rel_offset >= p->payload_len); DEBUG_VALIDATE_BUG_ON(frame->offset >= p->payload_len);
if (frame->rel_offset >= p->payload_len) if (frame->offset >= p->payload_len)
return; return;
int frame_len = frame->len != -1 ? frame->len : p->payload_len - frame->rel_offset; uint32_t frame_len;
if (frame->len == -1) {
if (frame->rel_offset + frame_len > p->payload_len) { frame_len = p->payload_len - frame->offset;
frame_len = p->payload_len - frame->rel_offset; } else {
frame_len = (uint32_t)frame->len;
}
if (frame->offset + frame_len > p->payload_len) {
frame_len = p->payload_len - frame->offset;
JB_SET_FALSE(js, "complete"); JB_SET_FALSE(js, "complete");
} else { } else {
JB_SET_TRUE(js, "complete"); JB_SET_TRUE(js, "complete");
} }
const uint8_t *data = p->payload + frame->rel_offset; const uint8_t *data = p->payload + frame->offset;
const uint32_t data_len = frame_len; const uint32_t data_len = frame_len;
const uint32_t log_data_len = MIN(data_len, 256); const uint32_t log_data_len = MIN(data_len, 256);
@ -227,12 +231,11 @@ void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, const Flow
if (ipproto == IPPROTO_TCP) { if (ipproto == IPPROTO_TCP) {
DEBUG_VALIDATE_BUG_ON(stream == NULL); DEBUG_VALIDATE_BUG_ON(stream == NULL);
int64_t abs_offset = frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream); jb_set_uint(jb, "stream_offset", frame->offset);
jb_set_uint(jb, "stream_offset", (uint64_t)abs_offset);
if (frame->len < 0) { if (frame->len < 0) {
uint64_t usable = StreamTcpGetUsable(stream, true); uint64_t usable = StreamTcpGetUsable(stream, true);
uint64_t len = usable - abs_offset; uint64_t len = usable - frame->offset;
jb_set_uint(jb, "length", len); jb_set_uint(jb, "length", len);
} else { } else {
jb_set_uint(jb, "length", frame->len); jb_set_uint(jb, "length", frame->len);
@ -323,15 +326,12 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p)
for (uint32_t idx = 0; idx < frames->cnt; idx++) { for (uint32_t idx = 0; idx < frames->cnt; idx++) {
Frame *frame = FrameGetByIndex(frames, idx); Frame *frame = FrameGetByIndex(frames, idx);
if (frame != NULL && frame->rel_offset >= 0) { if (frame != NULL) {
if (frame->flags & FRAME_FLAG_LOGGED) if (frame->flags & FRAME_FLAG_LOGGED)
continue; continue;
int64_t abs_offset = (int64_t)frame->rel_offset + (int64_t)STREAM_BASE_OFFSET(stream); int64_t abs_offset = (int64_t)frame->offset + (int64_t)STREAM_BASE_OFFSET(stream);
int64_t win = STREAM_APP_PROGRESS(stream) - abs_offset; int64_t win = STREAM_APP_PROGRESS(stream) - abs_offset;
// SCLogDebug("abs_offset %" PRIi64 ", frame->rel_offset %" PRIi64
// ", frames->progress_rel %d win %" PRIi64,
// abs_offset, frame->rel_offset, frames->progress_rel, win);
if (!eof && win < frame->len && win < 2500) { if (!eof && win < frame->len && win < 2500) {
SCLogDebug("frame id %" PRIi64 " len %" PRIi64 ", win %" PRIi64 SCLogDebug("frame id %" PRIi64 " len %" PRIi64 ", win %" PRIi64

Write Preview
Loading…
Cancel
Save