From c862bbdc4b3a5ce1287ab0bc6d35c94a0a8e5756 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 8 May 2017 13:43:42 -0600 Subject: [PATCH] app-layer: notify parsers of gaps if enabled A parser can now set a flag that will tell the application layer that it is capable of handling gaps. If enabled, and a gap occurs, the app-layer needs to be prepared to accept input that is NULL with a length, where the length is the number of bytes lost. It is up to the app-layer to determine if it can sync up with the input data again. --- src/app-layer-parser.c | 28 +++++++++++++++++++++------- src/app-layer-parser.h | 6 ++++++ src/stream-tcp-reassemble.c | 35 +++++++++++++++++------------------ 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 9fe63e08eb..a7840b5a3f 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -127,6 +127,9 @@ typedef struct AppLayerParserProtoCtx_ * STREAM_TOSERVER, STREAM_TOCLIENT */ uint8_t first_data_dir; + /* Option flags such as supporting gaps or not. */ + uint64_t flags; + #ifdef UNITTESTS void (*RegisterUnittests)(void); #endif @@ -366,6 +369,16 @@ void AppLayerParserRegisterParserAcceptableDataDirection(uint8_t ipproto, AppPro SCReturn; } +void AppLayerParserRegisterOptionFlags(uint8_t ipproto, AppProto alproto, + uint64_t flags) +{ + SCEnter(); + + alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].flags |= flags; + + SCReturn; +} + void AppLayerParserRegisterStateFuncs(uint8_t ipproto, AppProto alproto, void *(*StateAlloc)(void), void (*StateFree)(void *)) @@ -994,14 +1007,15 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow if (p->StateAlloc == NULL) goto end; - /* Do this check before calling AppLayerParse */ if (flags & STREAM_GAP) { - SCLogDebug("stream gap detected (missing packets), " - "this is not yet supported."); - - if (f->alstate != NULL) - AppLayerParserStreamTruncated(f->proto, alproto, f->alstate, flags); - goto error; + if (!(p->flags & APP_LAYER_PARSER_OPT_ACCEPT_GAPS)) { + SCLogDebug("app-layer parser does not accept gaps"); + if (f->alstate != NULL) { + AppLayerParserStreamTruncated(f->proto, alproto, f->alstate, + flags); + } + goto error; + } } /* Get the parser state (if any) */ diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index dc8667b028..ccf8e99c07 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -30,11 +30,15 @@ #include "util-file.h" #include "stream-tcp-private.h" +/* Flags for AppLayerParserState. */ #define APP_LAYER_PARSER_EOF 0x01 #define APP_LAYER_PARSER_NO_INSPECTION 0x02 #define APP_LAYER_PARSER_NO_REASSEMBLY 0x04 #define APP_LAYER_PARSER_NO_INSPECTION_PAYLOAD 0x08 +/* Flags for AppLayerParserProtoCtx. */ +#define APP_LAYER_PARSER_OPT_ACCEPT_GAPS BIT_U64(0) + int AppLayerParserProtoIsRegistered(uint8_t ipproto, AppProto alproto); /***** transaction handling *****/ @@ -115,6 +119,8 @@ int AppLayerParserRegisterParser(uint8_t ipproto, AppProto alproto, void AppLayerParserRegisterParserAcceptableDataDirection(uint8_t ipproto, AppProto alproto, uint8_t direction); +void AppLayerParserRegisterOptionFlags(uint8_t ipproto, AppProto alproto, + uint64_t flags); void AppLayerParserRegisterStateFuncs(uint8_t ipproto, AppProto alproto, void *(*StateAlloc)(void), void (*StateFree)(void *)); diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index debf6642ad..d7eca441ab 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -881,7 +881,7 @@ static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data "got data at %"PRIu64". GAP of size %"PRIu64, offset, blk->offset, blk->offset - offset); *data = NULL; - *data_len = 0; + *data_len = blk->offset - offset; } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) { SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u", @@ -964,33 +964,36 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv, TcpSession *ssn, TcpStream *stream, Packet *p, enum StreamUpdateDir dir) { - const uint64_t app_progress = STREAM_APP_PROGRESS(stream); + uint64_t app_progress = STREAM_APP_PROGRESS(stream); SCLogDebug("app progress %"PRIu64, app_progress); SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq); const uint8_t *mydata; uint32_t mydata_len; - GetAppBuffer(stream, &mydata, &mydata_len, app_progress); - SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len); - if (mydata == NULL || mydata_len == 0) { - if (CheckGap(ssn, stream, p)) { - /* send gap signal */ - SCLogDebug("sending GAP to app-layer"); + while (1) { + GetAppBuffer(stream, &mydata, &mydata_len, app_progress); + if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, stream, p)) { + SCLogNotice("sending GAP to app-layer (size: %u)", mydata_len); + AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream, - NULL, 0, + NULL, mydata_len, StreamGetAppLayerFlags(ssn, stream, p, dir)|STREAM_GAP); AppLayerProfilingStore(ra_ctx->app_tctx, p); - /* set a GAP flag and make sure not bothering this stream anymore */ - SCLogDebug("STREAMTCP_STREAM_FLAG_GAP set"); - stream->flags |= STREAMTCP_STREAM_FLAG_GAP; - StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP); StatsIncr(tv, ra_ctx->counter_tcp_reass_gap); + + stream->app_progress_rel += mydata_len; + app_progress += mydata_len; + continue; + } else if (mydata == NULL || mydata_len == 0) { + /* Possibly a gap, but no new data. */ + return 0; } - return 0; + SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len); + break; } //PrintRawDataFp(stdout, mydata, mydata_len); @@ -1069,10 +1072,6 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, SCLogDebug("stream no reassembly flag set or app-layer disabled."); SCReturnInt(0); } - if (stream->flags & STREAMTCP_STREAM_FLAG_GAP) { - SCReturnInt(0); - } - SCLogDebug("stream->seg_list %p", stream->seg_list); #ifdef DEBUG