From 7074ca373bb14acdecf2b7798650786c5f49e006 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 5 Nov 2013 14:41:45 +0100 Subject: [PATCH] proto detection: add limit for one sided sessions If a session only has data in one direction, like ftp data sessions, protocol detection will only run in one direction. This led to a situation where reassembly would hold all the segments as proto detection was never flagged as complete. This patch introduces a limit for protocol detection in this case. If the limit is reached, detection will give up. --- src/app-layer-detect-proto.c | 25 ++++++++++++++++++ src/app-layer-detect-proto.h | 6 +++-- src/app-layer.c | 50 ++++++++++++++++++++++++++++++++++++ src/stream-tcp.c | 13 ++++++++++ src/stream-tcp.h | 1 + 5 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c index 19aefd043d..386f544abd 100644 --- a/src/app-layer-detect-proto.c +++ b/src/app-layer-detect-proto.c @@ -62,7 +62,10 @@ #include "util-cuda.h" #include "util-debug.h" +#include "conf.h" + #define INSPECT_BYTES 32 +#define ASYNC_MAX 75000 /** global app layer detection context */ AlpProtoDetectCtx alp_proto_ctx; @@ -82,6 +85,28 @@ void AlpProtoInit(AlpProtoDetectCtx *ctx) { ctx->toclient.min_len = INSPECT_BYTES; ctx->toserver.min_len = INSPECT_BYTES; + intmax_t value = 0; + if ((ConfGetInt("app-layer.proto-detect.toclient-async-max", &value)) == 1) { + if (value >= 0 && value <= 1048576) { + ctx->toclient.async_max = (uint32_t)value; + } else { + ctx->toclient.async_max = (uint32_t)ASYNC_MAX; + } + } else { + ctx->toclient.async_max = (uint32_t)ASYNC_MAX; + } + if ((ConfGetInt("app-layer.proto-detect.toserver-async-max", &value)) == 1) { + if (value >= 0 && value <= 1048576) { + ctx->toserver.async_max = (uint32_t)value; + } else { + ctx->toserver.async_max = (uint32_t)ASYNC_MAX; + } + } else { + ctx->toserver.async_max = (uint32_t)ASYNC_MAX; + } + SCLogDebug("toclient.async_max %u toserver.async_max %u", + ctx->toclient.async_max, ctx->toserver.async_max); + ctx->mpm_pattern_id_store = MpmPatternIdTableInitHash(); } diff --git a/src/app-layer-detect-proto.h b/src/app-layer-detect-proto.h index f069f9fa29..54e63bda26 100644 --- a/src/app-layer-detect-proto.h +++ b/src/app-layer-detect-proto.h @@ -46,12 +46,14 @@ typedef struct AlpProtoDetectDirection_ { uint32_t id; uint16_t map[ALP_DETECT_MAX]; /**< a mapping between condition id's and protocol */ - uint16_t max_len; /**< max length of all patterns, so we can + uint16_t max_len; /**< max length of all patterns, so we can limit the search */ - uint16_t min_len; /**< min length of all patterns, so we can + uint16_t min_len; /**< min length of all patterns, so we can tell the stream engine to feed data to app layer as soon as it has min size data */ + uint32_t async_max; /**< max bytes in this direction while 0 in + the other, before we give up. */ } AlpProtoDetectDirection; typedef struct AlpProtoDetectCtx_ { diff --git a/src/app-layer.c b/src/app-layer.c index 194c03520f..0ce187d925 100644 --- a/src/app-layer.c +++ b/src/app-layer.c @@ -156,6 +156,9 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, alproto_otherdir = &f->alproto_ts; dir = 1; } + SCLogDebug("dir %u alproto %u alproto_other_dir %u", + dir, *alproto, *alproto_otherdir); + //PrintRawDataFp(stdout, data, data_len); /* if we don't know the proto yet and we have received a stream * initializer message, we run proto detection. @@ -187,6 +190,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, *alproto = AppLayerDetectGetProto(&alp_proto_ctx, dp_ctx, f, data, data_len, flags, IPPROTO_TCP); PACKET_PROFILING_APP_PD_END(dp_ctx); + SCLogDebug("alproto %u", *alproto); if (*alproto != ALPROTO_UNKNOWN) { if (*alproto_otherdir != ALPROTO_UNKNOWN && *alproto_otherdir != *alproto) { @@ -204,6 +208,9 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, } f->alproto = *alproto; + SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted " + "on stream %p (%s)", stream, (stream == &ssn->client) ? + "ssn->client" : "ssn->server"); StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream); /* if we have seen data from the other direction first, send @@ -215,6 +222,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, * will now call shortly for the opposing direction. */ if ((ssn->data_first_seen_dir & (STREAM_TOSERVER | STREAM_TOCLIENT)) && !(flags & ssn->data_first_seen_dir)) { + SCLogDebug("entering opposing dir hack"); TcpStream *opposing_stream = NULL; if (stream == &ssn->client) { opposing_stream = &ssn->server; @@ -260,8 +268,11 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, p->flowflags |= FLOW_PKT_TOSERVER; } } + SCLogDebug("ret %d", ret); if (ret < 0) { FlowSetSessionNoApplayerInspectionFlag(f); + SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted " + "on both streams"); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server); r = -1; @@ -292,6 +303,8 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, AppLayerDecoderEventsSetEventRaw(p->app_layer_events, APPLAYER_WRONG_DIRECTION_FIRST_DATA); FlowSetSessionNoApplayerInspectionFlag(f); + SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted " + "on both streams"); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client); /* Set a value that is neither STREAM_TOSERVER, nor STREAM_TOCLIENT */ @@ -328,6 +341,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, PACKET_PROFILING_APP_END(dp_ctx, *alproto); f->data_al_so_far[dir] = 0; } else { + SCLogDebug("alproto == ALPROTO_UNKNOWN (%u)", *alproto); if (*alproto_otherdir != ALPROTO_UNKNOWN) { /* this would handle this test case - * http parser which says it wants to see toserver data first only. @@ -350,6 +364,8 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, { r = -1; FlowSetSessionNoApplayerInspectionFlag(f); + SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted " + "on both streams"); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client); goto end; @@ -365,14 +381,48 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, if (FLOW_IS_PM_DONE(f, flags) && FLOW_IS_PP_DONE(f, flags)) { AppLayerDecoderEventsSetEventRaw(p->app_layer_events, APPLAYER_DETECT_PROTOCOL_ONLY_ONE_DIRECTION); + SCLogDebug("calling StreamTcpSetStreamFlagAppProtoDetectionCompleted " + "on stream %p (%s)", stream, (stream == &ssn->client) ? + "ssn->client" : "ssn->server"); StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream); f->data_al_so_far[dir] = 0; } else { f->data_al_so_far[dir] = data_len; + SCLogDebug("data_len %u stored in flow for dir %u", data_len, dir); } } else { + + SCLogDebug("both unknown FLOW_IS_PM_DONE(f, STREAM_TOSERVER) %s " + "FLOW_IS_PP_DONE(f, STREAM_TOSERVER) %s " + "FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) %s " + "FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) %s," + " stream ts %u stream tc %u", + FLOW_IS_PM_DONE(f, STREAM_TOSERVER)?"true":"false", + FLOW_IS_PP_DONE(f, STREAM_TOSERVER)?"true":"false", + FLOW_IS_PM_DONE(f, STREAM_TOCLIENT)?"true":"false", + FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)?"true":"false", + StreamTcpGetStreamSize(&ssn->client), StreamTcpGetStreamSize(&ssn->server)); + + int flow_done = 0; if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)) { + SCLogDebug("proto detection failed for both streams"); + flow_done = 1; + } else if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && + StreamTcpGetStreamSize(&ssn->server) == 0 && + StreamTcpGetStreamSize(&ssn->client) > alp_proto_ctx.toserver.async_max) { + SCLogDebug("%u bytes toserver and no proto, no data to " + "client, giving up", alp_proto_ctx.toserver.async_max); + flow_done = 1; + } else if (FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) && + StreamTcpGetStreamSize(&ssn->client) == 0 && + StreamTcpGetStreamSize(&ssn->server) > alp_proto_ctx.toclient.async_max) { + SCLogDebug("%u bytes toclient and no proto, no data to " + "server, giving up", alp_proto_ctx.toclient.async_max); + flow_done = 1; + } + + if (flow_done) { FlowSetSessionNoApplayerInspectionFlag(f); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->server); StreamTcpSetStreamFlagAppProtoDetectionCompleted(&ssn->client); diff --git a/src/stream-tcp.c b/src/stream-tcp.c index cc3ed39866..50b2d6536c 100644 --- a/src/stream-tcp.c +++ b/src/stream-tcp.c @@ -701,6 +701,19 @@ void StreamTcpSetOSPolicy(TcpStream *stream, Packet *p) } +/** + * \brief get the size of a stream + * + * \note this just calculates the diff between isn and last_ack + * and will not consider sequence wrap arounds (streams + * bigger than 4gb). + * + * \retval size stream size + */ +uint32_t StreamTcpGetStreamSize(TcpStream *stream) { + return (stream->last_ack - stream->isn - 1); +} + /** * \brief macro to update last_ack only if the new value is higher * diff --git a/src/stream-tcp.h b/src/stream-tcp.h index 0f4ead0ae0..87bfcca9e1 100644 --- a/src/stream-tcp.h +++ b/src/stream-tcp.h @@ -206,6 +206,7 @@ TmEcode StreamTcpThreadDeinit(ThreadVars *tv, void *data); int StreamTcpPacket (ThreadVars *tv, Packet *p, StreamTcpThread *stt, PacketQueue *pq); void StreamTcpSessionClear(void *ssnptr); +uint32_t StreamTcpGetStreamSize(TcpStream *stream); #endif /* __STREAM_TCP_H__ */