proto detect: TCP cleanup

Split function into multiple smaller ones.

src/app-layer.c

@@ -115,29 +115,163 @@ void AppLayerIncTxCounter(ThreadVars *tv, Flow *f, uint64_t step)
     }
 }
 
-int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
+/* See if we're going to have to give up:
-                          Packet *p, Flow *f,
+ *
-                          TcpSession *ssn, TcpStream *stream,
+ * If we're getting a lot of data in one direction and the
-                          uint8_t *data, uint32_t data_len,
+ * proto for this direction is unknown, proto detect will
+ * hold up segments in the segment list in the stream.
+ * They are held so that if we detect the protocol on the
+ * opposing stream, we can still parse this side of the stream
+ * as well. However, some sessions are very unbalanced. FTP
+ * data channels, large PUT/POST request and many others, can
+ * lead to cases where we would have to store many megabytes
+ * worth of segments before we see the opposing stream. This
+ * leads to risks of resource starvation.
+ *
+ * Here a cutoff point is enforced. If we've stored 100k in
+ * one direction and we've seen no data in the other direction,
+ * we give up.
+ *
+ * Giving up means we disable applayer an set an applayer event
+ */
+static void TCPProtoDetectCheckBailConditions(Flow *f, TcpSession *ssn, Packet *p)
+{
+    uint32_t size_ts = ssn->client.last_ack - ssn->client.isn - 1;
+    uint32_t size_tc = ssn->server.last_ack - ssn->server.isn - 1;
+    SCLogDebug("size_ts %u, size_tc %u", size_ts, size_tc);
+
+#ifdef DEBUG_VALIDATION
+    if (!(ssn->client.flags & STREAMTCP_STREAM_FLAG_GAP))
+        BUG_ON(size_ts > 1000000UL);
+    if (!(ssn->server.flags & STREAMTCP_STREAM_FLAG_GAP))
+        BUG_ON(size_tc > 1000000UL);
+#endif /* DEBUG_VALIDATION */
+
+    if (ProtoDetectDone(f, ssn, STREAM_TOSERVER) &&
+        ProtoDetectDone(f, ssn, STREAM_TOCLIENT))
+    {
+        DisableAppLayer(f);
+        ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+
+    } else if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
+            size_ts > 100000 && size_tc == 0)
+    {
+        DisableAppLayer(f);
+        ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                APPLAYER_PROTO_DETECTION_SKIPPED);
+
+    } else if (FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) &&
+            size_tc > 100000 && size_ts == 0)
+    {
+        DisableAppLayer(f);
+        ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                APPLAYER_PROTO_DETECTION_SKIPPED);
+
+    /* little data in ts direction, pp done, pm not done (max
+     * depth not reached), ts direction done, lots of data in
+     * tc direction. */
+    } else if (size_tc > 100000 &&
+            FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
+            FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT))
+    {
+        DisableAppLayer(f);
+        ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                APPLAYER_PROTO_DETECTION_SKIPPED);
+
+    /* little data in tc direction, pp done, pm not done (max
+     * depth not reached), tc direction done, lots of data in
+     * ts direction. */
+    } else if (size_ts > 100000 &&
+            FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) && !(FLOW_IS_PM_DONE(f, STREAM_TOCLIENT)) &&
+            FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER))
+    {
+        DisableAppLayer(f);
+        ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                APPLAYER_PROTO_DETECTION_SKIPPED);
+
+    /* in case of really low TS data (e.g. 4 bytes) we can have
+     * the PP complete, PM not complete (depth not reached) and
+     * the TC side also not recognized (proto unknown) */
+    } else if (size_tc > 100000 &&
+            FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
+            (!FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && !FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)))
+    {
+        DisableAppLayer(f);
+        ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+        AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+                APPLAYER_PROTO_DETECTION_SKIPPED);
+    }
+}
+
+/** \todo modifying p this way seems too hacky */
+static int TCPProtoDetectTriggerOpposingSide(ThreadVars *tv,
+        TcpReassemblyThreadCtx *ra_ctx,
+        Packet *p, TcpSession *ssn, TcpStream *stream,
         uint8_t flags)
 {
-    SCEnter();
+    TcpStream *opposing_stream = NULL;
+    if (stream == &ssn->client) {
+        opposing_stream = &ssn->server;
+        if (StreamTcpInlineMode()) {
+            p->flowflags &= ~FLOW_PKT_TOSERVER;
+            p->flowflags |= FLOW_PKT_TOCLIENT;
+        } else {
+            p->flowflags &= ~FLOW_PKT_TOCLIENT;
+            p->flowflags |= FLOW_PKT_TOSERVER;
+        }
+    } else {
+        opposing_stream = &ssn->client;
+        if (StreamTcpInlineMode()) {
+            p->flowflags &= ~FLOW_PKT_TOCLIENT;
+            p->flowflags |= FLOW_PKT_TOSERVER;
+        } else {
+            p->flowflags &= ~FLOW_PKT_TOSERVER;
+            p->flowflags |= FLOW_PKT_TOCLIENT;
+        }
+    }
 
-    DEBUG_ASSERT_FLOW_LOCKED(f);
+    int ret = 0;
+    /* if the opposing side is not going to work, then
+     * we just have to give up. */
+    if (opposing_stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)
+        ret = -1;
+    else
+        ret = StreamTcpReassembleAppLayer(tv, ra_ctx, ssn,
+                opposing_stream, p);
+    if (stream == &ssn->client) {
+        if (StreamTcpInlineMode()) {
+            p->flowflags &= ~FLOW_PKT_TOCLIENT;
+            p->flowflags |= FLOW_PKT_TOSERVER;
+        } else {
+            p->flowflags &= ~FLOW_PKT_TOSERVER;
+            p->flowflags |= FLOW_PKT_TOCLIENT;
+        }
+    } else {
+        if (StreamTcpInlineMode()) {
+            p->flowflags &= ~FLOW_PKT_TOSERVER;
+            p->flowflags |= FLOW_PKT_TOCLIENT;
+        } else {
+            p->flowflags &= ~FLOW_PKT_TOCLIENT;
+            p->flowflags |= FLOW_PKT_TOSERVER;
+        }
+    }
+    return ret;
+}
 
-    AppLayerThreadCtx *app_tctx = ra_ctx->app_tctx;
+/** \todo data const */
+static int TCPProtoDetect(ThreadVars *tv,
+        TcpReassemblyThreadCtx *ra_ctx, AppLayerThreadCtx *app_tctx,
+        Packet *p, Flow *f, TcpSession *ssn, TcpStream *stream,
+        uint8_t *data, uint32_t data_len, uint8_t flags)
+{
+    uint32_t data_al_so_far = 0;
     AppProto *alproto;
     AppProto *alproto_otherdir;
-    uint8_t dir;
+    int dir;
-    uint32_t data_al_so_far;
-    int r = 0;
-    uint8_t first_data_dir;
-
-    SCLogDebug("data_len %u flags %02X", data_len, flags);
-    if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) {
-        SCLogDebug("STREAMTCP_FLAG_APP_LAYER_DISABLED is set");
-        goto end;
-    }
 
     if (flags & STREAM_TOSERVER) {
         alproto = &f->alproto_ts;
@@ -148,19 +282,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         alproto_otherdir = &f->alproto_ts;
         dir = 1;
     }
-
+    if (data_len > 0)
-    /* if we don't know the proto yet and we have received a stream
-     * initializer message, we run proto detection.
-     * We receive 2 stream init msgs (one for each direction) but we
-     * only run the proto detection once. */
-    if (*alproto == ALPROTO_UNKNOWN && (flags & STREAM_GAP)) {
-        StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream);
-        StreamTcpSetSessionNoReassemblyFlag(ssn, dir);
-        SCLogDebug("ALPROTO_UNKNOWN flow %p, due to GAP in stream start", f);
-    } else if (*alproto == ALPROTO_UNKNOWN && (flags & STREAM_START)) {
-        if (data_len == 0)
-            data_al_so_far = 0;
-        else
         data_al_so_far = f->data_al_so_far[dir];
 
     SCLogDebug("Stream initializer (len %" PRIu32 ")", data_len);
@@ -176,8 +298,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
 
     PACKET_PROFILING_APP_PD_START(app_tctx);
     *alproto = AppLayerProtoDetectGetProto(app_tctx->alpd_tctx,
-                                f,
+            f, data, data_len,
-                                data, data_len,
             IPPROTO_TCP, flags);
     PACKET_PROFILING_APP_PD_END(app_tctx);
 
@@ -198,9 +319,10 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
 
         f->alproto = *alproto;
         StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream);
-            TcpSessionSetReassemblyDepth(ssn, AppLayerParserGetStreamDepth(f->proto, *alproto));
+        TcpSessionSetReassemblyDepth(ssn,
+                AppLayerParserGetStreamDepth(f->proto, *alproto));
 
-            /* account flow if we have both side */
+        /* account flow if we have both sides */
         if (*alproto_otherdir != ALPROTO_UNKNOWN) {
             AppLayerIncFlowCounter(tv, f);
         }
@@ -213,54 +335,11 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
          * called by the very same StreamReassembly() function that we
          * will now call shortly for the opposing direction. */
         if ((ssn->data_first_seen_dir & (STREAM_TOSERVER | STREAM_TOCLIENT)) &&
-                !(flags & ssn->data_first_seen_dir)) {
+                !(flags & ssn->data_first_seen_dir))
-                TcpStream *opposing_stream = NULL;
+        {
-                if (stream == &ssn->client) {
+            if (TCPProtoDetectTriggerOpposingSide(tv, ra_ctx,
-                    opposing_stream = &ssn->server;
+                p, ssn, stream, flags) != 0)
-                    if (StreamTcpInlineMode()) {
+            {
-                        p->flowflags &= ~FLOW_PKT_TOSERVER;
-                        p->flowflags |= FLOW_PKT_TOCLIENT;
-                    } else {
-                        p->flowflags &= ~FLOW_PKT_TOCLIENT;
-                        p->flowflags |= FLOW_PKT_TOSERVER;
-                    }
-                } else {
-                    opposing_stream = &ssn->client;
-                    if (StreamTcpInlineMode()) {
-                        p->flowflags &= ~FLOW_PKT_TOCLIENT;
-                        p->flowflags |= FLOW_PKT_TOSERVER;
-                    } else {
-                        p->flowflags &= ~FLOW_PKT_TOSERVER;
-                        p->flowflags |= FLOW_PKT_TOCLIENT;
-                    }
-                }
-
-                int ret = 0;
-                /* if the opposing side is not going to work, then
-                 * we just have to give up. */
-                if (opposing_stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)
-                    ret = -1;
-                else
-                    ret = StreamTcpReassembleAppLayer(tv, ra_ctx, ssn,
-                                                      opposing_stream, p);
-                if (stream == &ssn->client) {
-                    if (StreamTcpInlineMode()) {
-                        p->flowflags &= ~FLOW_PKT_TOCLIENT;
-                        p->flowflags |= FLOW_PKT_TOSERVER;
-                    } else {
-                        p->flowflags &= ~FLOW_PKT_TOSERVER;
-                        p->flowflags |= FLOW_PKT_TOCLIENT;
-                    }
-                } else {
-                    if (StreamTcpInlineMode()) {
-                        p->flowflags &= ~FLOW_PKT_TOSERVER;
-                        p->flowflags |= FLOW_PKT_TOCLIENT;
-                    } else {
-                        p->flowflags &= ~FLOW_PKT_TOCLIENT;
-                        p->flowflags |= FLOW_PKT_TOSERVER;
-                    }
-                }
-                if (ret < 0) {
                 DisableAppLayer(f);
                 goto failure;
             }
@@ -283,6 +362,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
          *       flow, it shows something's fishy.
          */
         if (ssn->data_first_seen_dir != APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER) {
+            uint8_t first_data_dir;
             first_data_dir = AppLayerParserGetFirstDataDir(f->proto, *alproto);
 
             if (first_data_dir && !(first_data_dir & ssn->data_first_seen_dir)) {
@@ -314,12 +394,16 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         /* Set a value that is neither STREAM_TOSERVER, nor STREAM_TOCLIENT */
         ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
 
+        /* finally, invoke the parser */
         PACKET_PROFILING_APP_START(app_tctx, *alproto);
-            r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, *alproto,
+        int r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, *alproto,
                 flags, data + data_al_so_far,
                 data_len - data_al_so_far);
         PACKET_PROFILING_APP_END(app_tctx, *alproto);
         f->data_al_so_far[dir] = 0;
+        if (r < 0)
+            goto failure;
+
     } else {
         /* if the ssn is midstream, we may end up with a case where the
          * start of an HTTP request is missing. We won't detect HTTP based
@@ -335,7 +419,9 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
          * start of the request/toserver is incomplete and no reliable
          * detection and parsing is possible. So we give up.
          */
-            if ((ssn->flags & STREAMTCP_FLAG_MIDSTREAM) && !(ssn->flags & STREAMTCP_FLAG_MIDSTREAM_SYNACK)) {
+        if ((ssn->flags & STREAMTCP_FLAG_MIDSTREAM) &&
+                !(ssn->flags & STREAMTCP_FLAG_MIDSTREAM_SYNACK))
+        {
             if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER)) {
                 SCLogDebug("midstream end pd %p", ssn);
                 /* midstream and toserver detection failed: give up */
@@ -346,6 +432,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         }
 
         if (*alproto_otherdir != ALPROTO_UNKNOWN) {
+            uint8_t first_data_dir;
             first_data_dir = AppLayerParserGetFirstDataDir(f->proto, *alproto_otherdir);
 
             /* this would handle this test case -
@@ -374,7 +461,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                 ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
 
             PACKET_PROFILING_APP_START(app_tctx, *alproto_otherdir);
-                r = AppLayerParserParse(tv, app_tctx->alp_tctx, f,
+            int r = AppLayerParserParse(tv, app_tctx->alp_tctx, f,
                     *alproto_otherdir, flags,
                     data + data_al_so_far,
                     data_len - data_al_so_far);
@@ -384,92 +471,74 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
                         APPLAYER_DETECT_PROTOCOL_ONLY_ONE_DIRECTION);
                 StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream);
                 f->data_al_so_far[dir] = 0;
-                    TcpSessionSetReassemblyDepth(ssn, AppLayerParserGetStreamDepth(f->proto, *alproto));
+                TcpSessionSetReassemblyDepth(ssn,
+                        AppLayerParserGetStreamDepth(f->proto, *alproto));
             } else {
                 f->data_al_so_far[dir] = data_len;
             }
+            if (r < 0)
+                goto failure;
+
         } else {
-                /* See if we're going to have to give up:
+            /* both sides unknown, let's see if we need to give up */
-                 *
+            TCPProtoDetectCheckBailConditions(f, ssn, p);
-                 * If we're getting a lot of data in one direction and the
+        }
-                 * proto for this direction is unknown, proto detect will
+    }
-                 * hold up segments in the segment list in the stream.
+end:
-                 * They are held so that if we detect the protocol on the
+    return 0;
-                 * opposing stream, we can still parse this side of the stream
-                 * as well. However, some sessions are very unbalanced. FTP
-                 * data channels, large PUT/POST request and many others, can
-                 * lead to cases where we would have to store many megabytes
-                 * worth of segments before we see the opposing stream. This
-                 * leads to risks of resource starvation.
-                 *
-                 * Here a cutoff point is enforced. If we've stored 100k in
-                 * one direction and we've seen no data in the other direction,
-                 * we give up. */
-                uint32_t size_ts = ssn->client.last_ack - ssn->client.isn - 1;
-                uint32_t size_tc = ssn->server.last_ack - ssn->server.isn - 1;
-                SCLogDebug("size_ts %u, size_tc %u", size_ts, size_tc);
-#ifdef DEBUG_VALIDATION
-                if (!(ssn->client.flags & STREAMTCP_STREAM_FLAG_GAP))
-                    BUG_ON(size_ts > 1000000UL);
-                if (!(ssn->server.flags & STREAMTCP_STREAM_FLAG_GAP))
-                    BUG_ON(size_tc > 1000000UL);
-#endif /* DEBUG_VALIDATION */
 
-                if (ProtoDetectDone(f, ssn, STREAM_TOSERVER) &&
+failure:
-                    ProtoDetectDone(f, ssn, STREAM_TOCLIENT))
+    return -1;
-                {
+}
-                    DisableAppLayer(f);
-                    ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
 
-                } else if (FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER) &&
+/** \brief handle TCP data for the app-layer.
-                        size_ts > 100000 && size_tc == 0)
+ *
-                {
+ *  First run protocol detection and then when the protocol is known invoke
-                    DisableAppLayer(f);
+ *  the app layer parser.
-                    ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+ */
-                    AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
-                                                     APPLAYER_PROTO_DETECTION_SKIPPED);
+                          Packet *p, Flow *f,
-                } else if (FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) &&
+                          TcpSession *ssn, TcpStream *stream,
-                        size_tc > 100000 && size_ts == 0)
+                          uint8_t *data, uint32_t data_len,
-                {
+                          uint8_t flags)
-                    DisableAppLayer(f);
-                    ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
-                    AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                                                     APPLAYER_PROTO_DETECTION_SKIPPED);
-                /* little data in ts direction, pp done, pm not done (max
-                 * depth not reached), ts direction done, lots of data in
-                 * tc direction. */
-                } else if (size_tc > 100000 &&
-                           FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
-                           FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && FLOW_IS_PP_DONE(f, STREAM_TOCLIENT))
-                {
-                    DisableAppLayer(f);
-                    ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
-                    AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                                                     APPLAYER_PROTO_DETECTION_SKIPPED);
-                /* little data in tc direction, pp done, pm not done (max
-                 * depth not reached), tc direction done, lots of data in
-                 * ts direction. */
-                } else if (size_ts > 100000 &&
-                           FLOW_IS_PP_DONE(f, STREAM_TOCLIENT) && !(FLOW_IS_PM_DONE(f, STREAM_TOCLIENT)) &&
-                           FLOW_IS_PM_DONE(f, STREAM_TOSERVER) && FLOW_IS_PP_DONE(f, STREAM_TOSERVER))
-                {
-                    DisableAppLayer(f);
-                    ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
-                    AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
-                                                     APPLAYER_PROTO_DETECTION_SKIPPED);
-                /* in case of really low TS data (e.g. 4 bytes) we can have
-                 * the PP complete, PM not complete (depth not reached) and
-                 * the TC side also not recognized (proto unknown) */
-                } else if (size_tc > 100000 &&
-                           FLOW_IS_PP_DONE(f, STREAM_TOSERVER) && !(FLOW_IS_PM_DONE(f, STREAM_TOSERVER)) &&
-                           (!FLOW_IS_PM_DONE(f, STREAM_TOCLIENT) && !FLOW_IS_PP_DONE(f, STREAM_TOCLIENT)))
 {
-                    DisableAppLayer(f);
+    SCEnter();
-                    ssn->data_first_seen_dir = APP_LAYER_DATA_ALREADY_SENT_TO_APP_LAYER;
+
-                    AppLayerDecoderEventsSetEventRaw(&p->app_layer_events,
+    DEBUG_ASSERT_FLOW_LOCKED(f);
-                                                     APPLAYER_PROTO_DETECTION_SKIPPED);
+
+    AppLayerThreadCtx *app_tctx = ra_ctx->app_tctx;
+    AppProto alproto;
+    uint8_t dir;
+    int r = 0;
+
+    SCLogDebug("data_len %u flags %02X", data_len, flags);
+    if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) {
+        SCLogDebug("STREAMTCP_FLAG_APP_LAYER_DISABLED is set");
+        goto end;
     }
+
+    if (flags & STREAM_TOSERVER) {
+        alproto = f->alproto_ts;
+        dir = 0;
+    } else {
+        alproto = f->alproto_tc;
+        dir = 1;
     }
+
+    /* if we don't know the proto yet and we have received a stream
+     * initializer message, we run proto detection.
+     * We receive 2 stream init msgs (one for each direction) but we
+     * only run the proto detection once. */
+    if (alproto == ALPROTO_UNKNOWN && (flags & STREAM_GAP)) {
+        StreamTcpSetStreamFlagAppProtoDetectionCompleted(stream);
+        StreamTcpSetSessionNoReassemblyFlag(ssn, dir);
+        SCLogDebug("ALPROTO_UNKNOWN flow %p, due to GAP in stream start", f);
+
+    } else if (alproto == ALPROTO_UNKNOWN && (flags & STREAM_START)) {
+        /* run protocol detection */
+        if (TCPProtoDetect(tv, ra_ctx, app_tctx, p, f, ssn, stream,
+                           data, data_len, flags) != 0) {
+            goto failure;
         }
     } else {
         SCLogDebug("stream data (len %" PRIu32 " alproto "

src/detect-app-layer-protocol.c

@@ -31,19 +31,21 @@
 #include "util-unittest.h"
 #include "util-unittest-helper.h"
 
-void DetectAppLayerProtocolRegisterTests(void);
+static void DetectAppLayerProtocolRegisterTests(void);
 
-int DetectAppLayerProtocolMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx,
+static int DetectAppLayerProtocolMatch(ThreadVars *t, DetectEngineThreadCtx *det_ctx,
                                 Flow *f, uint8_t flags, void *state,
                                 Signature *s, SigMatch *m)
 {
+    SCEnter();
+
     int r = 0;
     DetectAppLayerProtocolData *data = (DetectAppLayerProtocolData *)m->ctx;
 
     r = (data->negated) ? (f->alproto != data->alproto) :
         (f->alproto == data->alproto);
 
-    return r;
+    SCReturnInt(r);
 }
 
 static DetectAppLayerProtocolData *DetectAppLayerProtocolParse(const char *arg)
@@ -86,8 +88,8 @@ static DetectAppLayerProtocolData *DetectAppLayerProtocolParse(const char *arg)
     return data;
 }
 
-int DetectAppLayerProtocolSetup(DetectEngineCtx *de_ctx, Signature *s,
+static int DetectAppLayerProtocolSetup(DetectEngineCtx *de_ctx,
-                                char *arg)
+        Signature *s, char *arg)
 {
     DetectAppLayerProtocolData *data = NULL;
     SigMatch *sm = NULL;
@@ -125,10 +127,9 @@ error:
     return -1;
 }
 
-void DetectAppLayerProtocolFree(void *ptr)
+static void DetectAppLayerProtocolFree(void *ptr)
 {
     SCFree(ptr);
-
     return;
 }
 
@@ -389,7 +390,7 @@ int DetectAppLayerProtocolTest09(void)
 
 #endif /* UNITTESTS */
 
-void DetectAppLayerProtocolRegisterTests(void)
+static void DetectAppLayerProtocolRegisterTests(void)
 {
 #ifdef UNITTESTS /* UNITTESTS */
     UtRegisterTest("DetectAppLayerProtocolTest01",