app-layer/pd: only consider actual available data

For size limit checks consider only available data at the stream start
and before any GAPS.

The old check would consider too much data if there were temporary gaps,
like when a data packet was in-window but (far) ahead of the expected
segment.

src/app-layer.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2020 Open Information Security Foundation
+/* Copyright (C) 2007-2021 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -205,11 +205,9 @@ static void TCPProtoDetectCheckBailConditions(ThreadVars *tv,
         return;
     }
 
-    const uint64_t size_ts = STREAM_HAS_SEEN_DATA(&ssn->client) ?
-        STREAM_RIGHT_EDGE(&ssn->client) : 0;
-    const uint64_t size_tc = STREAM_HAS_SEEN_DATA(&ssn->server) ?
-        STREAM_RIGHT_EDGE(&ssn->server) : 0;
-    SCLogDebug("size_ts %"PRIu64", size_tc %"PRIu64, size_ts, size_tc);
+    const uint32_t size_ts = StreamDataAvailableForProtoDetect(&ssn->client);
+    const uint32_t size_tc = StreamDataAvailableForProtoDetect(&ssn->server);
+    SCLogDebug("size_ts %" PRIu32 ", size_tc %" PRIu32, size_ts, size_tc);
 
     DEBUG_VALIDATE_BUG_ON(size_ts > 1000000UL);
     DEBUG_VALIDATE_BUG_ON(size_tc > 1000000UL);

src/stream-tcp-reassemble.c

@@ -587,6 +587,20 @@ static uint32_t StreamTcpReassembleCheckDepth(TcpSession *ssn, TcpStream *stream
     SCReturnUInt(0);
 }
 
+uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream)
+{
+    if (RB_EMPTY(&stream->sb.sbb_tree)) {
+        if (stream->sb.stream_offset != 0)
+            return 0;
+
+        return stream->sb.buf_offset;
+    } else {
+        DEBUG_VALIDATE_BUG_ON(stream->sb.head == NULL);
+        DEBUG_VALIDATE_BUG_ON(stream->sb.sbb_size == 0);
+        return stream->sb.sbb_size;
+    }
+}
+
 /**
  *  \brief Insert a packets TCP data into the stream reassembly engine.
  *

src/stream-tcp-reassemble.h

@@ -140,5 +140,7 @@ static inline bool STREAM_LASTACK_GT_BASESEQ(const TcpStream *stream)
     return false;
 }
 
+uint32_t StreamDataAvailableForProtoDetect(TcpStream *stream);
+
 #endif /* __STREAM_TCP_REASSEMBLE_H__ */