stream: handle no stream scanning case

Now that detect moves the raw progress forward, it's important
to deal with the case where detect don't consider raw inspection.

If no 'stream' rules are active, disable raw. For this the disable
raw flag is now per stream.

src/detect-engine-mpm.c

@@ -1194,6 +1194,25 @@ static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx,
     return NULL;
 }
 
+static void SetRawReassemblyFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
+{
+    const Signature *s = NULL;
+    uint32_t sig;
+
+    for (sig = 0; sig < sgh->sig_cnt; sig++) {
+        s = sgh->match_array[sig];
+        if (s == NULL)
+            continue;
+
+        if (SignatureHasStreamContent(s) == 1) {
+            sgh->flags |= SIG_GROUP_HEAD_HAVERAWSTREAM;
+            SCLogDebug("rule group %p has SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
+            return;
+        }
+    }
+    SCLogDebug("rule group %p does NOT have SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
+}
+
 /** \brief Prepare the pattern matcher ctx in a sig group head.
  *
  */
@@ -1211,6 +1230,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             if (mpm_store != NULL) {
                 PrefilterPktStreamRegister(sh, mpm_store->mpm_ctx);
             }
+
+            SetRawReassemblyFlag(de_ctx, sh);
         }
         if (SGH_DIRECTION_TC(sh)) {
             mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_PKT_TC);
@@ -1222,6 +1243,8 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             if (mpm_store != NULL) {
                 PrefilterPktStreamRegister(sh, mpm_store->mpm_ctx);
             }
+
+            SetRawReassemblyFlag(de_ctx, sh);
        }
     } else if (SGH_PROTO(sh, IPPROTO_UDP)) {
         if (SGH_DIRECTION_TS(sh)) {

src/detect.c

@@ -860,6 +860,14 @@ DetectPostInspectFirstSGH(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
         pflow->sgh_toserver = sgh;
         pflow->flags |= FLOW_SGH_TOSERVER;
 
+        if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
+            if (pflow->protoctx != NULL) {
+                TcpSession *ssn = pflow->protoctx;
+                SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.client");
+                ssn->client.flags |= STREAMTCP_STREAM_FLAG_DISABLE_RAW;
+            }
+        }
+
         DetectPostInspectFileFlagsUpdate(pflow,
                 pflow->sgh_toserver, STREAM_TOSERVER);
 
@@ -867,6 +875,14 @@ DetectPostInspectFirstSGH(const Packet *p, Flow *pflow, const SigGroupHead *sgh)
         pflow->sgh_toclient = sgh;
         pflow->flags |= FLOW_SGH_TOCLIENT;
 
+        if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) {
+            if (pflow->protoctx != NULL) {
+                TcpSession *ssn = pflow->protoctx;
+                SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.server");
+                ssn->server.flags |= STREAMTCP_STREAM_FLAG_DISABLE_RAW;
+            }
+        }
+
         DetectPostInspectFileFlagsUpdate(pflow,
                 pflow->sgh_toclient, STREAM_TOCLIENT);
     }
@@ -1276,8 +1292,10 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
                 }
                 /* no match? then inspect packet payload */
                 if (pmatch == 0) {
-                    SCLogDebug("no match in smsg, fall back to packet payload");
+                    SCLogDebug("no match in stream, fall back to packet payload");
 
+                    /* skip if we don't have to inspect the packet and segment was
+                     * added to stream */
                     if (!(sflags & SIG_FLAG_REQUIRE_PACKET) && (p->flags & PKT_STREAM_ADD)) {
                         goto next;
                     }

src/detect.h

@@ -946,6 +946,7 @@ typedef struct SigTableElmt_ {
 
 } SigTableElmt;
 
+#define SIG_GROUP_HEAD_HAVERAWSTREAM    (1 << 0)
 #ifdef HAVE_MAGIC
 #define SIG_GROUP_HEAD_HAVEFILEMAGIC    (1 << 20)
 #endif

src/stream-tcp-list.c

@@ -628,7 +628,7 @@ static inline uint64_t GetLeftEdge(TcpSession *ssn, TcpStream *stream)
         use_app = 0;
     }
 
-    if (ssn->flags & STREAMTCP_FLAG_DISABLE_RAW) {
+    if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) {
         // raw is dead
         use_raw = 0;
     }

src/stream-tcp-private.h

@@ -134,8 +134,7 @@ enum
 #define STREAMTCP_FLAG_TIMESTAMP                    0x0008
 /** Server supports wscale (even though it can be 0) */
 #define STREAMTCP_FLAG_SERVER_WSCALE                0x0010
-/** 'Raw' reassembly is disabled for this ssn. */
-#define STREAMTCP_FLAG_DISABLE_RAW                  0x0020
+// vacancy
 /** Flag to indicate that the session is handling asynchronous stream.*/
 #define STREAMTCP_FLAG_ASYNC                        0x0040
 /** Flag to indicate we're dealing with 4WHS: SYN, SYN, SYN/ACK, ACK
@@ -148,9 +147,7 @@ enum
 #define STREAMTCP_FLAG_CLIENT_SACKOK                0x0200
 /** Flag to indicate both sides of the session permit SACK (SYN + SYN/ACK) */
 #define STREAMTCP_FLAG_SACKOK                       0x0400
-/** Flag for triggering RAW reassembly before the size limit is reached or
-    the stream reaches EOF. */
-//#define STREAMTCP_FLAG_TRIGGER_RAW_REASSEMBLY       0x0800
+// vacancy
 /** 3WHS confirmed by server -- if suri sees 3whs ACK but server doesn't (pkt
  *  is lost on the way to server), SYN/ACK is retransmitted. If server sends
  *  normal packet we assume 3whs to be completed. Only used for SYN/ACK resend
@@ -184,7 +181,10 @@ enum
 #define STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED 0x0100
 /** Raw reassembly disabled for new segments */
 #define STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED 0x0200
-// vacancy 2x
+/** Raw reassembly disabled completely */
+#define STREAMTCP_STREAM_FLAG_DISABLE_RAW 0x400
+// vacancy 1x
+
 /** NOTE: flags field is 12 bits */
 
 

src/stream-tcp-reassemble.c

@@ -794,7 +794,7 @@ int StreamNeedsReassembly(TcpSession *ssn, int direction)
         use_app = 0;
     }
 
-    if (ssn->flags & STREAMTCP_FLAG_DISABLE_RAW) {
+    if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW) {
         // raw is dead
         use_raw = 0;
     }

src/stream-tcp.c

@@ -560,7 +560,7 @@ void StreamTcpInitConfig(char quiet)
     int enable_raw = 1;
     if (ConfGetBool("stream.reassembly.raw", &enable_raw) == 1) {
         if (!enable_raw) {
-            stream_config.ssn_init_flags = STREAMTCP_FLAG_DISABLE_RAW;
+            stream_config.stream_init_flags = STREAMTCP_STREAM_FLAG_DISABLE_RAW;
         }
     } else {
         enable_raw = 1;
@@ -639,8 +639,9 @@ TcpSession *StreamTcpNewSession (Packet *p, int id)
 
         ssn->state = TCP_NONE;
         ssn->reassembly_depth = stream_config.reassembly_depth;
-        ssn->flags = stream_config.ssn_init_flags;
         ssn->tcp_packet_flags = p->tcph ? p->tcph->th_flags : 0;
+        ssn->server.flags = stream_config.stream_init_flags;
+        ssn->client.flags = stream_config.stream_init_flags;
 
         StreamingBuffer x = STREAMING_BUFFER_INITIALIZER(&stream_config.sbcnf);
         ssn->client.sb = x;

src/stream-tcp.h

@@ -44,7 +44,7 @@ typedef struct TcpStreamCnf_ {
     uint64_t memcap;
     uint64_t reassembly_memcap; /**< max memory usage for stream reassembly */
 
-    uint32_t ssn_init_flags; /**< new ssn flags will be initialized to this */
+    uint16_t stream_init_flags; /**< new stream flags will be initialized to this */
 
     uint32_t prealloc_sessions; /**< ssns to prealloc per stream thread */
     uint32_t prealloc_segments; /**< segments to prealloc per stream thread */