connect/starttls: handle detection corner cases

When switching protocol from http to tls the following corner case
was observed:

 pkt 6, TC "200 connection established"
 pkt 7, TS acks pkt 6 + adds "client hello"
 pkt 8 TC, acks pkt 7
 pkt 8 is where normally the detect on the 200 connection established
       would run however before detection runs the app-layer is called
       and it resets the state

So the issue is missed detection on the last data in the original
protocol before the switch.

Another case was:

TS ->    STARTTLS
TC ->    Ack "STARTTLS data"
         220
TS ->    Ack "220 data"
         Client Hello

In IDS mode, this made a rule that wanted to look at content:"STARTTLS"
in combination with the protocol SMTP 'alert smtp ... content:"STARTTLS";'
impossible. By the time the content would match, the protocol was already
switched.

This patch fixes this case by creating a 'Detect/Log Flush' packet in
both directions. This will force final inspection and logging of the
pre-upgrade protocol (SMTP in this example) before doing the final
switch.

src/decode.c

@@ -572,6 +572,9 @@ const char *PktSrcToString(enum PktSrcEnum pkt_src)
         case PKT_SRC_STREAM_TCP_STREAM_END_PSEUDO:
             pkt_src_str = "stream";
             break;
+        case PKT_SRC_STREAM_TCP_DETECTLOG_FLUSH:
+            pkt_src_str = "stream (detect/log)";
+            break;
         case PKT_SRC_FFR:
             pkt_src_str = "stream (flow timeout)";
             break;

src/decode.h

@@ -54,6 +54,7 @@ enum PktSrcEnum {
     PKT_SRC_DEFRAG,
     PKT_SRC_STREAM_TCP_STREAM_END_PSEUDO,
     PKT_SRC_FFR,
+    PKT_SRC_STREAM_TCP_DETECTLOG_FLUSH,
 };
 
 #include "source-nflog.h"
@@ -1117,9 +1118,12 @@ int DecoderParseDataFromFileSerie(char *fileprefix, DecoderFunc Decoder);
                                                      * fragments. */
 #define PKT_DETECT_HAS_STREAMDATA       (1<<26)     /**< Set by Detect() if raw stream data is available. */
 
+#define PKT_PSEUDO_DETECTLOG_FLUSH      (1<<27)     /**< Detect/log flush for protocol upgrade */
+
 
 /** \brief return 1 if the packet is a pseudo packet */
-#define PKT_IS_PSEUDOPKT(p) ((p)->flags & PKT_PSEUDO_STREAM_END)
+#define PKT_IS_PSEUDOPKT(p) \
+    ((p)->flags & (PKT_PSEUDO_STREAM_END|PKT_PSEUDO_DETECTLOG_FLUSH))
 
 #define PKT_SET_SRC(p, src_val) ((p)->pkt_src = src_val)
 

src/flow-worker.c

@@ -214,6 +214,10 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data, PacketQueue *pr
         StreamTcp(tv, p, fw->stream_thread, &fw->pq, NULL);
         FLOWWORKER_PROFILING_END(p, PROFILE_FLOWWORKER_STREAM);
 
+        if (FlowChangeProto(p->flow)) {
+            StreamTcpDetectLogFlush(tv, fw->stream_thread, p->flow, p, &fw->pq);
+        }
+
         /* Packets here can safely access p->flow as it's locked */
         SCLogDebug("packet %"PRIu64": extra packets %u", p->pcap_cnt, fw->pq.len);
         Packet *x;

src/stream-tcp.c

@@ -2063,6 +2063,7 @@ static int HandleEstablishedPacketToServer(ThreadVars *tv, TcpSession *ssn, Pack
 
         /* handle data (if any) */
         StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, &ssn->client, p, pq);
+
     } else {
         SCLogDebug("ssn %p: toserver => SEQ out of window, packet SEQ "
                 "%" PRIu32 ", payload size %" PRIu32 " (%" PRIu32 "),"
@@ -5722,6 +5723,90 @@ void StreamTcpPseudoPacketCreateStreamEndPacket(ThreadVars *tv, StreamTcpThread
     SCReturn;
 }
 
+/** \brief Create a pseudo packet injected into the engine to signal the
+ *         opposing direction of this stream trigger detection/logging.
+ *
+ *  \param p real packet
+ *  \param pq packet queue to store the new pseudo packet in
+ *  \param dir 0 ts 1 tc
+ */
+static void StreamTcpPseudoPacketCreateDetectLogFlush(ThreadVars *tv,
+        StreamTcpThread *stt, Packet *p, TcpSession *ssn, PacketQueue *pq, bool dir)
+{
+    SCEnter();
+
+    if (p->flags & PKT_PSEUDO_DETECTLOG_FLUSH) {
+        SCReturn;
+    }
+
+    Packet *np = StreamTcpPseudoSetup(p, GET_PKT_DATA(p), GET_PKT_LEN(p));
+    if (np == NULL) {
+        SCLogDebug("The packet received from packet allocation is NULL");
+        StatsIncr(tv, stt->counter_tcp_pseudo_failed);
+        SCReturn;
+    }
+    PKT_SET_SRC(np, PKT_SRC_STREAM_TCP_DETECTLOG_FLUSH);
+
+    /* Setup the IP and TCP headers */
+    StreamTcpPseudoPacketSetupHeader(np,p);
+
+    np->tenant_id = p->flow->tenant_id;
+    np->flowflags = p->flowflags;
+
+    np->flags |= PKT_STREAM_EST;
+    np->flags |= PKT_HAS_FLOW;
+    np->flags |= PKT_IGNORE_CHECKSUM;
+    np->flags |= PKT_PSEUDO_DETECTLOG_FLUSH;
+
+    if (p->flags & PKT_NOPACKET_INSPECTION) {
+        DecodeSetNoPacketInspectionFlag(np);
+    }
+    if (p->flags & PKT_NOPAYLOAD_INSPECTION) {
+        DecodeSetNoPayloadInspectionFlag(np);
+    }
+
+    if (dir == false) {
+        SCLogDebug("pseudo is to_client");
+        np->flowflags &= ~(FLOW_PKT_TOSERVER|FLOW_PKT_TOCLIENT);
+        np->flowflags |= FLOW_PKT_TOCLIENT;
+#ifdef DEBUG
+        BUG_ON(!(PKT_IS_TOCLIENT(np)));
+        BUG_ON((PKT_IS_TOSERVER(np)));
+#endif
+    } else {
+        SCLogDebug("pseudo is to_server");
+        np->flowflags &= ~(FLOW_PKT_TOCLIENT|FLOW_PKT_TOSERVER);
+        np->flowflags |= FLOW_PKT_TOSERVER;
+#ifdef DEBUG
+        BUG_ON(!(PKT_IS_TOSERVER(np)));
+        BUG_ON((PKT_IS_TOCLIENT(np)));
+#endif
+    }
+
+    PacketEnqueue(pq, np);
+
+    StatsIncr(tv, stt->counter_tcp_pseudo);
+    SCReturn;
+}
+
+/** \brief create packets in both directions to flush out logging
+ *         and detection before switching protocols.
+ *         In IDS mode, create first in packet dir, 2nd in opposing
+ *         In IPS mode, do the reverse.
+ *         Flag TCP engine that data needs to be inspected regardless
+ *         of how far we are wrt inspect limits.
+ */
+void StreamTcpDetectLogFlush(ThreadVars *tv, StreamTcpThread *stt, Flow *f, Packet *p, PacketQueue *pq)
+{
+    TcpSession *ssn = f->protoctx;
+    ssn->client.flags |= STREAMTCP_STREAM_FLAG_TRIGGER_RAW;
+    ssn->server.flags |= STREAMTCP_STREAM_FLAG_TRIGGER_RAW;
+    bool ts = PKT_IS_TOSERVER(p) ? true : false;
+    ts ^= StreamTcpInlineMode();
+    StreamTcpPseudoPacketCreateDetectLogFlush(tv, stt, p, ssn, pq, ts^0);
+    StreamTcpPseudoPacketCreateDetectLogFlush(tv, stt, p, ssn, pq, ts^1);
+}
+
 /**
  * \brief Run callback function on each TCP segment
  *

src/stream-tcp.h

@@ -122,6 +122,7 @@ typedef int (*StreamReassembleRawFunc)(void *data, const uint8_t *input, const u
 int StreamReassembleRaw(TcpSession *ssn, const Packet *p,
         StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out);
 void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress);
+void StreamTcpDetectLogFlush(ThreadVars *tv, StreamTcpThread *stt, Flow *f, Packet *p, PacketQueue *pq);
 
 /** ------- Inline functions: ------ */