stream: add TCP urgent handling options

TCP urgent handling is a complex topic due to conflicting RFCs and
implementations.

Until now the URG flag and urgent pointer values were simply ignored,
leading to an effective "inline" processing of urgent data. Many
implementations however, do not default to this behavior.

Many actual implementations use the urgent mechanism to send 1 byte of
data out of band to the application.

Complicating the matter is that the way the urgent logic is handled is
generally configurable both of the OS and the app level. So from the
network it is impossible to know with confidence what the settings are.

This patch adds the following policies:

`stream.reassembly.urgent.policy`:

- drop: drop URG packets before they affect the stream engine

- inline: ignore the urgent pointer and process all data inline

- oob (out of band): treat the last byte as out of band

- gap: skip the last byte, but do no adjust sequence offsets, leading to
       gaps in the data

For the `oob` option, tracking of a sequence number offset is required,
as the OOB data does "consume" sequence number space. This is limited to
64k. For this reason, there is a second policy:

`stream.reassembly.urgent.oob-limit-policy`:

- drop: drop URG packets before they affect the stream engine

- inline: ignore the urgent pointer and process all data inline

- gap: skip the last byte, but do no adjust sequence offsets, leading to
       gaps in the data

Bug: #7411.

etc/schema.json

@@ -1129,7 +1129,8 @@
                     "$ref": "#/$defs/dns.additionals"
                 },
                 "query": {
-                    "$comment": "EVE DNS v2 style query logging; as of Suricata 8 only used in DNS records when v2 logging is enabled, not used for DNS records logged as part of an event.",
+                    "$comment":
+                            "EVE DNS v2 style query logging; as of Suricata 8 only used in DNS records when v2 logging is enabled, not used for DNS records logged as part of an event.",
                     "type": "array",
                     "minItems": 1,
                     "items": {
@@ -5185,6 +5186,11 @@
                                             "Number of packets dropped due to stream reassembly exception policy",
                                     "type": "integer"
                                 },
+                                "stream_urgent": {
+                                    "description":
+                                            "Number of packets dropped due to TCP urgent flag",
+                                    "type": "integer"
+                                },
                                 "nfq_error": {
                                     "description":
                                             "Number of packets dropped due to no NFQ verdict",
@@ -6072,7 +6078,8 @@
                             "type": "integer"
                         },
                         "get_used": {
-                            "description": "Number of reused flows from the hash table in case memcap was reached and spare pool was empty",
+                            "description":
+                                    "Number of reused flows from the hash table in case memcap was reached and spare pool was empty",
                             "type": "integer"
                         },
                         "get_used_eval": {
@@ -6125,7 +6132,8 @@
                             "type": "integer"
                         },
                         "tcp_reuse": {
-                            "description": "Number of TCP flows that were reused as they seemed to share the same flow tuple",
+                            "description":
+                                    "Number of TCP flows that were reused as they seemed to share the same flow tuple",
                             "type": "integer"
                         },
                         "total": {
@@ -6550,6 +6558,10 @@
                         "urg": {
                             "description": "Number of TCP packets with the urgent flag set",
                             "type": "integer"
+                        },
+                        "urgent_oob_data": {
+                            "description": "Number of OOB bytes tracked in TCP urgent handling",
+                            "type": "integer"
                         }
                     },
                     "additionalProperties": false

rules/stream-events.rules

@@ -109,5 +109,7 @@ alert tcp any any -> any any (msg:"SURICATA STREAM FIN SYN reuse"; stream-event:
 # Depth setting reached for a stream. Very common in normal traffic, so disable by default.
 #alert tcp any any -> any any (msg:"SURICATA STREAM reassembly depth reached"; stream-event:reassembly_depth_reached; classtype:protocol-command-decode; sid:2210062; rev:1;)
 
-# next sid 2210066
+alert tcp any any -> any any (msg:"SURICATA STREAM urgent OOB limit reached";  stream-event:reassembly_urgent_oob_limit_reached; classtype:protocol-command-decode; sid:2210066; rev:1;)
+
+# next sid 2210067
 

src/app-layer.c

@@ -735,6 +735,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
     /* If a gap notification, relay the notification on to the
      * app-layer if known. */
     if (flags & STREAM_GAP) {
+        SCLogDebug("GAP of size %u", data_len);
         if (alproto == ALPROTO_UNKNOWN) {
             StreamTcpSetStreamFlagAppProtoDetectionCompleted(*stream);
             SCLogDebug("ALPROTO_UNKNOWN flow %p, due to GAP in stream start", f);

src/decode-events.c

@@ -872,6 +872,10 @@ const struct DecodeEvents_ DEvents[] = {
             "stream.reassembly_insert_invalid",
             STREAM_REASSEMBLY_INSERT_INVALID,
     },
+    {
+            "stream.reassembly_urgent_oob_limit_reached",
+            STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED,
+    },
 
     /* ARP EVENTS */
     {

src/decode-events.h

@@ -296,6 +296,7 @@ enum {
     STREAM_REASSEMBLY_INSERT_MEMCAP,
     STREAM_REASSEMBLY_INSERT_LIMIT,
     STREAM_REASSEMBLY_INSERT_INVALID,
+    STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED,
 
     /* ARP EVENTS */
     ARP_PKT_TOO_SMALL,         /**< arp packet smaller than minimum size */

src/decode.c

@@ -889,6 +889,8 @@ const char *PacketDropReasonToString(enum PacketDropReason r)
             return "stream memcap";
         case PKT_DROP_REASON_STREAM_MIDSTREAM:
             return "stream midstream";
+        case PKT_DROP_REASON_STREAM_URG:
+            return "stream urgent";
         case PKT_DROP_REASON_STREAM_REASSEMBLY:
             return "stream reassembly";
         case PKT_DROP_REASON_APPLAYER_ERROR:
@@ -929,6 +931,8 @@ static const char *PacketDropReasonToJsonString(enum PacketDropReason r)
             return "ips.drop_reason.stream_memcap";
         case PKT_DROP_REASON_STREAM_MIDSTREAM:
             return "ips.drop_reason.stream_midstream";
+        case PKT_DROP_REASON_STREAM_URG:
+            return "ips.drop_reason.stream_urgent";
         case PKT_DROP_REASON_STREAM_REASSEMBLY:
             return "ips.drop_reason.stream_reassembly";
         case PKT_DROP_REASON_APPLAYER_ERROR:

src/decode.h

@@ -371,6 +371,7 @@ enum PacketDropReason {
     PKT_DROP_REASON_STREAM_MEMCAP,
     PKT_DROP_REASON_STREAM_MIDSTREAM,
     PKT_DROP_REASON_STREAM_REASSEMBLY,
+    PKT_DROP_REASON_STREAM_URG,
     PKT_DROP_REASON_NFQ_ERROR,    /**< no nfq verdict, must be error */
     PKT_DROP_REASON_INNER_PACKET, /**< drop issued by inner (tunnel) packet */
     PKT_DROP_REASON_MAX,

src/stream-tcp-list.c

@@ -632,8 +632,7 @@ static void StreamTcpSegmentAddPacketData(
  *  In case of error, this function returns the segment to the pool
  */
 int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
-        TcpStream *stream, TcpSegment *seg, Packet *p,
-        uint8_t *pkt_data, uint16_t pkt_datalen)
+        TcpStream *stream, TcpSegment *seg, Packet *p, uint8_t *pkt_data, uint16_t pkt_datalen)
 {
     SCEnter();
 

src/stream-tcp-private.h

@@ -288,6 +288,8 @@ typedef struct TcpSession_ {
     int8_t data_first_seen_dir;
     /** track all the tcp flags we've seen */
     uint8_t tcp_packet_flags;
+    uint16_t urg_offset_ts;            /**< SEQ offset from accepted OOB urg bytes */
+    uint16_t urg_offset_tc;            /**< SEQ offset from accepted OOB urg bytes */
     /* coccinelle: TcpSession:flags:STREAMTCP_FLAG */
     uint32_t flags;
     uint32_t reassembly_depth; /**< reassembly depth for the stream */

src/stream-tcp-reassemble.c

@@ -28,10 +28,12 @@
 
 #include "suricata-common.h"
 #include "suricata.h"
+#include "packet.h"
 #include "detect.h"
 #include "flow.h"
 #include "threads.h"
 #include "conf.h"
+#include "action-globals.h"
 
 #include "flow-util.h"
 
@@ -604,6 +606,15 @@ void StreamTcpReassembleFreeThreadCtx(TcpReassemblyThreadCtx *ra_ctx)
     SCReturn;
 }
 
+static void StreamTcpReassembleExceptionPolicyStatsIncr(
+        ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, enum ExceptionPolicy policy)
+{
+    uint16_t id = ra_ctx->counter_tcp_reas_eps.eps_id[policy];
+    if (likely(tv && id > 0)) {
+        StatsIncr(tv, id);
+    }
+}
+
 /**
  *  \brief check if stream in pkt direction has depth reached
  *
@@ -757,12 +768,65 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
         SCReturnInt(0);
     }
 
+    uint16_t *urg_offset;
+    if (PKT_IS_TOSERVER(p)) {
+        urg_offset = &ssn->urg_offset_ts;
+    } else {
+        urg_offset = &ssn->urg_offset_tc;
+    }
+
     const TCPHdr *tcph = PacketGetTCP(p);
+    /* segment sequence number, offset by previously accepted
+     * URG OOB data. */
+    uint32_t seg_seq = TCP_GET_RAW_SEQ(tcph) - (*urg_offset);
+    uint8_t urg_data = 0;
+
+    /* if stream_config.urgent_policy == TCP_STREAM_URGENT_DROP, we won't get here */
+    if (tcph->th_flags & TH_URG) {
+        const uint16_t urg_ptr = SCNtohs(tcph->th_urp);
+        if (urg_ptr > 0 && urg_ptr <= p->payload_len &&
+                (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB ||
+                        stream_config.urgent_policy == TCP_STREAM_URGENT_GAP)) {
+            /* track up to 64k out of band URG bytes. Fall back to inline
+             * when that budget is exceeded. */
+            if ((*urg_offset) < UINT16_MAX) {
+                if (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB)
+                    (*urg_offset)++;
+
+                if ((*urg_offset) == UINT16_MAX) {
+                    StreamTcpSetEvent(p, STREAM_REASSEMBLY_URGENT_OOB_LIMIT_REACHED);
+                }
+            } else {
+                /* OOB limit DROP is handled here */
+                if (stream_config.urgent_oob_limit_policy == TCP_STREAM_URGENT_DROP) {
+                    PacketDrop(p, ACTION_DROP, PKT_DROP_REASON_STREAM_URG);
+                    SCReturnInt(0);
+                }
+            }
+            urg_data = 1; /* only treat last 1 byte as out of band. */
+            if (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB) {
+                StatsIncr(tv, ra_ctx->counter_tcp_urgent_oob);
+            }
+
+            /* depending on hitting the OOB limit, update urg_data or not */
+            if (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB &&
+                    (*urg_offset) == UINT16_MAX &&
+                    stream_config.urgent_oob_limit_policy == TCP_STREAM_URGENT_INLINE) {
+                urg_data = 0;
+            } else {
+                if (urg_ptr == 1 && p->payload_len == 1) {
+                    SCLogDebug("no non-URG data");
+                    SCReturnInt(0);
+                }
+            }
+        }
+    }
+
+    const uint16_t payload_len = p->payload_len - urg_data;
 
     /* If we have reached the defined depth for either of the stream, then stop
        reassembling the TCP session */
-    uint32_t size =
-            StreamTcpReassembleCheckDepth(ssn, stream, TCP_GET_RAW_SEQ(tcph), p->payload_len);
+    uint32_t size = StreamTcpReassembleCheckDepth(ssn, stream, seg_seq, payload_len);
     SCLogDebug("ssn %p: check depth returned %"PRIu32, ssn, size);
 
     if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED) {
@@ -776,9 +840,9 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
         SCReturnInt(0);
     }
 
-    DEBUG_VALIDATE_BUG_ON(size > p->payload_len);
-    if (size > p->payload_len)
-        size = p->payload_len;
+    DEBUG_VALIDATE_BUG_ON(size > payload_len);
+    if (size > payload_len)
+        size = payload_len;
 
     TcpSegment *seg = StreamTcpGetSegment(tv, ra_ctx);
     if (seg == NULL) {
@@ -790,7 +854,8 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
 
     DEBUG_VALIDATE_BUG_ON(size > UINT16_MAX);
     TCP_SEG_LEN(seg) = (uint16_t)size;
-    seg->seq = TCP_GET_RAW_SEQ(tcph);
+    /* set SEQUENCE number, adjusted to any URG pointer offset */
+    seg->seq = seg_seq;
 
     /* HACK: for TFO SYN packets the seq for data starts at + 1 */
     if (TCP_HAS_TFO(p) && p->payload_len && (tcph->th_flags & TH_SYN))
@@ -804,8 +869,7 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
                 APPLAYER_PROTO_DETECTION_SKIPPED);
     }
 
-    int r = StreamTcpReassembleInsertSegment(
-            tv, ra_ctx, stream, seg, p, p->payload, p->payload_len);
+    int r = StreamTcpReassembleInsertSegment(tv, ra_ctx, stream, seg, p, p->payload, payload_len);
     if (r < 0) {
         if (r == -SC_ENOMEM) {
             ssn->flags |= STREAMTCP_FLAG_LOSSY_BE_LIBERAL;
@@ -1933,15 +1997,6 @@ static int StreamTcpReassembleHandleSegmentUpdateACK (ThreadVars *tv,
     SCReturnInt(0);
 }
 
-static void StreamTcpReassembleExceptionPolicyStatsIncr(
-        ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, enum ExceptionPolicy policy)
-{
-    uint16_t id = ra_ctx->counter_tcp_reas_eps.eps_id[policy];
-    if (likely(tv && id > 0)) {
-        StatsIncr(tv, id);
-    }
-}
-
 int StreamTcpReassembleHandleSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         TcpSession *ssn, TcpStream *stream, Packet *p)
 {

src/stream-tcp-reassemble.h

@@ -83,6 +83,9 @@ typedef struct TcpReassemblyThreadCtx_ {
 
     uint16_t counter_tcp_reass_data_normal_fail;
     uint16_t counter_tcp_reass_data_overlap_fail;
+
+    /** count OOB bytes */
+    uint16_t counter_tcp_urgent_oob;
 } TcpReassemblyThreadCtx;
 
 #define OS_POLICY_DEFAULT   OS_POLICY_BSD

src/stream-tcp.c

@@ -438,6 +438,17 @@ static inline bool StreamTcpInlineDropInvalid(void)
             && (stream_config.flags & STREAMTCP_INIT_FLAG_DROP_INVALID));
 }
 
+/** \internal
+ *  \brief See if stream engine is dropping URG packets in inline mode
+ *  \retval false no
+ *  \retval true yes
+ */
+static inline bool StreamTcpInlineDropUrg(void)
+{
+    return ((stream_config.flags & STREAMTCP_INIT_FLAG_INLINE) &&
+            stream_config.urgent_policy == TCP_STREAM_URGENT_DROP);
+}
+
 /* hack: stream random range code expects random values in range of 0-RAND_MAX,
  * but we can get both <0 and >RAND_MAX values from RandomGet
  */
@@ -452,6 +463,22 @@ static int RandomGetWrap(void)
     return r % RAND_MAX;
 }
 
+static const char *UrgentPolicyToString(enum TcpStreamUrgentHandling pol)
+{
+    switch (pol) {
+        case TCP_STREAM_URGENT_OOB:
+            return "oob";
+        case TCP_STREAM_URGENT_INLINE:
+            return "inline";
+        case TCP_STREAM_URGENT_DROP:
+            return "drop";
+        case TCP_STREAM_URGENT_GAP:
+            return "gap";
+    }
+    return NULL;
+}
+
+
 /** \brief          To initialize the stream global configuration data
  *
  *  \param  quiet   It tells the mode of operation, if it is true nothing will
@@ -601,6 +628,46 @@ void StreamTcpInitConfig(bool quiet)
         stream_config.flags |= STREAMTCP_INIT_FLAG_DROP_INVALID;
     }
 
+    const char *temp_urgpol = NULL;
+    if (ConfGet("stream.reassembly.urgent.policy", &temp_urgpol) == 1 && temp_urgpol != NULL) {
+        if (strcmp(temp_urgpol, "inline") == 0) {
+            stream_config.urgent_policy = TCP_STREAM_URGENT_INLINE;
+        } else if (strcmp(temp_urgpol, "drop") == 0) {
+            stream_config.urgent_policy = TCP_STREAM_URGENT_DROP;
+        } else if (strcmp(temp_urgpol, "oob") == 0) {
+            stream_config.urgent_policy = TCP_STREAM_URGENT_OOB;
+        } else if (strcmp(temp_urgpol, "gap") == 0) {
+            stream_config.urgent_policy = TCP_STREAM_URGENT_GAP;
+        } else {
+            FatalError("stream.reassembly.urgent.policy: invalid value '%s'", temp_urgpol);
+        }
+    } else {
+        stream_config.urgent_policy = TCP_STREAM_URGENT_DEFAULT;
+    }
+    if (!quiet) {
+        SCLogConfig("stream.reassembly.urgent.policy\": %s", UrgentPolicyToString(stream_config.urgent_policy));
+    }
+    if (stream_config.urgent_policy == TCP_STREAM_URGENT_OOB) {
+        const char *temp_urgoobpol = NULL;
+        if (ConfGet("stream.reassembly.urgent.oob-limit-policy", &temp_urgoobpol) == 1 &&
+                temp_urgoobpol != NULL) {
+            if (strcmp(temp_urgoobpol, "inline") == 0) {
+                stream_config.urgent_oob_limit_policy = TCP_STREAM_URGENT_INLINE;
+            } else if (strcmp(temp_urgoobpol, "drop") == 0) {
+                stream_config.urgent_oob_limit_policy = TCP_STREAM_URGENT_DROP;
+            } else if (strcmp(temp_urgoobpol, "gap") == 0) {
+                stream_config.urgent_oob_limit_policy = TCP_STREAM_URGENT_GAP;
+            } else {
+                FatalError("stream.reassembly.urgent.oob-limit-policy: invalid value '%s'", temp_urgoobpol);
+            }
+        } else {
+            stream_config.urgent_oob_limit_policy = TCP_STREAM_URGENT_DEFAULT;
+        }
+        if (!quiet) {
+            SCLogConfig("stream.reassembly.urgent.oob-limit-policy\": %s", UrgentPolicyToString(stream_config.urgent_oob_limit_policy));
+        }
+    }
+
     if ((ConfGetInt("stream.max-syn-queued", &value)) == 1) {
         if (value >= 0 && value <= 255) {
             stream_config.max_syn_queued = (uint8_t)value;
@@ -5532,6 +5599,12 @@ int StreamTcpPacket (ThreadVars *tv, Packet *p, StreamTcpThread *stt,
         StreamTcpSetEvent(p, STREAM_PKT_BROKEN_ACK);
     }
 
+    if ((tcph->th_flags & TH_URG) && StreamTcpInlineDropUrg()) {
+        PacketDrop(p, ACTION_DROP, PKT_DROP_REASON_STREAM_URG);
+        SCLogDebug("dropping urgent packet");
+        SCReturnInt(0);
+    }
+
     /* If we are on IPS mode, and got a drop action triggered from
      * the IP only module, or from a reassembled msg and/or from an
      * applayer detection, then drop the rest of the packets of the
@@ -5996,6 +6069,7 @@ TmEcode StreamTcpThreadInit(ThreadVars *tv, void *initdata, void **data)
 
     stt->ra_ctx->counter_tcp_reass_data_normal_fail = StatsRegisterCounter("tcp.insert_data_normal_fail", tv);
     stt->ra_ctx->counter_tcp_reass_data_overlap_fail = StatsRegisterCounter("tcp.insert_data_overlap_fail", tv);
+    stt->ra_ctx->counter_tcp_urgent_oob = StatsRegisterCounter("tcp.urgent_oob_data", tv);
 
     SCLogDebug("StreamTcp thread specific ctx online at %p, reassembly ctx %p",
                 stt, stt->ra_ctx);

src/stream-tcp.h

@@ -39,6 +39,16 @@
 #define STREAMTCP_INIT_FLAG_DROP_INVALID           BIT_U8(1)
 #define STREAMTCP_INIT_FLAG_BYPASS                 BIT_U8(2)
 #define STREAMTCP_INIT_FLAG_INLINE                 BIT_U8(3)
+/** flag to drop packets with URG flag set */
+#define STREAMTCP_INIT_FLAG_DROP_URG BIT_U8(4)
+
+enum TcpStreamUrgentHandling {
+    TCP_STREAM_URGENT_INLINE, /**< treat as inline data */
+#define TCP_STREAM_URGENT_DEFAULT TCP_STREAM_URGENT_INLINE
+    TCP_STREAM_URGENT_DROP, /**< drop TCP packet with URG flag */
+    TCP_STREAM_URGENT_OOB,  /**< treat 1 byte of URG data as OOB */
+    TCP_STREAM_URGENT_GAP,  /**< treat 1 byte of URG data as GAP */
+};
 
 /*global flow data*/
 typedef struct TcpStreamCnf_ {
@@ -70,6 +80,8 @@ typedef struct TcpStreamCnf_ {
     enum ExceptionPolicy ssn_memcap_policy;
     enum ExceptionPolicy reassembly_memcap_policy;
     enum ExceptionPolicy midstream_policy;
+    enum TcpStreamUrgentHandling urgent_policy;
+    enum TcpStreamUrgentHandling urgent_oob_limit_policy;
 
     /* default to "LINUX" timestamp behavior if true*/
     bool liberal_timestamps;

suricata.yaml.in

@@ -1615,6 +1615,9 @@ stream:
   #midstream-policy: ignore
   inline: auto                  # auto will use inline mode in IPS mode, yes or no set it statically
   reassembly:
+    urgent:
+      policy: oob              # drop, inline, oob (1 byte, see RFC 6093, 3.1), gap
+      oob-limit-policy: drop
     memcap: 256 MiB
     #memcap-policy: ignore
     depth: 1 MiB                # reassemble 1 MiB into a stream