stream: conditionally add packet header to segment

This patch optionally adds packet header to the TCP segment
and update the for each segment function by changing the
callback.

This patch is based on the work by Scott Jordan <scottfgjordan@gmail.com>

src/alert-debuglog.c

@@ -137,7 +137,8 @@ static void AlertDebugLogPktVars(AlertDebugLogThread *aft, const Packet *p)
 
 /** \todo doc
  * assume we have aft lock */
-static int AlertDebugPrintStreamSegmentCallback(const Packet *p, void *data, const uint8_t *buf, uint32_t buflen)
+static int AlertDebugPrintStreamSegmentCallback(
+        const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
 {
     AlertDebugLogThread *aft = (AlertDebugLogThread *)data;
 
@@ -293,9 +294,9 @@ static TmEcode AlertDebugLogger(ThreadVars *tv, const Packet *p, void *thread_da
             /* IDS mode reverse the data */
             /** \todo improve the order selection policy */
             if (p->flowflags & FLOW_PKT_TOSERVER) {
-                flag = FLOW_PKT_TOCLIENT;
+                flag = STREAM_DUMP_TOCLIENT;
             } else {
-                flag = FLOW_PKT_TOSERVER;
+                flag = STREAM_DUMP_TOSERVER;
             }
             ret = StreamSegmentForEach((const Packet *)p, flag,
                                  AlertDebugPrintStreamSegmentCallback,

src/output-json-alert.c

@@ -123,7 +123,8 @@ typedef struct JsonAlertLogThread_ {
 
 /* Callback function to pack payload contents from a stream into a buffer
  * so we can report them in JSON output. */
-static int AlertJsonDumpStreamSegmentCallback(const Packet *p, void *data, const uint8_t *buf, uint32_t buflen)
+static int AlertJsonDumpStreamSegmentCallback(
+        const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen)
 {
     MemBuffer *payload = (MemBuffer *)data;
     MemBufferWriteRaw(payload, buf, buflen);
@@ -731,9 +732,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
                 MemBufferReset(payload);
 
                 if (p->flowflags & FLOW_PKT_TOSERVER) {
-                    flag = FLOW_PKT_TOCLIENT;
+                    flag = STREAM_DUMP_TOCLIENT;
                 } else {
-                    flag = FLOW_PKT_TOSERVER;
+                    flag = STREAM_DUMP_TOSERVER;
                 }
 
                 StreamSegmentForEach((const Packet *)p, flag,

src/stream-tcp-list.c

@@ -562,6 +562,58 @@ static int DoHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
     return 0;
 }
 
+/**
+ * \brief Adds the following information to the TcpSegment from the current
+ *  packet being processed: time values, packet length, and the
+ *  header data of the packet. This information is added to the TcpSegment so
+ *  that it can be used in pcap capturing (log-pcap-stream) to dump the tcp
+ *  session at the beginning of the pcap capture.
+ * \param seg TcpSegment where information is being stored.
+ * \param p Packet being processed.
+ * \param tv Thread-specific variables.
+ * \param ra_ctx TcpReassembly thread-specific variables
+ */
+static void StreamTcpSegmentAddPacketData(
+        TcpSegment *seg, Packet *p, ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx)
+{
+    if (seg->pcap_hdr_storage == NULL || seg->pcap_hdr_storage->pkt_hdr == NULL) {
+        return;
+    }
+
+    /* FIXME we need to address pseudo packet */
+
+    if (GET_PKT_DATA(p) != NULL && GET_PKT_LEN(p) > p->payload_len) {
+        seg->pcap_hdr_storage->ts.tv_sec = p->ts.tv_sec;
+        seg->pcap_hdr_storage->ts.tv_usec = p->ts.tv_usec;
+        seg->pcap_hdr_storage->pktlen = GET_PKT_LEN(p) - p->payload_len;
+        /*
+         * pkt_hdr members are initially allocated 64 bytes of memory. Thus,
+         * need to check that this is sufficient and allocate more memory if
+         * not.
+         */
+        if (GET_PKT_LEN(p) - p->payload_len > seg->pcap_hdr_storage->alloclen) {
+            uint8_t *tmp_pkt_hdr =
+                    SCRealloc(seg->pcap_hdr_storage->pkt_hdr, GET_PKT_LEN(p) - p->payload_len);
+            if (tmp_pkt_hdr == NULL) {
+                SCLogDebug("Failed to realloc");
+                seg->pcap_hdr_storage->ts.tv_sec = 0;
+                seg->pcap_hdr_storage->ts.tv_usec = 0;
+                seg->pcap_hdr_storage->pktlen = 0;
+                return;
+            } else {
+                seg->pcap_hdr_storage->pkt_hdr = tmp_pkt_hdr;
+                seg->pcap_hdr_storage->alloclen = GET_PKT_LEN(p) - p->payload_len;
+            }
+        }
+        memcpy(seg->pcap_hdr_storage->pkt_hdr, GET_PKT_DATA(p),
+                (size_t)GET_PKT_LEN(p) - p->payload_len);
+    } else {
+        seg->pcap_hdr_storage->ts.tv_sec = 0;
+        seg->pcap_hdr_storage->ts.tv_usec = 0;
+        seg->pcap_hdr_storage->pktlen = 0;
+    }
+}
+
 /**
  *  \return 0 ok
  *  \return -1 segment not inserted due to memcap issue
@@ -586,6 +638,9 @@ int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_
         StreamTcpSegmentReturntoPool(seg);
         SCReturnInt(-1);
     }
+    if (IsTcpSessionDumpingEnabled()) {
+        StreamTcpSegmentAddPacketData(seg, p, tv, ra_ctx);
+    }
 
     if (likely(r == 0)) {
         /* no overlap, straight data insert */

src/stream-tcp-private.h

@@ -58,12 +58,27 @@ int TcpSackCompare(struct StreamTcpSackRecord *a, struct StreamTcpSackRecord *b)
 RB_HEAD(TCPSACK, StreamTcpSackRecord);
 RB_PROTOTYPE(TCPSACK, StreamTcpSackRecord, rb, TcpSackCompare);
 
+#define TCPSEG_PKT_HDR_DEFAULT_SIZE 64
+
+/*
+ * Struct to add the additional information required to use TcpSegments to dump
+ * a packet capture to file with the stream-pcap-log output option. This is only
+ * used if the session-dump option is enabled.
+ */
+typedef struct TcpSegmentPcapHdrStorage_ {
+    struct timeval ts;
+    uint32_t pktlen;
+    uint32_t alloclen;
+    uint8_t *pkt_hdr;
+} TcpSegmentPcapHdrStorage;
+
 typedef struct TcpSegment {
     PoolThreadReserved res;
     uint16_t payload_len;       /**< actual size of the payload */
     uint32_t seq;
     RB_ENTRY(TcpSegment) __attribute__((__packed__)) rb;
     StreamingBufferSegment sbseg;
+    TcpSegmentPcapHdrStorage *pcap_hdr_storage;
 } __attribute__((__packed__)) TcpSegment;
 
 /** \brief compare function for the Segment tree

src/stream-tcp-reassemble.c

@@ -81,6 +81,18 @@ static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER;
 /* Memory use counter */
 SC_ATOMIC_DECLARE(uint64_t, ra_memuse);
 
+static int g_tcp_session_dump_enabled = 0;
+
+inline bool IsTcpSessionDumpingEnabled(void)
+{
+    return g_tcp_session_dump_enabled == 1;
+}
+
+void EnableTcpSessionDumping(void)
+{
+    g_tcp_session_dump_enabled = 1;
+}
+
 /* prototypes */
 TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *);
 void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
@@ -251,19 +263,64 @@ static void *TcpSegmentPoolAlloc(void)
     seg = SCMalloc(sizeof (TcpSegment));
     if (unlikely(seg == NULL))
         return NULL;
+
+    if (IsTcpSessionDumpingEnabled()) {
+        uint32_t memuse =
+                sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
+        if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
+            SCFree(seg);
+            return NULL;
+        }
+
+        seg->pcap_hdr_storage = SCCalloc(1, sizeof(TcpSegmentPcapHdrStorage));
+        if (seg->pcap_hdr_storage == NULL) {
+            SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate memory for "
+                                         "TcpSegmentPcapHdrStorage");
+            SCFree(seg);
+            return NULL;
+        } else {
+            seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
+            seg->pcap_hdr_storage->pkt_hdr =
+                    SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE);
+            if (seg->pcap_hdr_storage->pkt_hdr == NULL) {
+                SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate memory for "
+                                             "packet header data within "
+                                             "TcpSegmentPcapHdrStorage");
+                SCFree(seg->pcap_hdr_storage);
+                SCFree(seg);
+                return NULL;
+            }
+        }
+    } else {
+        seg->pcap_hdr_storage = NULL;
+    }
+
     return seg;
 }
 
 static int TcpSegmentPoolInit(void *data, void *initdata)
 {
     TcpSegment *seg = (TcpSegment *) data;
+    TcpSegmentPcapHdrStorage *pcap_hdr;
+
+    pcap_hdr = seg->pcap_hdr_storage;
 
     /* do this before the can bail, so TcpSegmentPoolCleanup
      * won't have uninitialized memory to consider. */
     memset(seg, 0, sizeof (TcpSegment));
 
-    if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
-        return 0;
+    if (IsTcpSessionDumpingEnabled()) {
+        uint32_t memuse =
+                sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE;
+        seg->pcap_hdr_storage = pcap_hdr;
+        if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) {
+            return 0;
+        }
+        StreamTcpReassembleIncrMemuse(memuse);
+    } else {
+        if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) {
+            return 0;
+        }
     }
 
 #ifdef DEBUG
@@ -284,6 +341,17 @@ static void TcpSegmentPoolCleanup(void *ptr)
     if (ptr == NULL)
         return;
 
+    TcpSegment *seg = (TcpSegment *)ptr;
+    if (seg && seg->pcap_hdr_storage) {
+        if (seg->pcap_hdr_storage->pkt_hdr) {
+            SCFree(seg->pcap_hdr_storage->pkt_hdr);
+            StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen);
+        }
+        SCFree(seg->pcap_hdr_storage);
+        seg->pcap_hdr_storage = NULL;
+        StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage));
+    }
+
     StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment));
 
 #ifdef DEBUG
@@ -305,6 +373,10 @@ void StreamTcpSegmentReturntoPool(TcpSegment *seg)
     if (seg == NULL)
         return;
 
+    if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) {
+        seg->pcap_hdr_storage->pktlen = 0;
+    }
+
     PoolThreadReturn(segment_thread_pool, seg);
 }
 

src/stream-tcp-reassemble.h

@@ -130,6 +130,9 @@ int StreamTcpCheckStreamContents(uint8_t *, uint16_t , TcpStream *);
 bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p);
 void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth);
 
+bool IsTcpSessionDumpingEnabled(void);
+void EnableTcpSessionDumping(void);
+
 static inline bool STREAM_LASTACK_GT_BASESEQ(const TcpStream *stream)
 {
     /* last ack not yet initialized */

src/stream-tcp.c

@@ -6319,7 +6319,7 @@ int StreamTcpSegmentForEach(const Packet *p, uint8_t flag, StreamSegmentCallback
         return 0;
     }
 
-    if (flag & FLOW_PKT_TOSERVER) {
+    if (flag & STREAM_DUMP_TOSERVER) {
         stream = &(ssn->server);
     } else {
         stream = &(ssn->client);
@@ -6343,7 +6343,7 @@ int StreamTcpSegmentForEach(const Packet *p, uint8_t flag, StreamSegmentCallback
         uint32_t seg_datalen;
         StreamingBufferSegmentGetData(&stream->sb, &seg->sbseg, &seg_data, &seg_datalen);
 
-        int ret = CallbackFunc(p, data, seg_data, seg_datalen);
+        int ret = CallbackFunc(p, seg, data, seg_data, seg_datalen);
         if (ret != 1) {
             SCLogDebug("Callback function has failed");
             return -1;

src/stream.h

@@ -25,10 +25,16 @@
 #define __STREAM_H__
 
 #include "flow.h"
+#include "stream-tcp-private.h"
 
 #define STREAM_FLAGS_FOR_PACKET(p) PKT_IS_TOSERVER((p)) ? STREAM_TOSERVER : STREAM_TOCLIENT
 
-typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t);
+#define STREAM_DUMP_TOCLIENT BIT_U8(1)
+#define STREAM_DUMP_TOSERVER BIT_U8(2)
+#define STREAM_DUMP_HEADERS  BIT_U8(3)
+
+typedef int (*StreamSegmentCallback)(
+        const Packet *, TcpSegment *, void *, const uint8_t *, uint32_t);
 int StreamSegmentForEach(const Packet *p, uint8_t flag,
                          StreamSegmentCallback CallbackFunc,
                          void *data);