From b416a4455c9b367cdd308792ea0c26e412a7f5e9 Mon Sep 17 00:00:00 2001 From: Eric Leblond Date: Sat, 30 Jan 2021 21:57:32 +0100 Subject: [PATCH] stream: conditionally add packet header to segment This patch optionally adds packet header to the TCP segment and update the for each segment function by changing the callback. This patch is based on the work by Scott Jordan --- src/alert-debuglog.c | 7 ++-- src/output-json-alert.c | 7 ++-- src/stream-tcp-list.c | 55 +++++++++++++++++++++++++++ src/stream-tcp-private.h | 15 ++++++++ src/stream-tcp-reassemble.c | 76 ++++++++++++++++++++++++++++++++++++- src/stream-tcp-reassemble.h | 3 ++ src/stream-tcp.c | 4 +- src/stream.h | 8 +++- 8 files changed, 164 insertions(+), 11 deletions(-) diff --git a/src/alert-debuglog.c b/src/alert-debuglog.c index a272740dd0..9dbb193d88 100644 --- a/src/alert-debuglog.c +++ b/src/alert-debuglog.c @@ -137,7 +137,8 @@ static void AlertDebugLogPktVars(AlertDebugLogThread *aft, const Packet *p) /** \todo doc * assume we have aft lock */ -static int AlertDebugPrintStreamSegmentCallback(const Packet *p, void *data, const uint8_t *buf, uint32_t buflen) +static int AlertDebugPrintStreamSegmentCallback( + const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen) { AlertDebugLogThread *aft = (AlertDebugLogThread *)data; @@ -293,9 +294,9 @@ static TmEcode AlertDebugLogger(ThreadVars *tv, const Packet *p, void *thread_da /* IDS mode reverse the data */ /** \todo improve the order selection policy */ if (p->flowflags & FLOW_PKT_TOSERVER) { - flag = FLOW_PKT_TOCLIENT; + flag = STREAM_DUMP_TOCLIENT; } else { - flag = FLOW_PKT_TOSERVER; + flag = STREAM_DUMP_TOSERVER; } ret = StreamSegmentForEach((const Packet *)p, flag, AlertDebugPrintStreamSegmentCallback, diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 3f32692d96..6e90f8ab06 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -123,7 +123,8 @@ typedef struct JsonAlertLogThread_ { /* Callback function to pack payload contents from a stream into a buffer * so we can report them in JSON output. */ -static int AlertJsonDumpStreamSegmentCallback(const Packet *p, void *data, const uint8_t *buf, uint32_t buflen) +static int AlertJsonDumpStreamSegmentCallback( + const Packet *p, TcpSegment *seg, void *data, const uint8_t *buf, uint32_t buflen) { MemBuffer *payload = (MemBuffer *)data; MemBufferWriteRaw(payload, buf, buflen); @@ -731,9 +732,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) MemBufferReset(payload); if (p->flowflags & FLOW_PKT_TOSERVER) { - flag = FLOW_PKT_TOCLIENT; + flag = STREAM_DUMP_TOCLIENT; } else { - flag = FLOW_PKT_TOSERVER; + flag = STREAM_DUMP_TOSERVER; } StreamSegmentForEach((const Packet *)p, flag, diff --git a/src/stream-tcp-list.c b/src/stream-tcp-list.c index eb23b1db3f..c02a235213 100644 --- a/src/stream-tcp-list.c +++ b/src/stream-tcp-list.c @@ -562,6 +562,58 @@ static int DoHandleData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, return 0; } +/** + * \brief Adds the following information to the TcpSegment from the current + * packet being processed: time values, packet length, and the + * header data of the packet. This information is added to the TcpSegment so + * that it can be used in pcap capturing (log-pcap-stream) to dump the tcp + * session at the beginning of the pcap capture. + * \param seg TcpSegment where information is being stored. + * \param p Packet being processed. + * \param tv Thread-specific variables. + * \param ra_ctx TcpReassembly thread-specific variables + */ +static void StreamTcpSegmentAddPacketData( + TcpSegment *seg, Packet *p, ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx) +{ + if (seg->pcap_hdr_storage == NULL || seg->pcap_hdr_storage->pkt_hdr == NULL) { + return; + } + + /* FIXME we need to address pseudo packet */ + + if (GET_PKT_DATA(p) != NULL && GET_PKT_LEN(p) > p->payload_len) { + seg->pcap_hdr_storage->ts.tv_sec = p->ts.tv_sec; + seg->pcap_hdr_storage->ts.tv_usec = p->ts.tv_usec; + seg->pcap_hdr_storage->pktlen = GET_PKT_LEN(p) - p->payload_len; + /* + * pkt_hdr members are initially allocated 64 bytes of memory. Thus, + * need to check that this is sufficient and allocate more memory if + * not. + */ + if (GET_PKT_LEN(p) - p->payload_len > seg->pcap_hdr_storage->alloclen) { + uint8_t *tmp_pkt_hdr = + SCRealloc(seg->pcap_hdr_storage->pkt_hdr, GET_PKT_LEN(p) - p->payload_len); + if (tmp_pkt_hdr == NULL) { + SCLogDebug("Failed to realloc"); + seg->pcap_hdr_storage->ts.tv_sec = 0; + seg->pcap_hdr_storage->ts.tv_usec = 0; + seg->pcap_hdr_storage->pktlen = 0; + return; + } else { + seg->pcap_hdr_storage->pkt_hdr = tmp_pkt_hdr; + seg->pcap_hdr_storage->alloclen = GET_PKT_LEN(p) - p->payload_len; + } + } + memcpy(seg->pcap_hdr_storage->pkt_hdr, GET_PKT_DATA(p), + (size_t)GET_PKT_LEN(p) - p->payload_len); + } else { + seg->pcap_hdr_storage->ts.tv_sec = 0; + seg->pcap_hdr_storage->ts.tv_usec = 0; + seg->pcap_hdr_storage->pktlen = 0; + } +} + /** * \return 0 ok * \return -1 segment not inserted due to memcap issue @@ -586,6 +638,9 @@ int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ StreamTcpSegmentReturntoPool(seg); SCReturnInt(-1); } + if (IsTcpSessionDumpingEnabled()) { + StreamTcpSegmentAddPacketData(seg, p, tv, ra_ctx); + } if (likely(r == 0)) { /* no overlap, straight data insert */ diff --git a/src/stream-tcp-private.h b/src/stream-tcp-private.h index b229edfae8..8f35c8264b 100644 --- a/src/stream-tcp-private.h +++ b/src/stream-tcp-private.h @@ -58,12 +58,27 @@ int TcpSackCompare(struct StreamTcpSackRecord *a, struct StreamTcpSackRecord *b) RB_HEAD(TCPSACK, StreamTcpSackRecord); RB_PROTOTYPE(TCPSACK, StreamTcpSackRecord, rb, TcpSackCompare); +#define TCPSEG_PKT_HDR_DEFAULT_SIZE 64 + +/* + * Struct to add the additional information required to use TcpSegments to dump + * a packet capture to file with the stream-pcap-log output option. This is only + * used if the session-dump option is enabled. + */ +typedef struct TcpSegmentPcapHdrStorage_ { + struct timeval ts; + uint32_t pktlen; + uint32_t alloclen; + uint8_t *pkt_hdr; +} TcpSegmentPcapHdrStorage; + typedef struct TcpSegment { PoolThreadReserved res; uint16_t payload_len; /**< actual size of the payload */ uint32_t seq; RB_ENTRY(TcpSegment) __attribute__((__packed__)) rb; StreamingBufferSegment sbseg; + TcpSegmentPcapHdrStorage *pcap_hdr_storage; } __attribute__((__packed__)) TcpSegment; /** \brief compare function for the Segment tree diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index 37d5670047..ae8f7a80ec 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -81,6 +81,18 @@ static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER; /* Memory use counter */ SC_ATOMIC_DECLARE(uint64_t, ra_memuse); +static int g_tcp_session_dump_enabled = 0; + +inline bool IsTcpSessionDumpingEnabled(void) +{ + return g_tcp_session_dump_enabled == 1; +} + +void EnableTcpSessionDumping(void) +{ + g_tcp_session_dump_enabled = 1; +} + /* prototypes */ TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *); void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t); @@ -251,19 +263,64 @@ static void *TcpSegmentPoolAlloc(void) seg = SCMalloc(sizeof (TcpSegment)); if (unlikely(seg == NULL)) return NULL; + + if (IsTcpSessionDumpingEnabled()) { + uint32_t memuse = + sizeof(TcpSegmentPcapHdrStorage) + sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE; + if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) { + SCFree(seg); + return NULL; + } + + seg->pcap_hdr_storage = SCCalloc(1, sizeof(TcpSegmentPcapHdrStorage)); + if (seg->pcap_hdr_storage == NULL) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate memory for " + "TcpSegmentPcapHdrStorage"); + SCFree(seg); + return NULL; + } else { + seg->pcap_hdr_storage->alloclen = sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE; + seg->pcap_hdr_storage->pkt_hdr = + SCCalloc(1, sizeof(uint8_t) * TCPSEG_PKT_HDR_DEFAULT_SIZE); + if (seg->pcap_hdr_storage->pkt_hdr == NULL) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate memory for " + "packet header data within " + "TcpSegmentPcapHdrStorage"); + SCFree(seg->pcap_hdr_storage); + SCFree(seg); + return NULL; + } + } + } else { + seg->pcap_hdr_storage = NULL; + } + return seg; } static int TcpSegmentPoolInit(void *data, void *initdata) { TcpSegment *seg = (TcpSegment *) data; + TcpSegmentPcapHdrStorage *pcap_hdr; + + pcap_hdr = seg->pcap_hdr_storage; /* do this before the can bail, so TcpSegmentPoolCleanup * won't have uninitialized memory to consider. */ memset(seg, 0, sizeof (TcpSegment)); - if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) { - return 0; + if (IsTcpSessionDumpingEnabled()) { + uint32_t memuse = + sizeof(TcpSegmentPcapHdrStorage) + sizeof(char) * TCPSEG_PKT_HDR_DEFAULT_SIZE; + seg->pcap_hdr_storage = pcap_hdr; + if (StreamTcpReassembleCheckMemcap(sizeof(TcpSegment) + memuse) == 0) { + return 0; + } + StreamTcpReassembleIncrMemuse(memuse); + } else { + if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment)) == 0) { + return 0; + } } #ifdef DEBUG @@ -284,6 +341,17 @@ static void TcpSegmentPoolCleanup(void *ptr) if (ptr == NULL) return; + TcpSegment *seg = (TcpSegment *)ptr; + if (seg && seg->pcap_hdr_storage) { + if (seg->pcap_hdr_storage->pkt_hdr) { + SCFree(seg->pcap_hdr_storage->pkt_hdr); + StreamTcpReassembleDecrMemuse(seg->pcap_hdr_storage->alloclen); + } + SCFree(seg->pcap_hdr_storage); + seg->pcap_hdr_storage = NULL; + StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegmentPcapHdrStorage)); + } + StreamTcpReassembleDecrMemuse((uint32_t)sizeof(TcpSegment)); #ifdef DEBUG @@ -305,6 +373,10 @@ void StreamTcpSegmentReturntoPool(TcpSegment *seg) if (seg == NULL) return; + if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) { + seg->pcap_hdr_storage->pktlen = 0; + } + PoolThreadReturn(segment_thread_pool, seg); } diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h index 02911bd820..d62e27b045 100644 --- a/src/stream-tcp-reassemble.h +++ b/src/stream-tcp-reassemble.h @@ -130,6 +130,9 @@ int StreamTcpCheckStreamContents(uint8_t *, uint16_t , TcpStream *); bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p); void StreamTcpReassemblySetMinInspectDepth(TcpSession *ssn, int direction, uint32_t depth); +bool IsTcpSessionDumpingEnabled(void); +void EnableTcpSessionDumping(void); + static inline bool STREAM_LASTACK_GT_BASESEQ(const TcpStream *stream) { /* last ack not yet initialized */ diff --git a/src/stream-tcp.c b/src/stream-tcp.c index 3a91cb9e6f..42c120fd9b 100644 --- a/src/stream-tcp.c +++ b/src/stream-tcp.c @@ -6319,7 +6319,7 @@ int StreamTcpSegmentForEach(const Packet *p, uint8_t flag, StreamSegmentCallback return 0; } - if (flag & FLOW_PKT_TOSERVER) { + if (flag & STREAM_DUMP_TOSERVER) { stream = &(ssn->server); } else { stream = &(ssn->client); @@ -6343,7 +6343,7 @@ int StreamTcpSegmentForEach(const Packet *p, uint8_t flag, StreamSegmentCallback uint32_t seg_datalen; StreamingBufferSegmentGetData(&stream->sb, &seg->sbseg, &seg_data, &seg_datalen); - int ret = CallbackFunc(p, data, seg_data, seg_datalen); + int ret = CallbackFunc(p, seg, data, seg_data, seg_datalen); if (ret != 1) { SCLogDebug("Callback function has failed"); return -1; diff --git a/src/stream.h b/src/stream.h index 78fbfcf1fe..e8f2cad7e0 100644 --- a/src/stream.h +++ b/src/stream.h @@ -25,10 +25,16 @@ #define __STREAM_H__ #include "flow.h" +#include "stream-tcp-private.h" #define STREAM_FLAGS_FOR_PACKET(p) PKT_IS_TOSERVER((p)) ? STREAM_TOSERVER : STREAM_TOCLIENT -typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t); +#define STREAM_DUMP_TOCLIENT BIT_U8(1) +#define STREAM_DUMP_TOSERVER BIT_U8(2) +#define STREAM_DUMP_HEADERS BIT_U8(3) + +typedef int (*StreamSegmentCallback)( + const Packet *, TcpSegment *, void *, const uint8_t *, uint32_t); int StreamSegmentForEach(const Packet *p, uint8_t flag, StreamSegmentCallback CallbackFunc, void *data);