diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index ffe473093d..baf07c3c64 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -1689,19 +1689,9 @@ static void StreamTcpSetupMsg(TcpSession *ssn, TcpStream *stream, Packet *p, if ((!StreamTcpInlineMode() && p->flowflags & FLOW_PKT_TOSERVER) || ( StreamTcpInlineMode() && p->flowflags & FLOW_PKT_TOCLIENT)) { - COPY_ADDRESS(&p->flow->src,&smsg->data.src_ip); - COPY_ADDRESS(&p->flow->dst,&smsg->data.dst_ip); - COPY_PORT(p->flow->sp,smsg->data.src_port); - COPY_PORT(p->flow->dp,smsg->data.dst_port); - smsg->flags |= STREAM_TOCLIENT; SCLogDebug("stream mesage is to_client"); } else { - COPY_ADDRESS(&p->flow->dst,&smsg->data.src_ip); - COPY_ADDRESS(&p->flow->src,&smsg->data.dst_ip); - COPY_PORT(p->flow->dp,smsg->data.src_port); - COPY_PORT(p->flow->sp,smsg->data.dst_port); - smsg->flags |= STREAM_TOSERVER; SCLogDebug("stream mesage is to_server"); } @@ -1737,26 +1727,28 @@ static int StreamTcpReassembleRawCheckLimit(TcpSession *ssn, TcpStream *stream, /* check if we have enough data to send to L7 */ if (p->flowflags & FLOW_PKT_TOCLIENT) { SCLogDebug("StreamMsgQueueGetMinChunkLen(STREAM_TOSERVER) %"PRIu32, - StreamMsgQueueGetMinChunkLen(STREAM_TOSERVER)); + StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOSERVER)); - if (StreamMsgQueueGetMinChunkLen(STREAM_TOSERVER) > + if (StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOSERVER) > (stream->last_ack - stream->ra_raw_base_seq)) { SCLogDebug("toserver min chunk len not yet reached: " - "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", len " + "last_ack %"PRIu32", ra_raw_base_seq %"PRIu32", %"PRIu32" < " "%"PRIu32"", stream->last_ack, stream->ra_raw_base_seq, - StreamMsgQueueGetMinChunkLen(STREAM_TOSERVER)); + (stream->last_ack - stream->ra_raw_base_seq), + StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOSERVER)); SCReturnInt(0); } } else { SCLogDebug("StreamMsgQueueGetMinChunkLen(STREAM_TOCLIENT) %"PRIu32, - StreamMsgQueueGetMinChunkLen(STREAM_TOCLIENT)); + StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOCLIENT)); - if (StreamMsgQueueGetMinChunkLen(STREAM_TOCLIENT) > + if (StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOCLIENT) > (stream->last_ack - stream->ra_raw_base_seq)) { SCLogDebug("toclient min chunk len not yet reached: " - "last_ack %"PRIu32", ra_base_seq %"PRIu32", len " + "last_ack %"PRIu32", ra_base_seq %"PRIu32", %"PRIu32" < " "%"PRIu32"", stream->last_ack, stream->ra_raw_base_seq, - StreamMsgQueueGetMinChunkLen(STREAM_TOCLIENT)); + (stream->last_ack - stream->ra_raw_base_seq), + StreamMsgQueueGetMinChunkLen(FLOW_PKT_TOCLIENT)); SCReturnInt(0); } } @@ -2595,8 +2587,7 @@ static int StreamTcpReassembleAppLayer (TcpReassemblyThreadCtx *ra_ctx, * and state is beyond established, we send an empty msg */ TcpSegment *seg_tail = stream->seg_list_tail; if (seg_tail == NULL || - (seg_tail->flags & SEGMENTTCP_FLAG_RAW_PROCESSED && - seg_tail->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) + (seg_tail->flags & SEGMENTTCP_FLAG_APPLAYER_PROCESSED)) { /* send an empty EOF msg if we have no segments but TCP state * is beyond ESTABLISHED */ @@ -6056,10 +6047,6 @@ static int StreamTcpReassembleTest38 (void) { ThreadVars tv; memset(&tv, 0, sizeof (ThreadVars)); - /* prevent L7 from kicking in */ - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0); - FLOW_INITIALIZE(&f); StreamTcpInitConfig(TRUE); TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); @@ -6110,13 +6097,13 @@ static int StreamTcpReassembleTest38 (void) { s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + printf("failed in segments reassembly, while processing toserver packet (1): "); goto end; } /* Check if we have stream smsgs in queue */ if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue\n"); + printf("there shouldn't be any stream smsgs in the queue (2): "); goto end; } @@ -6128,14 +6115,14 @@ static int StreamTcpReassembleTest38 (void) { s = &ssn.client; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + printf("failed in segments reassembly, while processing toserver packet (3): "); goto end; } /* Check if we have stream smsgs in queue */ if (ra_ctx->stream_q->len > 0) { printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet\n"); + " processed any smsg from toserver side till yet (4): "); goto end; } @@ -6146,17 +6133,13 @@ static int StreamTcpReassembleTest38 (void) { tcph.th_ack = htonl(53); s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + printf("failed in segments reassembly, while processing toserver packet (5): "); goto end; } /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue\n"); - goto end; - /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs\n"); + if (ra_ctx->stream_q->len != 0) { + printf("there should be no stream smsgs in the queue (6): "); goto end; } @@ -6167,7 +6150,7 @@ static int StreamTcpReassembleTest38 (void) { tcph.th_ack = htonl(100); s = &ssn.client; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); + printf("failed in segments reassembly, while processing toserver packet (8): "); goto end; } @@ -6179,22 +6162,10 @@ static int StreamTcpReassembleTest38 (void) { s = &ssn.server; if (StreamTcpReassembleHandleSegment(&tv, ra_ctx, &ssn, s, p, &pq) == -1) { - printf("failed in segments reassembly, while processing toserver packet\n"); - goto end; - } -#if 0 - /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len == 0) { - printf("there should be a stream smsgs in the queue, as we have detected" - " the app layer protocol and one smsg from toserver side has " - "been sent\n"); - goto end; - /* Process stream smsgs we may have in queue */ - } else if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { - printf("failed in processing stream smsgs\n"); + printf("failed in segments reassembly, while processing toserver packet (9): "); goto end; } -#endif + ret = 1; end: StreamTcpReassembleFreeThreadCtx(ra_ctx); @@ -6316,9 +6287,8 @@ static int StreamTcpReassembleTest39 (void) { } /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet (4): "); + if (ra_ctx->stream_q->len == 0) { + printf("there should be stream smsgs in the queue (4): "); goto end; } SCLogDebug("check client seg list %p", ssn.client.seg_list); @@ -6469,6 +6439,8 @@ static int StreamTcpReassembleTest40 (void) { FLOW_INITIALIZE(&f); StreamTcpInitConfig(TRUE); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 130); + TcpReassemblyThreadCtx *ra_ctx = StreamTcpReassembleInitThreadCtx(); AppLayerDetectProtoThreadInit(); @@ -6753,8 +6725,8 @@ static int StreamTcpReassembleTest41 (void) { uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 0); - StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 0); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 100); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 500); ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; ssn.server.isn = 9; @@ -6830,9 +6802,13 @@ static int StreamTcpReassembleTest41 (void) { } /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet: "); + if (ra_ctx->stream_q->len == 0) { + printf("there should be stream smsgs in the queue: "); + goto end; + } + + if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs: "); goto end; } @@ -6940,6 +6916,9 @@ static int StreamTcpReassembleTest42 (void) { uint8_t httpbuf2[] = "HTTP/1.0 200 OK\r\nServer: VictorServer/1.0\r\n\r\n"; uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 50); + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 50); + ssn.server.ra_raw_base_seq = ssn.server.ra_app_base_seq = 9; ssn.server.isn = 9; ssn.server.last_ack = 60; @@ -6989,6 +6968,9 @@ static int StreamTcpReassembleTest42 (void) { goto end; } + /* pause the reassembling */ + StreamTcpReassemblePause(&ssn, 1); + p->flowflags = FLOW_PKT_TOSERVER; p->payload = httpbuf1; p->payload_len = httplen1; @@ -7002,14 +6984,15 @@ static int StreamTcpReassembleTest42 (void) { } /* Check if we have stream smsgs in queue */ - if (ra_ctx->stream_q->len > 0) { - printf("there shouldn't be any stream smsgs in the queue, as we didn't" - " processed any smsg from toserver side till yet (4): "); + if (ra_ctx->stream_q->len == 0) { + printf("there should be stream smsgs in the queue (4): "); goto end; } - /* pause the reassembling */ - StreamTcpReassemblePause(&ssn, 1); + if (StreamTcpReassembleProcessAppLayer(ra_ctx) < 0) { + printf("failed in processing stream smsgs: "); + goto end; + } p->flowflags = FLOW_PKT_TOCLIENT; p->payload = httpbuf2; diff --git a/src/stream-tcp.c b/src/stream-tcp.c index 5fc4f6bc2f..9fd48bbee2 100644 --- a/src/stream-tcp.c +++ b/src/stream-tcp.c @@ -66,6 +66,8 @@ #define STREAMTCP_DEFAULT_MEMCAP 32 * 1024 * 1024 /* 32mb */ #define STREAMTCP_DEFAULT_REASSEMBLY_MEMCAP 64 * 1024 * 1024 /* 64mb */ #define STREAMTCP_DEFAULT_REASSEMBLY_WINDOW 3000 +#define STREAMTCP_DEFAULT_TOSERVER_CHUNK_SIZE 2560 +#define STREAMTCP_DEFAULT_TOCLIENT_CHUNK_SIZE 2560 #define STREAMTCP_NEW_TIMEOUT 60 #define STREAMTCP_EST_TIMEOUT 3600 @@ -421,6 +423,31 @@ void StreamTcpInitConfig(char quiet) SCLogInfo("stream.\"inline\": %s", stream_inline ? "enabled" : "disabled"); } + if ((ConfGetInt("stream.reassembly.toserver_chunk_size", &value)) == 1) { + stream_config.reassembly_toserver_chunk_size = (uint16_t)value; + } else { + stream_config.reassembly_toserver_chunk_size = + STREAMTCP_DEFAULT_TOSERVER_CHUNK_SIZE; + } + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, + stream_config.reassembly_toserver_chunk_size); + + if ((ConfGetInt("stream.reassembly.toclient_chunk_size", &value)) == 1) { + stream_config.reassembly_toclient_chunk_size = (uint16_t)value; + } else { + stream_config.reassembly_toclient_chunk_size = + STREAMTCP_DEFAULT_TOCLIENT_CHUNK_SIZE; + } + StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, + stream_config.reassembly_toclient_chunk_size); + + if (!quiet) { + SCLogInfo("stream.reassembly \"toserver_chunk_size\": %"PRIu16, + stream_config.reassembly_toserver_chunk_size); + SCLogInfo("stream.reassembly \"toclient_chunk_size\": %"PRIu16, + stream_config.reassembly_toclient_chunk_size); + } + /** \todo yaml part */ stream_config.reassembly_inline_window = STREAMTCP_DEFAULT_REASSEMBLY_WINDOW; @@ -1181,13 +1208,11 @@ static int StreamTcpPacketStateSynSent(ThreadVars *tv, Packet *p, StreamTcpPacketSetState(p, ssn, TCP_CLOSED); SCLogDebug("ssn %p: Reset received and state changed to " "TCP_CLOSED", ssn); - StreamTcpSessionPktFree(p); } } else { StreamTcpPacketSetState(p, ssn, TCP_CLOSED); SCLogDebug("ssn %p: Reset received and state changed to " "TCP_CLOSED", ssn); - StreamTcpSessionPktFree(p); } } else return -1; @@ -1499,8 +1524,6 @@ static int StreamTcpPacketStateSynRecv(ThreadVars *tv, Packet *p, if (ssn->flags & STREAMTCP_FLAG_TIMESTAMP) { StreamTcpHandleTimestamp(ssn, p); } - - StreamTcpSessionPktFree(p); } } else return -1; @@ -2366,7 +2389,13 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p, StreamTcpHandleTimestamp(ssn, p); } - StreamTcpSessionPktFree(p); + if (PKT_IS_TOSERVER(p)) { + StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, + &ssn->client, p, pq); + } else { + StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, + &ssn->server, p, pq); + } } else return -1; @@ -2512,7 +2541,13 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p, StreamTcpHandleTimestamp(ssn, p); } - StreamTcpSessionPktFree(p); + if (PKT_IS_TOSERVER(p)) { + StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, + &ssn->client, p, pq); + } else { + StreamTcpReassembleHandleSegment(tv, stt->ra_ctx, ssn, + &ssn->server, p, pq); + } } else return -1; @@ -3030,8 +3065,6 @@ static int StreamTcpPakcetStateLastAck(ThreadVars *tv, Packet *p, SCLogDebug("ssn %p: =+ next SEQ %" PRIu32 ", last ACK " "%" PRIu32 "", ssn, ssn->client.next_seq, ssn->server.last_ack); - - StreamTcpSessionPktFree(p); } break; default: @@ -3108,8 +3141,6 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p, SCLogDebug("ssn %p: =+ next SEQ %" PRIu32 ", last ACK " "%" PRIu32 "", ssn, ssn->client.next_seq, ssn->server.last_ack); - - StreamTcpSessionPktFree(p); } else { SCLogDebug("ssn %p: pkt (%" PRIu32 ") is to client: SEQ " "%" PRIu32 ", ACK %" PRIu32 "", ssn, p->payload_len, @@ -3150,8 +3181,6 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p, SCLogDebug("ssn %p: =+ next SEQ %" PRIu32 ", last ACK " "%" PRIu32 "", ssn, ssn->server.next_seq, ssn->client.last_ack); - - StreamTcpSessionPktFree(p); } break; default: @@ -3302,8 +3331,6 @@ static int StreamTcpPacket (ThreadVars *tv, Packet *p, StreamTcpThread *stt, &ssn->server, np, NULL); } - StreamTcpSessionPktFree(np); - /* enqueue this packet so we inspect it in detect etc */ PacketEnqueue(pq, np); } diff --git a/src/stream-tcp.h b/src/stream-tcp.h index 8e723fd492..47b18a0d1a 100644 --- a/src/stream-tcp.h +++ b/src/stream-tcp.h @@ -54,6 +54,9 @@ typedef struct TcpStreamCnf_ { uint32_t reassembly_memcap; /**< max memory usage for stream reassembly */ uint32_t reassembly_depth; /**< Depth until when we reassemble the stream */ + uint16_t reassembly_toserver_chunk_size; + uint16_t reassembly_toclient_chunk_size; + /** reassembly -- inline mode * * sliding window size for raw stream reassembly diff --git a/src/stream.c b/src/stream.c index 89e63a1389..bee05e96ab 100644 --- a/src/stream.c +++ b/src/stream.c @@ -20,7 +20,7 @@ * * \author Victor Julien * - * Stream Handling API + * Stream Chunk Handling API */ #include "suricata-common.h" diff --git a/src/stream.h b/src/stream.h index 543e5937fd..401fb53f13 100644 --- a/src/stream.h +++ b/src/stream.h @@ -36,17 +36,17 @@ #define MSG_DATA_SIZE 4024 /* 4096 - 72 (size of rest of the struct) */ typedef struct StreamMsg_ { - uint32_t id; /**< unique stream id */ uint8_t flags; /**< msg flags */ Flow *flow; /**< parent flow */ + struct StreamMsg_ *next; + struct StreamMsg_ *prev; + union { /* case !STREAM_EOF && !STREAM_GAP */ struct { - Address src_ip, dst_ip; /**< ipaddresses */ - Port src_port, dst_port; /**< ports */ uint8_t data[MSG_DATA_SIZE];/**< reassembled data */ - uint16_t data_len; /**< length of the data */ + uint32_t data_len; /**< length of the data */ uint32_t seq; /**< sequence number */ } data; /* case STREAM_GAP */ @@ -55,8 +55,6 @@ typedef struct StreamMsg_ { } gap; }; - struct StreamMsg_ *next; - struct StreamMsg_ *prev; } StreamMsg; typedef struct StreamMsgQueue_ { diff --git a/suricata.yaml b/suricata.yaml index 681e79c81a..9348d841c6 100644 --- a/suricata.yaml +++ b/suricata.yaml @@ -427,6 +427,10 @@ flow-timeouts: # reassembly: # memcap: 67108864 # 64mb tcp reassembly memcap # depth: 1048576 # 1 MB reassembly depth +# toserver_chunk_size: 2560 # inspect raw stream in chunks of at least +# # this size +# toclient_chunk_size: 2560 # inspect raw stream in chunks of at least +# # this size stream: memcap: 33554432 # 32mb checksum_validation: yes # reject wrong csums @@ -434,6 +438,8 @@ stream: reassembly: memcap: 67108864 # 64mb for reassembly depth: 1048576 # reassemble 1mb into a stream + toserver_chunk_size: 2560 + toclient_chunk_size: 2560 # Logging configuration. This is not about logging IDS alerts, but # IDS output about what its doing, errors, etc.