aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support for seperate memcaps for reassembly and stream engine

Browse Source
remotes/origin/master-1.0.x
Gurvinder Singh 16 years ago committed by Victor Julien
parent c6ddcda7f8
commit 8b0ca4f628
4 changed files with 222 additions and 41 deletions
Show Stats Download Patch File Download Diff File

179
src/stream-tcp-reassemble.c
Unescape Escape View File

@ -61,6 +61,36 @@ static uint64_t segment_pool_memuse = 0;
static uint64_t segment_pool_memcnt = 0;
#endif
/* We define several pools with prealloced segments with fixed size
* payloads. We do this to prevent having to do an SCMalloc call for every
* data segment we receive, which would be a large performance penalty.
* The cost is in memory of course. */
#define segment_pool_num 8
static uint16_t segment_pool_pktsizes[segment_pool_num] = {4, 16, 112, 248, 512,
768, 1448, 0xffff};
//static uint16_t segment_pool_poolsizes[segment_pool_num] = {2048, 3072, 3072,
// 3072, 3072, 8192,
// 8192, 512};
static uint16_t segment_pool_poolsizes[segment_pool_num] = {0, 0, 0,
0, 0, 0,
0, 0};
static uint16_t segment_pool_poolsizes_prealloc[segment_pool_num] = {256, 512, 512,
512, 512, 1024,
1024, 128};
static Pool *segment_pool[segment_pool_num];
static SCMutex segment_pool_mutex[segment_pool_num];
#ifdef DEBUG
static SCMutex segment_pool_cnt_mutex;
static uint64_t segment_pool_cnt = 0;
#endif
/* index to the right pool for all packet sizes. */
static uint16_t segment_pool_idx[65536]; /* O(1) lookups of the pool */
/* Memory use counters */
static SCSpinlock stream_reassembly_memuse_spinlock;
static uint32_t stream_reassembly_memuse;
static uint32_t stream_reassembly_memuse_max;
/* prototypes */
static int HandleSegmentStartsBeforeListSegment(TcpStream *, TcpSegment *,
TcpSegment *);
@ -74,10 +104,70 @@ TcpSegment* StreamTcpGetSegment(uint16_t);
void StreamTcpSegmentReturntoPool(TcpSegment *);
void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t, uint8_t);
/**
* \brief Function to Increment the memory usage counter for the TCP reassembly
* segments
*
* \param size Size of the TCP segment and its payload length memory allocated
*/
void StreamTcpReassembleIncrMemuse(uint32_t size) {
SCSpinLock(&stream_reassembly_memuse_spinlock);
stream_reassembly_memuse += size;
if (stream_reassembly_memuse > stream_reassembly_memuse_max)
stream_reassembly_memuse_max = stream_reassembly_memuse;
SCSpinUnlock(&stream_reassembly_memuse_spinlock);
}
/**
* \brief Function to Decrease the memory usage counter for the TCP reassembly
* segments
*
* \param size Size of the TCP segment and its payload length memory allocated
*/
void StreamTcpReassembleDecrMemuse(uint32_t size) {
SCSpinLock(&stream_reassembly_memuse_spinlock);
if (size <= stream_reassembly_memuse) {
stream_reassembly_memuse -= size;
} else {
BUG_ON(size > stream_reassembly_memuse);
stream_reassembly_memuse = 0;
}
SCSpinUnlock(&stream_reassembly_memuse_spinlock);
}
/**
* \brief Function to Check the reassembly memory usage counter against the
* allowed max memory usgae for TCP segments.
*
* \param size Size of the TCP segment and its payload length memory allocated
* \retval 1 if in bounds
* \retval 0 if not in bounds
*/
int StreamTcpReassembleCheckMemcap(uint32_t size) {
SCEnter();
int ret = 0;
SCSpinLock(&stream_reassembly_memuse_spinlock);
if (size + stream_reassembly_memuse <= stream_config.reassembly_memcap)
ret = 1;
SCSpinUnlock(&stream_reassembly_memuse_spinlock);
SCReturnInt(ret);
}
/** \brief alloc a tcp segment pool entry */
void *TcpSegmentPoolAlloc(void *payload_len) {
if (StreamTcpCheckMemcap((uint32_t)sizeof(TcpSegment) + *((uint16_t *) payload_len)) == 0)
if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(TcpSegment) +
*((uint16_t *) payload_len)) == 0)
{
return NULL;
}
TcpSegment *seg = SCMalloc(sizeof (TcpSegment));
if (seg == NULL)
@ -102,7 +192,7 @@ void *TcpSegmentPoolAlloc(void *payload_len) {
SCMutexUnlock(&segment_pool_memuse_mutex);
#endif
StreamTcpIncrMemuse((uint32_t)seg->pool_size + sizeof(TcpSegment));
StreamTcpReassembleIncrMemuse((uint32_t)seg->pool_size + sizeof(TcpSegment));
return seg;
}
@ -113,7 +203,7 @@ void TcpSegmentPoolFree(void *ptr) {
TcpSegment *seg = (TcpSegment *) ptr;
StreamTcpDecrMemuse((uint32_t)seg->pool_size + sizeof(TcpSegment));
StreamTcpReassembleDecrMemuse((uint32_t)seg->pool_size + sizeof(TcpSegment));
#ifdef DEBUG
SCMutexLock(&segment_pool_memuse_mutex);
@ -128,34 +218,15 @@ void TcpSegmentPoolFree(void *ptr) {
return;
}
/* We define serveral pools with prealloced segments with fixed size
* payloads. We do this to prevent having to do an SCMalloc call for every
* data segment we receive, which would be a large performance penalty.
* The cost is in memory of course. */
#define segment_pool_num 8
static uint16_t segment_pool_pktsizes[segment_pool_num] = {4, 16, 112, 248, 512,
768, 1448, 0xffff};
//static uint16_t segment_pool_poolsizes[segment_pool_num] = {2048, 3072, 3072,
// 3072, 3072, 8192,
// 8192, 512};
static uint16_t segment_pool_poolsizes[segment_pool_num] = {0, 0, 0,
0, 0, 0,
0, 0};
static uint16_t segment_pool_poolsizes_prealloc[segment_pool_num] = {256, 512, 512,
512, 512, 1024,
1024, 128};
static Pool *segment_pool[segment_pool_num];
static SCMutex segment_pool_mutex[segment_pool_num];
#ifdef DEBUG
static SCMutex segment_pool_cnt_mutex;
static uint64_t segment_pool_cnt = 0;
#endif
/* index to the right pool for all packet sizes. */
static uint16_t segment_pool_idx[65536]; /* O(1) lookups of the pool */
int StreamTcpReassembleInit(char quiet)
{
StreamMsgQueuesInit();
/* init the memcap and it's lock */
stream_reassembly_memuse = 0;
stream_reassembly_memuse_max = 0;
SCSpinInit(&stream_reassembly_memuse_spinlock, PTHREAD_PROCESS_PRIVATE);
#ifdef DEBUG
SCMutexInit(&segment_pool_memuse_mutex, NULL);
#endif
@ -211,6 +282,14 @@ void StreamTcpReassembleFree(char quiet)
StreamMsgQueuesDeinit(quiet);
if (!quiet) {
SCLogInfo("Max memuse of the stream reassembly engine %"PRIu32" (in use"
" %"PRIu32")", stream_reassembly_memuse_max,
stream_reassembly_memuse);
}
SCSpinDestroy(&stream_reassembly_memuse_spinlock);
#ifdef DEBUG
SCLogDebug("segment_pool_cnt %"PRIu64"", segment_pool_cnt);
SCLogDebug("segment_pool_memuse %"PRIu64"", segment_pool_memuse);
@ -5448,6 +5527,49 @@ end:
return ret;
}
/** \test Test the memcap incrementing/decrementing and memcap check */
static int StreamTcpReassembleTest44(void)
{
uint8_t ret = 0;
StreamTcpInitConfig(TRUE);
uint32_t memuse = stream_reassembly_memuse;
StreamTcpReassembleIncrMemuse(500);
if (stream_reassembly_memuse != (memuse+500)) {
printf("failed in incrementing the memory");
goto end;
}
StreamTcpReassembleDecrMemuse(500);
if (stream_reassembly_memuse != memuse) {
printf("failed in decrementing the memory");
goto end;
}
if (StreamTcpReassembleCheckMemcap(500) != 1) {
printf("failed in validating the memcap");
goto end;
}
if (StreamTcpReassembleCheckMemcap((memuse + stream_config.reassembly_memcap)) != 0) {
printf("failed in validating the memcap");
goto end;
}
StreamTcpFreeConfig(TRUE);
if (stream_reassembly_memuse != 0) {
printf("failed in clearing the memory");
goto end;
}
ret = 1;
return ret;
end:
StreamTcpFreeConfig(TRUE);
return ret;
}
#endif /* UNITTESTS */
/** \brief The Function Register the Unit tests to test the reassembly engine
@ -5499,5 +5621,6 @@ void StreamTcpReassembleRegisterTests(void) {
UtRegisterTest("StreamTcpReassembleTest41 -- app proto test", StreamTcpReassembleTest41, 1);
UtRegisterTest("StreamTcpReassembleTest42 -- pause/unpause reassembly test", StreamTcpReassembleTest42, 1);
UtRegisterTest("StreamTcpReassembleTest43 -- min smsg size test", StreamTcpReassembleTest43, 1);
UtRegisterTest("StreamTcpReassembleTest44 -- Memcap Test", StreamTcpReassembleTest44, 1);
#endif /* UNITTESTS */
}

76
src/stream-tcp.c
Unescape Escape View File

@ -55,6 +55,19 @@
//#define DEBUG
#define STREAMTCP_DEFAULT_SESSIONS 262144
#define STREAMTCP_DEFAULT_PREALLOC 32768
#define STREAMTCP_DEFAULT_MEMCAP 32 * 1024 * 1024 /* 32mb */
#define STREAMTCP_DEFAULT_REASSEMBLY_MEMCAP 64 * 1024 * 1024 /* 64mb */
#define STREAMTCP_NEW_TIMEOUT 60
#define STREAMTCP_EST_TIMEOUT 3600
#define STREAMTCP_CLOSED_TIMEOUT 120
#define STREAMTCP_EMERG_NEW_TIMEOUT 10
#define STREAMTCP_EMERG_EST_TIMEOUT 300
#define STREAMTCP_EMERG_CLOSED_TIMEOUT 20
typedef struct StreamTcpThread_ {
uint64_t pkts;
@ -79,18 +92,6 @@ int StreamTcpGetFlowState(void *);
static int ValidTimestamp(TcpSession * , Packet *);
void StreamTcpSetOSPolicy(TcpStream*, Packet*);
#define STREAMTCP_DEFAULT_SESSIONS 262144
#define STREAMTCP_DEFAULT_PREALLOC 32768
#define STREAMTCP_DEFAULT_MEMCAP 64 * 1024 * 1024 /* 64mb */
#define STREAMTCP_NEW_TIMEOUT 60
#define STREAMTCP_EST_TIMEOUT 3600
#define STREAMTCP_CLOSED_TIMEOUT 120
#define STREAMTCP_EMERG_NEW_TIMEOUT 10
#define STREAMTCP_EMERG_EST_TIMEOUT 300
#define STREAMTCP_EMERG_CLOSED_TIMEOUT 20
static Pool *ssn_pool = NULL;
static SCMutex ssn_pool_mutex;
#ifdef DEBUG
@ -381,6 +382,13 @@ void StreamTcpInitConfig(char quiet)
} else {
stream_config.memcap = STREAMTCP_DEFAULT_MEMCAP;
}
if ((ConfGetInt("stream.reassembly.memcap", &value)) == 1) {
stream_config.reassembly_memcap = (uint32_t)value;
} else {
stream_config.reassembly_memcap = STREAMTCP_DEFAULT_REASSEMBLY_MEMCAP;
}
if (!quiet) {
SCLogInfo("stream \"memcap\": %"PRIu32"", stream_config.memcap);
}
@ -5699,6 +5707,49 @@ end:
return ret;
}
/** \test Test the memcap incrementing/decrementing and memcap check */
static int StreamTcpTest28(void)
{
uint8_t ret = 0;
StreamTcpInitConfig(TRUE);
uint32_t memuse = stream_memuse;
StreamTcpIncrMemuse(500);
if (stream_memuse != (memuse+500)) {
printf("failed in incrementing the memory");
goto end;
}
StreamTcpDecrMemuse(500);
if (stream_memuse != memuse) {
printf("failed in decrementing the memory");
goto end;
}
if (StreamTcpCheckMemcap(500) != 1) {
printf("failed in validating the memcap");
goto end;
}
if (StreamTcpCheckMemcap((memuse + stream_config.memcap)) != 0) {
printf("failed in validating the memcap");
goto end;
}
StreamTcpFreeConfig(TRUE);
if (stream_memuse != 0) {
printf("failed in clearing the memory");
goto end;
}
ret = 1;
return ret;
end:
StreamTcpFreeConfig(TRUE);
return ret;
}
#endif /* UNITTESTS */
void StreamTcpRegisterTests (void) {
@ -5749,6 +5800,7 @@ void StreamTcpRegisterTests (void) {
StreamTcpTest26, 1);
UtRegisterTest("StreamTcpTest27 -- test ecn/cwr sessions",
StreamTcpTest27, 1);
UtRegisterTest("StreamTcpTest28 -- Memcap Test", StreamTcpTest28, 1);
/* set up the reassembly tests as well */
StreamTcpReassembleRegisterTests();

1
src/stream-tcp.h
Unescape Escape View File

@ -42,6 +42,7 @@ typedef struct TcpStreamCnf_ {
uint32_t prealloc_sessions;
int midstream;
int async_oneside;
uint32_t reassembly_memcap; /**< max memory usage for stream reassembly */
} TcpStreamCnf;
TcpStreamCnf stream_config;

7
suricata.yaml
Unescape Escape View File

@ -230,12 +230,17 @@ flow-timeouts:
# Stream engine settings.
# stream:
# memcap: 67108864 # 64mb memcap
# memcap: 33554432 # 32mb memcap
# reassembly:
# memcap: 67108864 # 64mb reassembly memcap
# max_sessions: 262144 # 256k concurrent sessions
# prealloc_sessions: 32768 # 32k sessions prealloc'd
# midstream: false # don't allow midstream session pickups
# async_oneside: false # don't enable async stream handling
stream:
memcap: 33554432
reassembly:
memcap: 67108864
# Logging configuration. This is not about logging IDS alerts, but
# IDS output about what its doing, errors, etc.

Write Preview
Loading…
Cancel
Save