aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Big update:

Browse Source 
- Implement "closing" state in flow.
- Add protocol specific timeouts.
- Lots of stream tracking updates, fixing a lot of out of window issues.
- Stream reassembly fixes.
- Implement a new IDS runmode with 4 stream and detect threads.
- Added a BUG_ON macro that aborts the engine if the expression is true.
- Better balance the flow queue handler for traffic that doesn't have flow (like icmp currently).
- Simplify application level protocol in the Tcp Session.
- Add some debugging memory counters.
remotes/origin/master-1.0.x
Victor Julien 17 years ago
parent b4f0e82463
commit b102ea2123
25 changed files with 1493 additions and 845 deletions
Show Stats Download Patch File Download Diff File

100
src/app-layer-detect-proto.c
Unescape Escape View File

@ -61,25 +61,7 @@ typedef struct AlpProtoDetectCtx_ {
static AlpProtoDetectCtx alp_proto_ctx; static AlpProtoDetectCtx alp_proto_ctx;
static AlpProtoDetectThreadCtx alp_proto_tctx; static AlpProtoDetectThreadCtx alp_proto_tctx;
static uint8_t al_proto_id = 0;
/** \brief data stored in the stream */
typedef struct AppLayerDetectProtoData_ {
uint8_t proto;
} AppLayerDetectProtoData;
static Pool *al_detect_proto_pool = NULL;
void *AppLayerDetectProtoAlloc(void *null) {
AppLayerDetectProtoData *d = malloc(sizeof(AppLayerDetectProtoData));
if (d == NULL) {
return NULL;
}
d->proto = ALPROTO_UNKNOWN;
return d;
}
#define AppLayerDetectProtoFree free
void AlpProtoInit(AlpProtoDetectCtx *ctx) { void AlpProtoInit(AlpProtoDetectCtx *ctx) {
memset(ctx, 0x00, sizeof(AlpProtoDetectCtx)); memset(ctx, 0x00, sizeof(AlpProtoDetectCtx));
@ -108,7 +90,7 @@ void AlpProtoDestroy(AlpProtoDetectCtx *ctx) {
* \param offset Offset setting for the content. E.g. 4 mean that the content has to match after the first 4 bytes of the stream. * \param offset Offset setting for the content. E.g. 4 mean that the content has to match after the first 4 bytes of the stream.
* \param flags Set STREAM_TOCLIENT or STREAM_TOSERVER for the direction in which to try to match the content. * \param flags Set STREAM_TOCLIENT or STREAM_TOSERVER for the direction in which to try to match the content.
*/ */
void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint8_t al_proto, char *content, uint16_t depth, uint16_t offset, uint8_t flags) { void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint16_t al_proto, char *content, uint16_t depth, uint16_t offset, uint8_t flags) {
DetectContentData *cd = DetectContentParse(content); DetectContentData *cd = DetectContentParse(content);
if (cd == NULL) { if (cd == NULL) {
return; return;
@ -159,13 +141,6 @@ void AlpProtoFinalizeGlobal(AlpProtoDetectCtx *ctx) {
} }
void AppLayerDetectProtoThreadInit(void) { void AppLayerDetectProtoThreadInit(void) {
al_proto_id = StreamL7RegisterModule();
al_detect_proto_pool = PoolInit(262144, 32768, AppLayerDetectProtoAlloc, NULL, AppLayerDetectProtoFree);
if (al_detect_proto_pool == NULL) {
exit(1);
}
AlpProtoInit(&alp_proto_ctx); AlpProtoInit(&alp_proto_ctx);
/** \todo register these in the protocol parser api */ /** \todo register these in the protocol parser api */
@ -267,12 +242,17 @@ end:
if (dir->mpm_ctx.Cleanup != NULL) { if (dir->mpm_ctx.Cleanup != NULL) {
dir->mpm_ctx.Cleanup(&tdir->mpm_ctx); dir->mpm_ctx.Cleanup(&tdir->mpm_ctx);
} }
//#define DEBUG
#ifdef DEBUG #ifdef DEBUG
printf("AppLayerDetectGetProto: returning %" PRIu16 " (%s): ", proto, flags & STREAM_TOCLIENT ? "TOCLIENT" : "TOSERVER"); printf("AppLayerDetectGetProto: returning %" PRIu16 " (%s): ", proto, flags & STREAM_TOCLIENT ? "TOCLIENT" : "TOSERVER");
switch (proto) { switch (proto) {
case ALPROTO_HTTP: case ALPROTO_HTTP:
printf("HTTP\n"); printf("HTTP: ");
/* print the first 32 bytes */
if (buflen > 0) {
PrintRawUriFp(stdout,buf,(buflen>32)?32:buflen);
}
printf("\n");
break; break;
case ALPROTO_FTP: case ALPROTO_FTP:
printf("FTP\n"); printf("FTP\n");
@ -283,6 +263,9 @@ end:
case ALPROTO_SSH: case ALPROTO_SSH:
printf("SSH\n"); printf("SSH\n");
break; break;
case ALPROTO_TLS:
printf("TLS\n");
break;
case ALPROTO_IMAP: case ALPROTO_IMAP:
printf("IMAP\n"); printf("IMAP\n");
break; break;
@ -297,8 +280,12 @@ end:
break; break;
case ALPROTO_UNKNOWN: case ALPROTO_UNKNOWN:
default: default:
printf("UNKNOWN\n"); printf("UNKNOWN (%u): cnt was %u (", proto, cnt);
PrintRawDataFp(stdout,buf,buflen); /* print the first 32 bytes */
if (buflen > 0) {
PrintRawUriFp(stdout,buf,(buflen>32)?32:buflen);
}
printf(")\n");
break; break;
} }
#endif #endif
@ -309,9 +296,7 @@ void *AppLayerDetectProtoThread(void *td)
{ {
ThreadVars *tv = (ThreadVars *)td; ThreadVars *tv = (ThreadVars *)td;
char run = TRUE; char run = TRUE;
AppLayerDetectProtoData *al_proto = NULL; uint16_t alproto = ALPROTO_UNKNOWN;
char store = 0;
void *al_data_ptr = NULL;
/* get the stream msg queue for this thread */ /* get the stream msg queue for this thread */
StreamMsgQueue *stream_q = StreamMsgQueueGetByPort(0); StreamMsgQueue *stream_q = StreamMsgQueueGetByPort(0);
@ -327,20 +312,13 @@ void *AppLayerDetectProtoThread(void *td)
StreamMsg *smsg = StreamMsgGetFromQueue(stream_q); StreamMsg *smsg = StreamMsgGetFromQueue(stream_q);
if (smsg != NULL) { if (smsg != NULL) {
mutex_lock(&smsg->flow->m); mutex_lock(&smsg->flow->m);
TcpSession *ssn = smsg->flow->stream; TcpSession *ssn = smsg->flow->protoctx;
if (ssn != NULL) { if (ssn != NULL) {
if (ssn->l7data == NULL) { alproto = ssn->alproto;
/* XXX we can use a pool here,
or make it part of the stream setup */
StreamL7DataPtrInit(ssn,StreamL7GetStorageSize());
}
if (ssn->l7data != NULL) {
al_data_ptr = ssn->l7data[al_proto_id];
}
} }
mutex_unlock(&smsg->flow->m); mutex_unlock(&smsg->flow->m);
if (ssn != NULL && ssn->l7data != NULL) { if (ssn != NULL) {
if (smsg->flags & STREAM_START) { if (smsg->flags & STREAM_START) {
//printf("L7AppDetectThread: stream initializer (len %" PRIu32 " (%" PRIu32 "))\n", smsg->data.data_len, MSG_DATA_SIZE); //printf("L7AppDetectThread: stream initializer (len %" PRIu32 " (%" PRIu32 "))\n", smsg->data.data_len, MSG_DATA_SIZE);
@ -348,17 +326,18 @@ void *AppLayerDetectProtoThread(void *td)
//PrintRawDataFp(stdout, smsg->init.data, smsg->init.data_len); //PrintRawDataFp(stdout, smsg->init.data, smsg->init.data_len);
//printf("=> Init Stream Data -- end\n"); //printf("=> Init Stream Data -- end\n");
if (al_data_ptr == NULL) { alproto = AppLayerDetectGetProto(&alp_proto_ctx, &alp_proto_tctx, smsg->data.data, smsg->data.data_len, smsg->flags);
al_proto = (AppLayerDetectProtoData *)PoolGet(al_detect_proto_pool); if (alproto != ALPROTO_UNKNOWN) {
if (al_proto != NULL) { /* store the proto and setup the L7 data array */
al_proto->proto = AppLayerDetectGetProto(&alp_proto_ctx, &alp_proto_tctx, smsg->data.data, smsg->data.data_len, smsg->flags); mutex_lock(&smsg->flow->m);
store = 1; StreamL7DataPtrInit(ssn,StreamL7GetStorageSize());
ssn->alproto = alproto;
mutex_unlock(&smsg->flow->m);
AppLayerParse(smsg->flow, al_proto->proto, smsg->flags, smsg->data.data, smsg->data.data_len); AppLayerParse(smsg->flow, alproto, smsg->flags, smsg->data.data, smsg->data.data_len);
}
} }
} else { } else {
//printf("AppLayerDetectThread: stream data (len %" PRIu32 " (%" PRIu32 "))\n", smsg->data.data_len, MSG_DATA_SIZE); //printf("AppLayerDetectThread: stream data (len %" PRIu32 " (%" PRIu32 ")), alproto %"PRIu16"\n", smsg->data.data_len, MSG_DATA_SIZE, alproto);
//printf("=> Stream Data -- start\n"); //printf("=> Stream Data -- start\n");
//PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); //PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len);
@ -366,11 +345,8 @@ void *AppLayerDetectProtoThread(void *td)
/* if we don't have a data object here we are not getting it /* if we don't have a data object here we are not getting it
* a start msg should have gotten us one */ * a start msg should have gotten us one */
if (al_data_ptr != NULL) { if (alproto != ALPROTO_UNKNOWN) {
al_proto = (AppLayerDetectProtoData *)al_data_ptr; AppLayerParse(smsg->flow, alproto, smsg->flags, smsg->data.data, smsg->data.data_len);
//printf("AppLayerDetectThread: already established that the proto is %" PRIu32 "\n", al_proto->proto);
AppLayerParse(smsg->flow, al_proto->proto, smsg->flags, smsg->data.data, smsg->data.data_len);
} else { } else {
//printf("AppLayerDetectThread: smsg not start, but no l7 data? Weird\n"); //printf("AppLayerDetectThread: smsg not start, but no l7 data? Weird\n");
} }
@ -378,19 +354,9 @@ void *AppLayerDetectProtoThread(void *td)
} }
mutex_lock(&smsg->flow->m); mutex_lock(&smsg->flow->m);
if (store == 1) {
/* store */
if (ssn != NULL && ssn->l7data != NULL) {
ssn->l7data[al_proto_id] = (void *)al_proto;
} else {
al_proto->proto = 0;
PoolReturn(al_detect_proto_pool,(void *)al_proto);
}
store = 0;
}
/* XXX we need to improve this logic */
smsg->flow->use_cnt--; smsg->flow->use_cnt--;
mutex_unlock(&smsg->flow->m); mutex_unlock(&smsg->flow->m);
/* return the used message to the queue */ /* return the used message to the queue */
StreamMsgReturnToPool(smsg); StreamMsgReturnToPool(smsg);
} }

73
src/app-layer-http.c
Unescape Escape View File

@ -208,7 +208,8 @@ static int HTTPParseRequest(void *http_state, AppLayerParserState *pstate, uint8
} }
break; break;
} }
case 2: #if 0
case 2: /* BODY */
{ {
//printf("HTTPParseRequest: request body\n"); //printf("HTTPParseRequest: request body\n");
@ -223,6 +224,7 @@ static int HTTPParseRequest(void *http_state, AppLayerParserState *pstate, uint8
break; break;
} }
#endif
} }
} }
@ -347,6 +349,7 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint
} }
break; break;
} }
#if 0
case 2: case 2:
{ {
//printf("HTTPParseResponse: response body\n"); //printf("HTTPParseResponse: response body\n");
@ -362,6 +365,7 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint
break; break;
} }
#endif
} }
} }
@ -370,17 +374,36 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint
return 1; return 1;
} }
#ifdef DEBUG
static pthread_mutex_t http_state_mem_lock = PTHREAD_MUTEX_INITIALIZER;
static uint64_t http_state_memuse = 0;
static uint64_t http_state_memcnt = 0;
#endif
static void *HTTPStateAlloc(void) { static void *HTTPStateAlloc(void) {
void *s = malloc(sizeof(HttpState)); void *s = malloc(sizeof(HttpState));
if (s == NULL) if (s == NULL)
return NULL; return NULL;
memset(s, 0, sizeof(HttpState)); memset(s, 0, sizeof(HttpState));
#ifdef DEBUG
mutex_lock(&http_state_mem_lock);
http_state_memcnt++;
http_state_memuse+=sizeof(HttpState);
mutex_unlock(&http_state_mem_lock);
#endif
return s; return s;
} }
static void HTTPStateFree(void *s) { static void HTTPStateFree(void *s) {
free(s); free(s);
#ifdef DEBUG
mutex_lock(&http_state_mem_lock);
http_state_memcnt--;
http_state_memuse-=sizeof(HttpState);
mutex_unlock(&http_state_mem_lock);
#endif
} }
void RegisterHTTPParsers(void) { void RegisterHTTPParsers(void) {
@ -396,6 +419,14 @@ void RegisterHTTPParsers(void) {
AppLayerRegisterStateFuncs(ALPROTO_HTTP, HTTPStateAlloc, HTTPStateFree); AppLayerRegisterStateFuncs(ALPROTO_HTTP, HTTPStateAlloc, HTTPStateFree);
} }
void HTTPAtExitPrintStats(void) {
#ifdef DEBUG
mutex_lock(&http_state_mem_lock);
printf("HTTPAtExitPrintStats: http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"\n", http_state_memcnt, http_state_memuse);
mutex_unlock(&http_state_mem_lock);
#endif
}
/* UNITTESTS */ /* UNITTESTS */
#ifdef UNITTESTS #ifdef UNITTESTS
@ -410,7 +441,7 @@ int HTTPParserTest01(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen); int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen);
if (r != 0) { if (r != 0) {
@ -419,7 +450,7 @@ int HTTPParserTest01(void) {
goto end; goto end;
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;
@ -447,7 +478,7 @@ int HTTPParserTest02(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen); int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen);
if (r != 0) { if (r != 0) {
@ -456,7 +487,7 @@ int HTTPParserTest02(void) {
goto end; goto end;
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
result = 0; result = 0;
goto end; goto end;
@ -487,7 +518,7 @@ int HTTPParserTest03(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) { if (r != 0) {
@ -510,7 +541,7 @@ int HTTPParserTest03(void) {
goto end; goto end;
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;
@ -542,7 +573,7 @@ int HTTPParserTest04(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) { if (r != 0) {
@ -565,7 +596,7 @@ int HTTPParserTest04(void) {
goto end; goto end;
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;
@ -598,7 +629,7 @@ int HTTPParserTest05(void) {
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) { if (r != 0) {
@ -621,7 +652,7 @@ int HTTPParserTest05(void) {
goto end; goto end;
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;
@ -650,7 +681,7 @@ int HTTPParserTest06(void) {
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START|STREAM_EOF, httpbuf1, httplen1); int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START|STREAM_EOF, httpbuf1, httplen1);
if (r != 0) { if (r != 0) {
@ -659,7 +690,7 @@ int HTTPParserTest06(void) {
goto end; goto end;
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;
@ -690,7 +721,7 @@ int HTTPParserTest07(void) {
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) { if (r != 0) {
@ -706,7 +737,7 @@ int HTTPParserTest07(void) {
goto end; goto end;
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;
@ -747,7 +778,7 @@ int HTTPParserTest08(void) {
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
if (r != 0) { if (r != 0) {
@ -791,7 +822,7 @@ int HTTPParserTest08(void) {
goto end; goto end;
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;
@ -827,7 +858,7 @@ int HTTPParserTest09(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
uint32_t u; uint32_t u;
for (u = 0; u < httplen1; u++) { for (u = 0; u < httplen1; u++) {
@ -860,7 +891,7 @@ int HTTPParserTest09(void) {
} }
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;
@ -894,7 +925,7 @@ int HTTPParserTest10(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
uint32_t u; uint32_t u;
for (u = 0; u < httplen1; u++) { for (u = 0; u < httplen1; u++) {
@ -912,7 +943,7 @@ int HTTPParserTest10(void) {
} }
} }
HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
if (http_state == NULL) { if (http_state == NULL) {
printf("no http state: "); printf("no http state: ");
result = 0; result = 0;

1
src/app-layer-http.h
Unescape Escape View File

@ -3,6 +3,7 @@
void RegisterHTTPParsers(void); void RegisterHTTPParsers(void);
void HTTPParserRegisterTests(void); void HTTPParserRegisterTests(void);
void HTTPAtExitPrintStats(void);
#endif /* __APP_LAYER_HTTP_H__ */ #endif /* __APP_LAYER_HTTP_H__ */

59
src/app-layer-parser.c
Unescape Escape View File

@ -411,6 +411,17 @@ AppLayerParserStateStore* AppLayerParserStateStoreAlloc(void) {
return s; return s;
} }
/** \brief free a AppLayerParserStateStore structure
* \param s AppLayerParserStateStore structure to free */
void AppLayerParserStateStoreFree(AppLayerParserStateStore *s) {
if (s->to_server.store != NULL)
free(s->to_server.store);
if (s->to_client.store != NULL)
free(s->to_client.store);
free(s);
}
static void AppLayerParserResultCleanup(AppLayerParserResult *result) { static void AppLayerParserResultCleanup(AppLayerParserResult *result) {
AppLayerParserResultElmt *e = result->head; AppLayerParserResultElmt *e = result->head;
while (e != NULL) { while (e != NULL) {
@ -493,22 +504,21 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_
uint16_t parser_idx = 0; uint16_t parser_idx = 0;
AppLayerProto *p = &al_proto_table[proto]; AppLayerProto *p = &al_proto_table[proto];
TcpSession *ssn = f->stream; TcpSession *ssn = f->protoctx;
if (ssn == NULL) { if (ssn == NULL) {
printf("AppLayerParse: no stream\n"); printf("AppLayerParse: no session\n");
return -1; return -1;
} }
/* Get the parser state (if any) */ /* Get the parser state (if any) */
AppLayerParserStateStore *parser_state_store = (AppLayerParserStateStore *)ssn->l7data[app_layer_sid]; AppLayerParserStateStore *parser_state_store = (AppLayerParserStateStore *)ssn->aldata[app_layer_sid];
if (parser_state_store == NULL) { if (parser_state_store == NULL) {
parser_state_store = AppLayerParserStateStoreAlloc(); parser_state_store = AppLayerParserStateStoreAlloc();
if (parser_state_store == NULL) if (parser_state_store == NULL)
return -1; return -1;
mutex_lock(&f->m); mutex_lock(&f->m);
if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */ ssn->aldata[app_layer_sid] = (void *)parser_state_store;
ssn->l7data[app_layer_sid] = (void *)parser_state_store;
mutex_unlock(&f->m); mutex_unlock(&f->m);
} }
@ -546,8 +556,7 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_
/* See if we already have a 'app layer' state */ /* See if we already have a 'app layer' state */
void *app_layer_state = NULL; void *app_layer_state = NULL;
mutex_lock(&f->m); mutex_lock(&f->m);
if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */ app_layer_state = ssn->aldata[p->storage_id];
app_layer_state = ssn->l7data[p->storage_id];
mutex_unlock(&f->m); mutex_unlock(&f->m);
if (app_layer_state == NULL) { if (app_layer_state == NULL) {
app_layer_state = p->StateAlloc(); app_layer_state = p->StateAlloc();
@ -555,8 +564,7 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_
return -1; return -1;
mutex_lock(&f->m); mutex_lock(&f->m);
if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */ ssn->aldata[p->storage_id] = app_layer_state;
ssn->l7data[p->storage_id] = app_layer_state;
mutex_unlock(&f->m); mutex_unlock(&f->m);
} }
@ -580,6 +588,39 @@ void RegisterAppLayerParsers(void) {
al_result_pool = PoolInit(100,10,AlpResultElmtPoolAlloc,NULL,AlpResultElmtPoolFree); al_result_pool = PoolInit(100,10,AlpResultElmtPoolAlloc,NULL,AlpResultElmtPoolFree);
} }
void AppLayerParserCleanupState(TcpSession *ssn) {
if (ssn == NULL) {
//printf("AppLayerParserCleanupState: no ssn\n");
return;
}
AppLayerProto *p = &al_proto_table[ssn->alproto];
if (p == NULL) {
//printf("AppLayerParserCleanupState: no parser state for %"PRIu16"\n", ssn->alproto);
return;
}
/* free the parser protocol state */
if (p->StateFree != NULL) {
if (ssn->aldata[p->storage_id] != NULL) {
//printf("AppLayerParserCleanupState: calling StateFree\n");
p->StateFree(ssn->aldata[p->storage_id]);
ssn->aldata[p->storage_id] = NULL;
}
}
if (ssn->aldata != NULL) {
if (ssn->aldata[app_layer_sid] != NULL) {
//printf("AppLayerParserCleanupState: calling AppLayerParserStateStoreFree\n");
AppLayerParserStateStoreFree(ssn->aldata[app_layer_sid]);
ssn->aldata[app_layer_sid] = NULL;
}
free(ssn->aldata);
ssn->aldata = NULL;
}
}
/** \brief Create a mapping between the individual parsers local field id's /** \brief Create a mapping between the individual parsers local field id's
* and the global field parser id's. * and the global field parser id's.
* *

3
src/app-layer-parser.h
Unescape Escape View File

@ -87,5 +87,8 @@ int AlpParseFieldByEOF(AppLayerParserResult *, AppLayerParserState *, uint16_t,
int AlpParseFieldByDelimiter(AppLayerParserResult *, AppLayerParserState *, uint16_t, const uint8_t *, uint8_t, uint8_t *, uint32_t, uint32_t *); int AlpParseFieldByDelimiter(AppLayerParserResult *, AppLayerParserState *, uint16_t, const uint8_t *, uint8_t, uint8_t *, uint32_t, uint32_t *);
uint16_t AlpGetStateIdx(uint16_t); uint16_t AlpGetStateIdx(uint16_t);
#include "stream-tcp-private.h"
void AppLayerParserCleanupState(TcpSession *);
#endif /* __APP_LAYER_PARSER_H__ */ #endif /* __APP_LAYER_PARSER_H__ */

16
src/app-layer-tls.c
Unescape Escape View File

@ -149,7 +149,7 @@ static int TLSParserTest01(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER|STREAM_EOF, tlsbuf, tlslen); int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER|STREAM_EOF, tlsbuf, tlslen);
if (r != 0) { if (r != 0) {
@ -158,7 +158,7 @@ static int TLSParserTest01(void) {
goto end; goto end;
} }
TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)]; TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)];
if (tls_state == NULL) { if (tls_state == NULL) {
printf("no tls state: "); printf("no tls state: ");
result = 0; result = 0;
@ -193,7 +193,7 @@ static int TLSParserTest02(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1); int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1);
if (r != 0) { if (r != 0) {
@ -209,7 +209,7 @@ static int TLSParserTest02(void) {
goto end; goto end;
} }
TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)]; TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)];
if (tls_state == NULL) { if (tls_state == NULL) {
printf("no tls state: "); printf("no tls state: ");
result = 0; result = 0;
@ -246,7 +246,7 @@ static int TLSParserTest03(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1); int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1);
if (r != 0) { if (r != 0) {
@ -269,7 +269,7 @@ static int TLSParserTest03(void) {
goto end; goto end;
} }
TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)]; TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)];
if (tls_state == NULL) { if (tls_state == NULL) {
printf("no tls state: "); printf("no tls state: ");
result = 0; result = 0;
@ -308,7 +308,7 @@ static int TLSParserTest04(void) {
memset(&f, 0, sizeof(f)); memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn)); memset(&ssn, 0, sizeof(ssn));
StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
f.stream = (void *)&ssn; f.protoctx = (void *)&ssn;
int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1); int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1);
if (r != 0) { if (r != 0) {
@ -338,7 +338,7 @@ static int TLSParserTest04(void) {
goto end; goto end;
} }
TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)]; TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)];
if (tls_state == NULL) { if (tls_state == NULL) {
printf("no tls state: "); printf("no tls state: ");
result = 0; result = 0;

3
src/eidps-common.h
Unescape Escape View File

@ -13,5 +13,8 @@
#include <stdint.h> #include <stdint.h>
#include <inttypes.h> #include <inttypes.h>
#include <assert.h>
#define BUG_ON(x) assert(!(x))
#endif /* __EIDPS_COMMON_H__ */ #endif /* __EIDPS_COMMON_H__ */

319
src/eidps.c
Unescape Escape View File

@ -624,6 +624,320 @@ int RunModeIdsPcap2(DetectEngineCtx *de_ctx, char *iface) {
return 0; return 0;
} }
/** \brief Live pcap mode with 4 stream tracking and reassembly threads, testing the flow queuehandler */
int RunModeIdsPcap3(DetectEngineCtx *de_ctx, char *iface) {
TimeModeSetLive();
/* create the threads */
ThreadVars *tv_receivepcap = TmThreadCreatePacketHandler("ReceivePcap","packetpool","packetpool","pickup-queue","simple","1slot_noinout");
if (tv_receivepcap == NULL) {
printf("ERROR: TmThreadsCreate failed\n");
exit(EXIT_FAILURE);
}
TmModule *tm_module = TmModuleGetByName("ReceivePcap");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed for ReceivePcap\n");
exit(EXIT_FAILURE);
}
Tm1SlotSetFunc(tv_receivepcap,tm_module,(void *)iface);
if (TmThreadSpawn(tv_receivepcap) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
ThreadVars *tv_decode1 = TmThreadCreatePacketHandler("Decode1","pickup-queue","simple","decode-queue1,decode-queue2,decode-queue3,decode-queue4","flow","1slot");
if (tv_decode1 == NULL) {
printf("ERROR: TmThreadsCreate failed for Decode1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("DecodePcap");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName DecodePcap failed\n");
exit(EXIT_FAILURE);
}
Tm1SlotSetFunc(tv_decode1,tm_module,NULL);
if (TmThreadSpawn(tv_decode1) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
ThreadVars *tv;
tv = TmThreadCreatePacketHandler("Stream1","decode-queue1","simple","packetpool","packetpool","varslot");
if (tv == NULL) {
printf("ERROR: TmThreadsCreate failed for Stream1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertFastlog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("LogHttplog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedLog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedAlert");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertDebuglog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
TmThreadSetCPUAffinity(tv, 0);
if (TmThreadSpawn(tv) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
tv = TmThreadCreatePacketHandler("Stream2","decode-queue2","simple","packetpool","packetpool","varslot");
if (tv == NULL) {
printf("ERROR: TmThreadsCreate failed for Stream1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertFastlog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("LogHttplog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedLog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedAlert");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertDebuglog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
TmThreadSetCPUAffinity(tv, 0);
if (TmThreadSpawn(tv) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
tv = TmThreadCreatePacketHandler("Stream3","decode-queue3","simple","packetpool","packetpool","varslot");
if (tv == NULL) {
printf("ERROR: TmThreadsCreate failed for Stream1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertFastlog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("LogHttplog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedLog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedAlert");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertDebuglog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
TmThreadSetCPUAffinity(tv, 1);
if (TmThreadSpawn(tv) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
tv = TmThreadCreatePacketHandler("Stream4","decode-queue4","simple","packetpool","packetpool","varslot");
if (tv == NULL) {
printf("ERROR: TmThreadsCreate failed for Stream1\n");
exit(EXIT_FAILURE);
}
tm_module = TmModuleGetByName("StreamTcp");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("Detect");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName StreamTcp failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx);
tm_module = TmModuleGetByName("RespondReject");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for RespondReject failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertFastlog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertFastlog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("LogHttplog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedLog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertUnifiedAlert");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
tm_module = TmModuleGetByName("AlertDebuglog");
if (tm_module == NULL) {
printf("ERROR: TmModuleGetByName failed\n");
exit(EXIT_FAILURE);
}
TmVarSlotSetFuncAppend(tv,tm_module,NULL);
TmThreadSetCPUAffinity(tv, 1);
if (TmThreadSpawn(tv) != 0) {
printf("ERROR: TmThreadSpawn failed\n");
exit(EXIT_FAILURE);
}
return 0;
}
int RunModeIpsNFQ(DetectEngineCtx *de_ctx) { int RunModeIpsNFQ(DetectEngineCtx *de_ctx) {
TimeModeSetLive(); TimeModeSetLive();
@ -1256,7 +1570,8 @@ int main(int argc, char **argv)
gettimeofday(&start_time, NULL); gettimeofday(&start_time, NULL);
if (mode == MODE_PCAP_DEV) { if (mode == MODE_PCAP_DEV) {
RunModeIdsPcap2(de_ctx, pcap_dev); RunModeIdsPcap3(de_ctx, pcap_dev);
//RunModeIdsPcap2(de_ctx, pcap_dev);
//RunModeIdsPcap(de_ctx, pcap_dev); //RunModeIdsPcap(de_ctx, pcap_dev);
} }
else if (mode == MODE_PCAP_FILE) { else if (mode == MODE_PCAP_FILE) {
@ -1337,6 +1652,8 @@ int main(int argc, char **argv)
FlowShutdown(); FlowShutdown();
FlowPrintQueueInfo(); FlowPrintQueueInfo();
StreamTcpFreeConfig(STREAM_VERBOSE);
HTTPAtExitPrintStats();
/** \todo review whats needed here */ /** \todo review whats needed here */
SigGroupCleanup(de_ctx); SigGroupCleanup(de_ctx);

4
src/flow-hash.c
Unescape Escape View File

@ -98,7 +98,7 @@ Flow *FlowGetFlowFromHash (Packet *p)
/* got one, now lock, initialize and return */ /* got one, now lock, initialize and return */
mutex_lock(&f->m); mutex_lock(&f->m);
FlowInit(f,p); FlowInit(f,p);
FlowRequeue(f, NULL, &flow_new_q); FlowRequeue(f, NULL, &flow_new_q[f->protomap]);
f->flags |= FLOW_NEW_LIST; f->flags |= FLOW_NEW_LIST;
f->fb = fb; f->fb = fb;
@ -139,7 +139,7 @@ Flow *FlowGetFlowFromHash (Packet *p)
/* lock, initialize and return */ /* lock, initialize and return */
mutex_lock(&f->m); mutex_lock(&f->m);
FlowInit(f,p); FlowInit(f,p);
FlowRequeue(f, NULL, &flow_new_q); FlowRequeue(f, NULL, &flow_new_q[f->protomap]);
f->flags |= FLOW_NEW_LIST; f->flags |= FLOW_NEW_LIST;
f->fb = fb; f->fb = fb;

38
src/flow-private.h
Unescape Escape View File

@ -9,10 +9,13 @@
/* per flow flags */ /* per flow flags */
#define FLOW_TO_SRC_SEEN 0x01 #define FLOW_TO_SRC_SEEN 0x01
#define FLOW_TO_DST_SEEN 0x02 #define FLOW_TO_DST_SEEN 0x02
#define FLOW_NEW_LIST 0x04 #define FLOW_NEW_LIST 0x04
#define FLOW_EST_LIST 0x08 #define FLOW_EST_LIST 0x08
#define FLOW_TOSERVER_IPONLY_SET 0x10 #define FLOW_CLOSED_LIST 0x10
#define FLOW_TOCLIENT_IPONLY_SET 0x20
#define FLOW_TOSERVER_IPONLY_SET 0x20
#define FLOW_TOCLIENT_IPONLY_SET 0x40
/* global flow flags */ /* global flow flags */
#define FLOW_EMERGENCY 0x01 #define FLOW_EMERGENCY 0x01
@ -38,16 +41,41 @@
#define FLOW_IPPROTO_ICMP_EMERG_NEW_TIMEOUT 10 #define FLOW_IPPROTO_ICMP_EMERG_NEW_TIMEOUT 10
#define FLOW_IPPROTO_ICMP_EMERG_EST_TIMEOUT 100 #define FLOW_IPPROTO_ICMP_EMERG_EST_TIMEOUT 100
enum {
FLOW_PROTO_DEFAULT = 0,
FLOW_PROTO_TCP,
FLOW_PROTO_UDP,
FLOW_PROTO_ICMP,
/* should be last */
FLOW_PROTO_MAX,
};
/* /*
* Variables * Variables
*/ */
FlowQueue flow_spare_q; /* Spare flow's. Prealloced flows in here */ /** FlowProto specific timeouts and free/state functions */
FlowQueue flow_new_q; /* Flows in the unreplied state live here */ FlowProto flow_proto[FLOW_PROTO_MAX];
FlowQueue flow_est_q; /* All other flows live here, the top holds the
/** spare/unused/prealloced flows live here */
FlowQueue flow_spare_q;
/** Flows in the new/unreplied state live here */
FlowQueue flow_new_q[FLOW_PROTO_MAX];
/** All "established" flows live here, the top holds the
* last recently used (lru) flow, so we can remove
* that in case of memory problems and check it for
* timeouts. */
FlowQueue flow_est_q[FLOW_PROTO_MAX];
/** All "closing" flows live here, the top holds the
* last recently used (lru) flow, so we can remove * last recently used (lru) flow, so we can remove
* that in case of memory problems and check it for * that in case of memory problems and check it for
* timeouts. */ * timeouts. */
FlowQueue flow_close_q[FLOW_PROTO_MAX];
FlowBucket *flow_hash; FlowBucket *flow_hash;
FlowConfig flow_config; FlowConfig flow_config;

22
src/flow-util.c
Unescape Escape View File

@ -49,6 +49,26 @@ void FlowFree(Flow *f)
free(f); free(f);
} }
/**
* \brief Function to map the protocol to the defined FLOW_PROTO_* enumeration.
*
* \param proto protocol which is needed to be mapped
*/
int FlowGetProtoMapping(uint8_t proto) {
switch (proto) {
case IPPROTO_TCP:
return FLOW_PROTO_TCP;
case IPPROTO_UDP:
return FLOW_PROTO_UDP;
case IPPROTO_ICMP:
return FLOW_PROTO_ICMP;
default:
return FLOW_PROTO_DEFAULT;
}
}
/* initialize the flow from the first packet /* initialize the flow from the first packet
* we see from it. */ * we see from it. */
void FlowInit(Flow *f, Packet *p) void FlowInit(Flow *f, Packet *p)
@ -81,5 +101,7 @@ void FlowInit(Flow *f, Packet *p)
} }
COPY_TIMESTAMP(&p->ts, &f->startts); COPY_TIMESTAMP(&p->ts, &f->startts);
f->protomap = FlowGetProtoMapping(f->proto);
} }

6
src/flow-util.h
Unescape Escape View File

@ -3,9 +3,6 @@
#ifndef __FLOW_UTIL_H__ #ifndef __FLOW_UTIL_H__
#define __FLOW_UTIL_H__ #define __FLOW_UTIL_H__
/** FlowProto specific timeouts and free/state functions */
FlowProto flow_proto[FLOW_PROTO_MAX];
#define COPY_TIMESTAMP(src,dst) ((dst)->tv_sec = (src)->tv_sec, (dst)->tv_usec = (src)->tv_usec) #define COPY_TIMESTAMP(src,dst) ((dst)->tv_sec = (src)->tv_sec, (dst)->tv_usec = (src)->tv_usec)
/* only clear the parts that won't be overwritten /* only clear the parts that won't be overwritten
@ -19,12 +16,13 @@ FlowProto flow_proto[FLOW_PROTO_MAX];
(f)->lastts.tv_usec = 0; \ (f)->lastts.tv_usec = 0; \
GenericVarFree((f)->flowvar); \ GenericVarFree((f)->flowvar); \
(f)->flowvar = NULL; \ (f)->flowvar = NULL; \
(f)->stream = NULL; \ (f)->protoctx = NULL; \
(f)->use_cnt = 0; \ (f)->use_cnt = 0; \
} }
Flow *FlowAlloc(void); Flow *FlowAlloc(void);
void FlowFree(Flow *); void FlowFree(Flow *);
int FlowGetProtoMapping(uint8_t);
void FlowInit(Flow *, Packet *); void FlowInit(Flow *, Packet *);
#endif /* __FLOW_UTIL_H__ */ #endif /* __FLOW_UTIL_H__ */

173
src/flow.c
Unescape Escape View File

@ -25,7 +25,6 @@
#include "flow-var.h" #include "flow-var.h"
#include "flow-private.h" #include "flow-private.h"
#include "util-unittest.h" #include "util-unittest.h"
#include "stream-tcp-private.h"
//#define FLOW_DEFAULT_HASHSIZE 262144 //#define FLOW_DEFAULT_HASHSIZE 262144
#define FLOW_DEFAULT_HASHSIZE 65536 #define FLOW_DEFAULT_HASHSIZE 65536
@ -40,32 +39,51 @@ static int FlowUpdateSpareFlows(void);
int FlowSetProtoTimeout(uint8_t , uint32_t ,uint32_t ,uint32_t); int FlowSetProtoTimeout(uint8_t , uint32_t ,uint32_t ,uint32_t);
int FlowSetProtoEmergencyTimeout(uint8_t , uint32_t ,uint32_t ,uint32_t); int FlowSetProtoEmergencyTimeout(uint8_t , uint32_t ,uint32_t ,uint32_t);
static int FlowClearMemory(Flow *,uint8_t ); static int FlowClearMemory(Flow *,uint8_t );
static int FlowGetProtoMapping(uint8_t);
int FlowSetProtoFreeFunc(uint8_t, void (*Free)(void *)); int FlowSetProtoFreeFunc(uint8_t, void (*Free)(void *));
int FlowSetFlowStateFunc (uint8_t , int (*GetProtoState)(void *)); int FlowSetFlowStateFunc (uint8_t , int (*GetProtoState)(void *));
/** \brief Update the flows position in the queue's /** \brief Update the flows position in the queue's
* \param f Flow to requeue. * \param f Flow to requeue.
* \todo if we have a flow state func rely on that soly
* *
* In-use flows are either in the flow_new_q or flow_est_q lists. * In-use flows are in the flow_new_q, flow_est_q lists or flow_close_q lists.
*/ */
static void FlowUpdateQueue(Flow *f) void FlowUpdateQueue(Flow *f)
{ {
if (f->flags & FLOW_NEW_LIST) { if (f->flags & FLOW_NEW_LIST) {
/* in the new list -- we consider a flow no longer /* in the new list -- we consider a flow no longer
* new if we have seen at least 2 pkts in both ways. */ * new if we have seen at least 2 pkts in both ways. */
if (f->todstpktcnt && f->tosrcpktcnt) { if (f->todstpktcnt && f->tosrcpktcnt) {
FlowRequeue(f, &flow_new_q, &flow_est_q); FlowRequeue(f, &flow_new_q[f->protomap], &flow_est_q[f->protomap]);
f->flags |= FLOW_EST_LIST; /* transition */ f->flags |= FLOW_EST_LIST; /* transition */
f->flags &= ~FLOW_NEW_LIST; f->flags &= ~FLOW_NEW_LIST;
} else { } else {
FlowRequeue(f, &flow_new_q, &flow_new_q); FlowRequeue(f, &flow_new_q[f->protomap], &flow_new_q[f->protomap]);
} }
} else if (f->flags & FLOW_EST_LIST) { } else if (f->flags & FLOW_EST_LIST) {
if (flow_proto[f->protomap].GetProtoState != NULL) {
uint8_t state = flow_proto[f->protomap].GetProtoState(f->protoctx);
if (state == FLOW_STATE_CLOSED) {
f->flags |= FLOW_CLOSED_LIST; /* transition */
f->flags &= ~FLOW_EST_LIST;
//printf("FlowUpdateQueue %p was put into closing queue ts %"PRIuMAX"\n", f, (uintmax_t)f->lastts.tv_sec);
FlowRequeue(f, &flow_est_q[f->protomap], &flow_close_q[f->protomap]);
} else {
/* Pull and put back -- this way the flows on /* Pull and put back -- this way the flows on
* top of the list are least recently used. */ * top of the list are least recently used. */
FlowRequeue(f, &flow_est_q, &flow_est_q); FlowRequeue(f, &flow_est_q[f->protomap], &flow_est_q[f->protomap]);
}
} else {
/* Pull and put back -- this way the flows on
* top of the list are least recently used. */
FlowRequeue(f, &flow_est_q[f->protomap], &flow_est_q[f->protomap]);
}
} else if (f->flags & FLOW_CLOSED_LIST){
/* Pull and put back -- this way the flows on
* top of the list are least recently used. */
FlowRequeue(f, &flow_close_q[f->protomap], &flow_close_q[f->protomap]);
} }
} }
@ -111,70 +129,69 @@ static int FlowPrune (FlowQueue *q, struct timeval *ts)
/*set the timeout value according to the flow operating mode, flow's state /*set the timeout value according to the flow operating mode, flow's state
and protocol.*/ and protocol.*/
uint32_t timeout = 0; uint32_t timeout = 0;
uint8_t proto_map;
proto_map = FlowGetProtoMapping(f->proto);
if (flow_flags & FLOW_EMERGENCY) { if (flow_flags & FLOW_EMERGENCY) {
if (flow_proto[f->protomap].GetProtoState != NULL) {
if (flow_proto[proto_map].GetProtoState != NULL) { switch(flow_proto[f->protomap].GetProtoState(f->protoctx)) {
switch(flow_proto[proto_map].GetProtoState(f->stream)) {
case FLOW_STATE_NEW: case FLOW_STATE_NEW:
timeout = flow_proto[proto_map].emerg_new_timeout; timeout = flow_proto[f->protomap].emerg_new_timeout;
break; break;
case FLOW_STATE_ESTABLISHED: case FLOW_STATE_ESTABLISHED:
timeout = flow_proto[proto_map].emerg_est_timeout; timeout = flow_proto[f->protomap].emerg_est_timeout;
break; break;
case FLOW_STATE_CLOSED: case FLOW_STATE_CLOSED:
timeout = flow_proto[proto_map].emerg_closed_timeout; timeout = flow_proto[f->protomap].emerg_closed_timeout;
break; break;
} }
} else { } else {
if (f->flags & FLOW_EST_LIST) if (f->flags & FLOW_EST_LIST)
timeout = flow_proto[proto_map].emerg_est_timeout; timeout = flow_proto[f->protomap].emerg_est_timeout;
else else
timeout = flow_proto[proto_map].emerg_new_timeout; timeout = flow_proto[f->protomap].emerg_new_timeout;
} }
} else { /* impliet not emergency */
} else { if (flow_proto[f->protomap].GetProtoState != NULL) {
switch(flow_proto[f->protomap].GetProtoState(f->protoctx)) {
if (flow_proto[proto_map].GetProtoState != NULL) {
switch(flow_proto[proto_map].GetProtoState(f->stream)) {
case FLOW_STATE_NEW: case FLOW_STATE_NEW:
timeout = flow_proto[proto_map].new_timeout; timeout = flow_proto[f->protomap].new_timeout;
break; break;
case FLOW_STATE_ESTABLISHED: case FLOW_STATE_ESTABLISHED:
timeout = flow_proto[proto_map].est_timeout; timeout = flow_proto[f->protomap].est_timeout;
break; break;
case FLOW_STATE_CLOSED: case FLOW_STATE_CLOSED:
timeout = flow_proto[proto_map].closed_timeout; timeout = flow_proto[f->protomap].closed_timeout;
break; break;
} }
} else { } else {
if (f->flags & FLOW_EST_LIST) if (f->flags & FLOW_EST_LIST)
timeout = flow_proto[proto_map].est_timeout; timeout = flow_proto[f->protomap].est_timeout;
else else
timeout = flow_proto[proto_map].new_timeout; timeout = flow_proto[f->protomap].new_timeout;
} }
} }
DEBUGPRINT("got lock, now check: %" PRId64 "+%" PRIu32 "=(%" PRId64 ") < %" PRId64 "", f->lastts.tv_sec, DEBUGPRINT("got lock, now check: %" PRId64 "+%" PRIu32 "=(%" PRId64 ") < %" PRId64 "", f->lastts.tv_sec,
timeout, f->lastts.tv_sec + timeout, ts->tv_sec); timeout, f->lastts.tv_sec + timeout, ts->tv_sec);
/** never prune a flow that is used by a packet or stream msg /* do the timeout check */
* we are currently processing in one of the threads */ if ((f->lastts.tv_sec + timeout) >= ts->tv_sec) {
if (f->use_cnt > 0) {
mutex_unlock(&f->fb->m); mutex_unlock(&f->fb->m);
mutex_unlock(&f->m); mutex_unlock(&f->m);
return 0; return 0;
} }
/* do the timeout check */ /** never prune a flow that is used by a packet or stream msg
if ((f->lastts.tv_sec + timeout) >= ts->tv_sec) { * we are currently processing in one of the threads */
if (f->use_cnt > 0) {
printf("FlowPrune: timed out but use_cnt > 0: %"PRIu16", %p, proto %"PRIu8"\n", f->use_cnt, f, f->proto);
mutex_unlock(&f->fb->m); mutex_unlock(&f->fb->m);
mutex_unlock(&f->m); mutex_unlock(&f->m);
return 0; return 0;
} }
//printf("timed out %" PRIuMAX "+%" PRIu32 "=(%" PRIuMAX ") < %" PRIuMAX ": %p, proto %"PRIu8"\n", (uintmax_t)f->lastts.tv_sec,
// timeout, (uintmax_t)(f->lastts.tv_sec + timeout), (uintmax_t)ts->tv_sec, f, f->proto);
/* remove from the hash */ /* remove from the hash */
if (f->hprev) if (f->hprev)
f->hprev->hnext = f->hnext; f->hprev->hnext = f->hnext;
@ -189,7 +206,7 @@ static int FlowPrune (FlowQueue *q, struct timeval *ts)
mutex_unlock(&f->fb->m); mutex_unlock(&f->fb->m);
f->fb = NULL; f->fb = NULL;
FlowClearMemory (f, proto_map); FlowClearMemory (f, f->protomap);
/* move to spare list */ /* move to spare list */
FlowRequeue(f, q, &flow_spare_q); FlowRequeue(f, q, &flow_spare_q);
@ -396,20 +413,26 @@ void FlowInitConfig (char quiet)
* \warning Not thread safe */ * \warning Not thread safe */
void FlowPrintQueueInfo (void) void FlowPrintQueueInfo (void)
{ {
int i;
printf("* Flow Queue info:\n"); printf("* Flow Queue info:\n");
printf(" - SPARE %" PRIu32 " (", flow_spare_q.len); printf(" - SPARE %" PRIu32 " (", flow_spare_q.len);
#ifdef DBG_PERF #ifdef DBG_PERF
printf("flow_spare_q.dbg_maxlen %" PRIu32 ")\n", flow_spare_q.dbg_maxlen); printf("flow_spare_q.dbg_maxlen %" PRIu32 ")\n", flow_spare_q.dbg_maxlen);
#endif #endif
printf(" - NEW %" PRIu32 " (", flow_new_q.len); for (i = 0; i < FLOW_PROTO_MAX; i++) {
printf(" - NEW %" PRIu32 " (", flow_new_q[i].len);
#ifdef DBG_PERF #ifdef DBG_PERF
printf("flow_new_q.dbg_maxlen %" PRIu32 ")\n", flow_new_q.dbg_maxlen); printf("flow_new_q.dbg_maxlen %" PRIu32 ")\n", flow_new_q[i].dbg_maxlen);
#endif #endif
printf(" - ESTABLISHED %" PRIu32 " (", flow_est_q.len); printf(" - ESTABLISHED %" PRIu32 " (", flow_est_q[i].len);
#ifdef DBG_PERF #ifdef DBG_PERF
printf("flow_est_q.dbg_maxlen %" PRIu32 ")\n", flow_est_q.dbg_maxlen); printf("flow_est_q.dbg_maxlen %" PRIu32 ")\n", flow_est_q[i].dbg_maxlen);
#endif #endif
printf(" - CLOSING %" PRIu32 " (", flow_close_q[i].len);
#ifdef DBG_PERF
printf("flow_closing_q.dbg_maxlen %" PRIu32 ")\n", flow_close_q[i].dbg_maxlen);
#endif
}
#ifdef FLOWBITS_STATS #ifdef FLOWBITS_STATS
printf("* Flowbits added: %" PRIu32 ", removed: %" PRIu32 ", ", flowbits_added, flowbits_removed); printf("* Flowbits added: %" PRIu32 ", removed: %" PRIu32 ", ", flowbits_added, flowbits_removed);
printf("max memory usage: %" PRIu32 "\n", flowbits_memuse_max); printf("max memory usage: %" PRIu32 "\n", flowbits_memuse_max);
@ -420,20 +443,28 @@ void FlowPrintQueueInfo (void)
* \warning Not thread safe */ * \warning Not thread safe */
void FlowShutdown(void) { void FlowShutdown(void) {
Flow *f; Flow *f;
int i;
while((f = FlowDequeue(&flow_spare_q))) { while((f = FlowDequeue(&flow_spare_q))) {
FlowFree(f); FlowFree(f);
} }
while((f = FlowDequeue(&flow_new_q))) { for (i = 0; i < FLOW_PROTO_MAX; i++) {
while((f = FlowDequeue(&flow_new_q[i]))) {
uint8_t proto_map = FlowGetProtoMapping(f->proto); uint8_t proto_map = FlowGetProtoMapping(f->proto);
FlowClearMemory(f, proto_map); FlowClearMemory(f, proto_map);
FlowFree(f); FlowFree(f);
} }
while((f = FlowDequeue(&flow_est_q))) { while((f = FlowDequeue(&flow_est_q[i]))) {
uint8_t proto_map = FlowGetProtoMapping(f->proto); uint8_t proto_map = FlowGetProtoMapping(f->proto);
FlowClearMemory(f, proto_map); FlowClearMemory(f, proto_map);
FlowFree(f); FlowFree(f);
} }
while((f = FlowDequeue(&flow_close_q[i]))) {
uint8_t proto_map = FlowGetProtoMapping(f->proto);
FlowClearMemory(f, proto_map);
FlowFree(f);
}
}
free(flow_hash); free(flow_hash);
flow_memuse -= flow_config.hash_size * sizeof(FlowBucket); flow_memuse -= flow_config.hash_size * sizeof(FlowBucket);
@ -458,7 +489,7 @@ void *FlowManagerThread(void *td)
{ {
ThreadVars *th_v = (ThreadVars *)td; ThreadVars *th_v = (ThreadVars *)td;
struct timeval ts; struct timeval ts;
uint32_t established_cnt = 0, new_cnt = 0, nowcnt; uint32_t established_cnt = 0, new_cnt = 0, closing_cnt = 0, nowcnt;
uint32_t sleeping = 0; uint32_t sleeping = 0;
uint8_t emerg = FALSE; uint8_t emerg = FALSE;
@ -484,24 +515,31 @@ void *FlowManagerThread(void *td)
DEBUGPRINT("ts %" PRId64 "", ts.tv_sec); DEBUGPRINT("ts %" PRId64 "", ts.tv_sec);
/* see if we still have enough spare flows */ /* see if we still have enough spare flows */
if (!(FlowUpdateSpareFlows()) && emerg == TRUE) { FlowUpdateSpareFlows();
/*timeout_new = flow_config.emerg_timeout_new;
timeout_est = flow_config.emerg_timeout_est;*/ int i;
for (i = 0; i < FLOW_PROTO_MAX; i++) {
/* prune closing list */
nowcnt = FlowPruneFlows(&flow_close_q[i], &ts);
if (nowcnt) {
DEBUGPRINT("Pruned %" PRIu32 " closing flows...\n", nowcnt);
closing_cnt += nowcnt;
} }
/* prune new list */ /* prune new list */
nowcnt = FlowPruneFlows(&flow_new_q, &ts); nowcnt = FlowPruneFlows(&flow_new_q[i], &ts);
if (nowcnt) { if (nowcnt) {
DEBUGPRINT("Pruned %" PRIu32 " new flows...\n", nowcnt); DEBUGPRINT("Pruned %" PRIu32 " new flows...\n", nowcnt);
new_cnt += nowcnt; new_cnt += nowcnt;
} }
/* prune established list */ /* prune established list */
nowcnt = FlowPruneFlows(&flow_est_q, &ts); nowcnt = FlowPruneFlows(&flow_est_q[i], &ts);
if (nowcnt) { if (nowcnt) {
DEBUGPRINT("Pruned %" PRIu32 " established flows...\n", nowcnt); DEBUGPRINT("Pruned %" PRIu32 " established flows...\n", nowcnt);
established_cnt += nowcnt; established_cnt += nowcnt;
} }
}
sleeping = 0; sleeping = 0;
@ -589,26 +627,6 @@ void FlowInitFlowProto(void) {
flow_proto[FLOW_PROTO_ICMP].GetProtoState = NULL; flow_proto[FLOW_PROTO_ICMP].GetProtoState = NULL;
} }
/**
* \brief Function to map the protocol to the defined FLOW_PROTO_* enumeration.
*
* \param proto protocol which is needed to be mapped
*/
static int FlowGetProtoMapping(uint8_t proto) {
switch (proto) {
case IPPROTO_TCP:
return FLOW_PROTO_TCP;
case IPPROTO_UDP:
return FLOW_PROTO_UDP;
case IPPROTO_ICMP:
return FLOW_PROTO_ICMP;
default:
return FLOW_PROTO_DEFAULT;
}
}
/** /**
* \brief Function clear the flow memory before queueing it to spare flow * \brief Function clear the flow memory before queueing it to spare flow
* queue. * queue.
@ -620,11 +638,10 @@ static int FlowGetProtoMapping(uint8_t proto) {
static int FlowClearMemory(Flow* f, uint8_t proto_map) { static int FlowClearMemory(Flow* f, uint8_t proto_map) {
/* call the protocol specific free function if we have one */ /* call the protocol specific free function if we have one */
if (flow_proto[proto_map].Freefunc != NULL) { if (flow_proto[proto_map].Freefunc != NULL) {
flow_proto[proto_map].Freefunc(f->stream); flow_proto[proto_map].Freefunc(f->protoctx);
} }
f->stream = NULL; f->protoctx = NULL;
//memset(f, 0, sizeof(Flow));
CLEAR_FLOW(f); CLEAR_FLOW(f);
return 1; return 1;
} }
@ -705,6 +722,9 @@ int FlowSetProtoEmergencyTimeout(uint8_t proto, uint32_t emerg_new_timeout, uint
return 1; return 1;
} }
#ifdef UNITTESTS
#include "stream-tcp-private.h"
/** /**
* \test Test the setting of the per protocol timeouts. * \test Test the setting of the per protocol timeouts.
* *
@ -813,7 +833,7 @@ static int FlowTestPrune(Flow *f, struct timeval *ts) {
return 0; return 0;
} }
if (f->stream != NULL){ if (f->protoctx != NULL){
printf("Failed in freeing the TcpSession\n"); printf("Failed in freeing the TcpSession\n");
return 0; return 0;
} }
@ -841,7 +861,7 @@ static int FlowTest03 (void) {
TimeGet(&ts); TimeGet(&ts);
f.lastts.tv_sec = ts.tv_sec - 5000; f.lastts.tv_sec = ts.tv_sec - 5000;
f.stream = &ssn; f.protoctx = &ssn;
f.fb = &fb; f.fb = &fb;
f.proto = IPPROTO_TCP; f.proto = IPPROTO_TCP;
@ -885,7 +905,7 @@ static int FlowTest04 (void) {
ssn.server = client; ssn.server = client;
ssn.state = TCP_ESTABLISHED; ssn.state = TCP_ESTABLISHED;
f.lastts.tv_sec = ts.tv_sec - 5000; f.lastts.tv_sec = ts.tv_sec - 5000;
f.stream = &ssn; f.protoctx = &ssn;
f.fb = &fb; f.fb = &fb;
f.proto = IPPROTO_TCP; f.proto = IPPROTO_TCP;
@ -918,7 +938,7 @@ static int FlowTest05 (void) {
TimeGet(&ts); TimeGet(&ts);
ssn.state = TCP_SYN_SENT; ssn.state = TCP_SYN_SENT;
f.lastts.tv_sec = ts.tv_sec - 300; f.lastts.tv_sec = ts.tv_sec - 300;
f.stream = &ssn; f.protoctx = &ssn;
f.fb = &fb; f.fb = &fb;
f.proto = IPPROTO_TCP; f.proto = IPPROTO_TCP;
f.flags = FLOW_EMERGENCY; f.flags = FLOW_EMERGENCY;
@ -963,7 +983,7 @@ static int FlowTest06 (void) {
ssn.server = client; ssn.server = client;
ssn.state = TCP_ESTABLISHED; ssn.state = TCP_ESTABLISHED;
f.lastts.tv_sec = ts.tv_sec - 5000; f.lastts.tv_sec = ts.tv_sec - 5000;
f.stream = &ssn; f.protoctx = &ssn;
f.fb = &fb; f.fb = &fb;
f.proto = IPPROTO_TCP; f.proto = IPPROTO_TCP;
f.flags = FLOW_EMERGENCY; f.flags = FLOW_EMERGENCY;
@ -974,15 +994,18 @@ static int FlowTest06 (void) {
return 1; return 1;
} }
#endif /* UNITTESTS */
/** /**
* \brief Function to register the Flow Unitests. * \brief Function to register the Flow Unitests.
*/ */
void FlowRegisterTests (void) { void FlowRegisterTests (void) {
#ifdef UNITTESTS
UtRegisterTest("FlowTest01 -- Protocol Specific Timeouts", FlowTest01, 1); UtRegisterTest("FlowTest01 -- Protocol Specific Timeouts", FlowTest01, 1);
UtRegisterTest("FlowTest02 -- Setting Protocol Specific Free Function", FlowTest02, 1); UtRegisterTest("FlowTest02 -- Setting Protocol Specific Free Function", FlowTest02, 1);
UtRegisterTest("FlowTest03 -- Timeout a flow having fresh TcpSession", FlowTest03, 1); UtRegisterTest("FlowTest03 -- Timeout a flow having fresh TcpSession", FlowTest03, 1);
UtRegisterTest("FlowTest04 -- Timeout a flow having TcpSession with segments", FlowTest04, 1); UtRegisterTest("FlowTest04 -- Timeout a flow having TcpSession with segments", FlowTest04, 1);
UtRegisterTest("FlowTest05 -- Timeout a flow in emergency having fresh TcpSession", FlowTest05, 1); UtRegisterTest("FlowTest05 -- Timeout a flow in emergency having fresh TcpSession", FlowTest05, 1);
UtRegisterTest("FlowTest06 -- Timeout a flow in emergency having TcpSession with segments", FlowTest06, 1); UtRegisterTest("FlowTest06 -- Timeout a flow in emergency having TcpSession with segments", FlowTest06, 1);
#endif /* UNITTESTS */
} }

23
src/flow.h
Unescape Escape View File

@ -67,9 +67,15 @@ typedef struct Flow_
uint32_t tosrcpktcnt; uint32_t tosrcpktcnt;
uint64_t bytecnt; uint64_t bytecnt;
void *stream; /** mapping to Flow's protocol specific protocols for timeouts
uint16_t use_cnt; /** how many pkts and stream msgs are and state and free functions. */
using the flow *right now* */ uint8_t protomap;
/** protocol specific data pointer, e.g. for TcpSession */
void *protoctx;
/** how many pkts and stream msgs are using the flow *right now* */
uint16_t use_cnt;
pthread_mutex_t m; pthread_mutex_t m;
@ -84,16 +90,6 @@ typedef struct Flow_
struct FlowBucket_ *fb; struct FlowBucket_ *fb;
} Flow; } Flow;
enum {
FLOW_PROTO_DEFAULT = 0,
FLOW_PROTO_TCP,
FLOW_PROTO_UDP,
FLOW_PROTO_ICMP,
/* should be last */
FLOW_PROTO_MAX,
};
enum { enum {
FLOW_STATE_NEW = 0, FLOW_STATE_NEW = 0,
FLOW_STATE_ESTABLISHED, FLOW_STATE_ESTABLISHED,
@ -126,6 +122,7 @@ int FlowSetProtoTimeout(uint8_t ,uint32_t ,uint32_t ,uint32_t);
int FlowSetProtoEmergencyTimeout(uint8_t ,uint32_t ,uint32_t ,uint32_t); int FlowSetProtoEmergencyTimeout(uint8_t ,uint32_t ,uint32_t ,uint32_t);
int FlowSetProtoFreeFunc (uint8_t , void (*Free)(void *)); int FlowSetProtoFreeFunc (uint8_t , void (*Free)(void *));
int FlowSetFlowStateFunc (uint8_t , int (*GetProtoState)(void *)); int FlowSetFlowStateFunc (uint8_t , int (*GetProtoState)(void *));
void FlowUpdateQueue(Flow *);
#endif /* __FLOW_H__ */ #endif /* __FLOW_H__ */

34
src/stream-tcp-private.h
Unescape Escape View File

@ -11,33 +11,34 @@ typedef struct TcpSegment_ {
} TcpSegment; } TcpSegment;
typedef struct TcpStream_ { typedef struct TcpStream_ {
uint32_t isn; /* initial sequence number */ uint32_t isn; /**< initial sequence number */
uint32_t next_seq; /* next expected sequence number */ uint32_t next_seq; /**< next expected sequence number */
uint32_t last_ack; /* last ack'd sequence number */ uint32_t last_ack; /**< last ack'd sequence number in this stream */
uint32_t next_win; /* next max seq within window */ uint32_t next_win; /**< next max seq within window */
uint8_t wscale; uint32_t window; /**< current window setting */
uint16_t window; uint8_t wscale; /**< wscale setting in this direction */
/* reassembly */ /* reassembly */
uint32_t ra_base_seq; /* reassembled seq. We've reassembled up to this point. */ uint32_t ra_base_seq; /**< reassembled seq. We've reassembled up to this point. */
TcpSegment *seg_list; TcpSegment *seg_list; /**< list of TCP segments that are not yet (fully) used in reassembly */
uint8_t os_policy; /* target based OS policy used for reassembly and handling packets*/ uint8_t os_policy; /**< target based OS policy used for reassembly and handling packets*/
} TcpStream; } TcpStream;
/* from /usr/include/netinet/tcp.h */ /* from /usr/include/netinet/tcp.h */
enum enum
{ {
TCP_ESTABLISHED = 1, TCP_NONE,
TCP_LISTEN,
TCP_SYN_SENT, TCP_SYN_SENT,
TCP_SYN_RECV, TCP_SYN_RECV,
TCP_ESTABLISHED,
TCP_FIN_WAIT1, TCP_FIN_WAIT1,
TCP_FIN_WAIT2, TCP_FIN_WAIT2,
TCP_TIME_WAIT, TCP_TIME_WAIT,
TCP_CLOSED,
TCP_CLOSE_WAIT,
TCP_LAST_ACK, TCP_LAST_ACK,
TCP_LISTEN, TCP_CLOSE_WAIT,
TCP_CLOSING /* now a valid state */ TCP_CLOSING,
TCP_CLOSED,
}; };
#define STREAMTCP_FLAG_MIDSTREAM 0x01 /*Flag for mid stream session*/ #define STREAMTCP_FLAG_MIDSTREAM 0x01 /*Flag for mid stream session*/
@ -53,9 +54,10 @@ enum
typedef struct TcpSession_ { typedef struct TcpSession_ {
uint8_t state; uint8_t state;
uint8_t flags;
uint16_t alproto; /**< application level protocol */
TcpStream server; TcpStream server;
TcpStream client; TcpStream client;
void **l7data; void **aldata; /**< application level storage ptrs */
u_int8_t flags;
} TcpSession; } TcpSession;
#endif /* __STREAM_TCP_PRIVATE_H__ */ #endif /* __STREAM_TCP_PRIVATE_H__ */

763
src/stream-tcp-reassemble.c
View File

File diff suppressed because it is too large Load Diff

1
src/stream-tcp-reassemble.h
Unescape Escape View File

@ -31,6 +31,7 @@ enum
int StreamTcpReassembleHandleSegment(TcpSession *, TcpStream *, Packet *); int StreamTcpReassembleHandleSegment(TcpSession *, TcpStream *, Packet *);
int StreamTcpReassembleInit(void); int StreamTcpReassembleInit(void);
void StreamTcpReassembleFree(void);
void StreamTcpReassembleRegisterTests(void); void StreamTcpReassembleRegisterTests(void);
void StreamTcpCreateTestPacket(u_int8_t *, u_int8_t, u_int8_t); void StreamTcpCreateTestPacket(u_int8_t *, u_int8_t, u_int8_t);

493
src/stream-tcp.c
View File

File diff suppressed because it is too large Load Diff

1
src/stream-tcp.h
Unescape Escape View File

@ -16,6 +16,7 @@ typedef struct TcpStreamCnf_ {
TcpStreamCnf stream_config; TcpStreamCnf stream_config;
void TmModuleStreamTcpRegister (void); void TmModuleStreamTcpRegister (void);
void StreamTcpInitConfig (char); void StreamTcpInitConfig (char);
void StreamTcpFreeConfig(char);
void StreamTcpRegisterTests (void); void StreamTcpRegisterTests (void);
#endif /* __STREAM_TCP_H__ */ #endif /* __STREAM_TCP_H__ */

17
src/stream.c
Unescape Escape View File

@ -8,6 +8,10 @@
#include "util-pool.h" #include "util-pool.h"
static pthread_mutex_t stream_pool_memuse_mutex;
static uint64_t stream_pool_memuse = 0;
static uint64_t stream_pool_memcnt = 0;
static StreamMsgQueue stream_q; static StreamMsgQueue stream_q;
/* per queue setting */ /* per queue setting */
@ -25,6 +29,11 @@ void *StreamMsgAlloc(void *null) {
return NULL; return NULL;
memset(s, 0, sizeof(StreamMsg)); memset(s, 0, sizeof(StreamMsg));
mutex_lock(&stream_pool_memuse_mutex);
stream_pool_memuse += sizeof(StreamMsg);
stream_pool_memcnt ++;
mutex_unlock(&stream_pool_memuse_mutex);
return s; return s;
} }
@ -140,6 +149,14 @@ void StreamMsgQueuesInit(void) {
stream_msg_pool = PoolInit(5000,250,StreamMsgAlloc,NULL,StreamMsgFree); stream_msg_pool = PoolInit(5000,250,StreamMsgAlloc,NULL,StreamMsgFree);
if (stream_msg_pool == NULL) if (stream_msg_pool == NULL)
exit(1); /* XXX */ exit(1); /* XXX */
pthread_mutex_init(&stream_pool_memuse_mutex, NULL);
}
void StreamMsgQueuesDeinit(void) {
PoolFree(stream_msg_pool);
printf("StreamMsgQueuesDeinit: stream_pool_memuse %"PRIu64", stream_pool_memcnt %"PRIu64"\n", stream_pool_memuse, stream_pool_memcnt);
} }
StreamMsgQueue *StreamMsgQueueGetByPort(uint16_t port) { StreamMsgQueue *StreamMsgQueueGetByPort(uint16_t port) {

1
src/stream.h
Unescape Escape View File

@ -49,6 +49,7 @@ typedef struct StreamMsgQueue_ {
/* prototypes */ /* prototypes */
void StreamMsgQueuesInit(void); void StreamMsgQueuesInit(void);
void StreamMsgQueuesDeinit(void);
StreamMsg *StreamMsgGetFromPool(void); StreamMsg *StreamMsgGetFromPool(void);
void StreamMsgReturnToPool(StreamMsg *); void StreamMsgReturnToPool(StreamMsg *);

8
src/tmqh-flow.c
Unescape Escape View File

@ -27,6 +27,7 @@
typedef struct TmqhFlowCtx_ { typedef struct TmqhFlowCtx_ {
uint16_t size; uint16_t size;
uint16_t *queues; uint16_t *queues;
uint16_t last;
} TmqhFlowCtx; } TmqhFlowCtx;
Packet *TmqhInputFlow(ThreadVars *t); Packet *TmqhInputFlow(ThreadVars *t);
@ -167,7 +168,12 @@ void TmqhOutputFlow(ThreadVars *tv, Packet *p)
uint16_t idx = addr % ctx->size; uint16_t idx = addr % ctx->size;
qid = ctx->queues[idx]; qid = ctx->queues[idx];
} else { } else {
qid = ctx->queues[0]; ctx->last++;
if (ctx->last == ctx->size)
ctx->last = 0;
qid = ctx->queues[ctx->last];
} }
PacketQueue *q = &trans_q[qid]; PacketQueue *q = &trans_q[qid];

13
src/util-pool.c
Unescape Escape View File

@ -113,6 +113,11 @@ void *PoolGet(Pool *p) {
} else { } else {
if (p->allocated < p->max_buckets) { if (p->allocated < p->max_buckets) {
p->allocated++; p->allocated++;
p->outstanding++;
if (p->outstanding > p->max_outstanding)
p->max_outstanding = p->outstanding;
return p->Alloc(p->AllocData); return p->Alloc(p->AllocData);
} else { } else {
return NULL; return NULL;
@ -121,6 +126,9 @@ void *PoolGet(Pool *p) {
void *ptr = pb->data; void *ptr = pb->data;
pb->data = NULL; pb->data = NULL;
p->outstanding++;
if (p->outstanding > p->max_outstanding)
p->max_outstanding = p->outstanding;
return ptr; return ptr;
} }
@ -141,9 +149,14 @@ void PoolReturn(Pool *p, void *data) {
p->alloc_list_size++; p->alloc_list_size++;
pb->data = data; pb->data = data;
p->outstanding--;
return; return;
} }
void PoolPrintSaturation(Pool *p) {
printf("PoolPrintSaturation: Pool %p is using %"PRIu32" out of %"PRIu32" items (%02.1f%%), max %"PRIu32" (%02.1f%%): pool struct memory %"PRIu64".\n", p, p->outstanding, p->max_buckets, (float)(p->outstanding/(float)(p->max_buckets))*100, p->max_outstanding, (float)(p->max_outstanding/(float)(p->max_buckets))*100, (uint64_t)(p->max_buckets * sizeof(PoolBucket)));
}
/* /*
* ONLY TESTS BELOW THIS COMMENT * ONLY TESTS BELOW THIS COMMENT
*/ */

4
src/util-pool.h
Unescape Escape View File

@ -23,12 +23,16 @@ typedef struct Pool_ {
void *(*Alloc)(void *); void *(*Alloc)(void *);
void *AllocData; void *AllocData;
void (*Free)(void *); void (*Free)(void *);
uint32_t outstanding;
uint32_t max_outstanding;
} Pool; } Pool;
/* prototypes */ /* prototypes */
Pool* PoolInit(uint32_t, uint32_t, void *(*Alloc)(void *), void *, void (*Free)(void *)); Pool* PoolInit(uint32_t, uint32_t, void *(*Alloc)(void *), void *, void (*Free)(void *));
void PoolFree(Pool *); void PoolFree(Pool *);
void PoolPrint(Pool *); void PoolPrint(Pool *);
void PoolPrintSaturation(Pool *p);
void *PoolGet(Pool *); void *PoolGet(Pool *);
void PoolReturn(Pool *, void *); void PoolReturn(Pool *, void *);

15
src/util-time.c
Unescape Escape View File

@ -4,6 +4,8 @@
#include "detect.h" #include "detect.h"
#include "threads.h" #include "threads.h"
//#define DEBUG
static struct timeval current_time = { 0,0 }; static struct timeval current_time = { 0,0 };
static pthread_mutex_t current_time_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t current_time_mutex = PTHREAD_MUTEX_INITIALIZER;
static char live = TRUE; static char live = TRUE;
@ -26,8 +28,11 @@ void TimeSet(struct timeval *tv) {
mutex_lock(&current_time_mutex); mutex_lock(&current_time_mutex);
current_time.tv_sec = tv->tv_sec; current_time.tv_sec = tv->tv_sec;
current_time.tv_usec = tv->tv_usec; current_time.tv_usec = tv->tv_usec;
//printf("TimeSet: time set to %" PRIu64 " sec, %" PRIu64 " usec\n",
// current_time.tv_sec, current_time.tv_usec); #ifdef DEBUG
printf("TimeSet: time set to %" PRIuMAX " sec, %" PRIuMAX " usec\n",
(uintmax_t)current_time.tv_sec, (uintmax_t)current_time.tv_usec);
#endif
mutex_unlock(&current_time_mutex); mutex_unlock(&current_time_mutex);
} }
@ -45,7 +50,9 @@ void TimeGet(struct timeval *tv) {
mutex_unlock(&current_time_mutex); mutex_unlock(&current_time_mutex);
} }
//printf("TimeGet: time we got is %" PRIu64 " sec, %" PRIu64 " usec\n", #ifdef DEBUG
// tv->tv_sec, tv->tv_usec); printf("TimeGet: time we got is %" PRIuMAX " sec, %" PRIuMAX " usec\n",
(uintmax_t)tv->tv_sec, (uintmax_t)tv->tv_usec);
#endif
} }

Write Preview
Loading…
Cancel
Save