From b102ea2123291efba6ccf226600ead0e4fed58b3 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 10 Sep 2009 23:54:04 +0200 Subject: [PATCH] Big update: - Implement "closing" state in flow. - Add protocol specific timeouts. - Lots of stream tracking updates, fixing a lot of out of window issues. - Stream reassembly fixes. - Implement a new IDS runmode with 4 stream and detect threads. - Added a BUG_ON macro that aborts the engine if the expression is true. - Better balance the flow queue handler for traffic that doesn't have flow (like icmp currently). - Simplify application level protocol in the Tcp Session. - Add some debugging memory counters. --- src/app-layer-detect-proto.c | 102 ++--- src/app-layer-http.c | 73 +++- src/app-layer-http.h | 1 + src/app-layer-parser.c | 61 ++- src/app-layer-parser.h | 3 + src/app-layer-tls.c | 16 +- src/eidps-common.h | 3 + src/eidps.c | 319 +++++++++++++- src/flow-hash.c | 6 +- src/flow-private.h | 44 +- src/flow-util.c | 22 + src/flow-util.h | 6 +- src/flow.c | 211 +++++---- src/flow.h | 23 +- src/stream-tcp-private.h | 34 +- src/stream-tcp-reassemble.c | 801 +++++++++++++++++++---------------- src/stream-tcp-reassemble.h | 1 + src/stream-tcp.c | 549 ++++++++++++++---------- src/stream-tcp.h | 1 + src/stream.c | 17 + src/stream.h | 1 + src/tmqh-flow.c | 8 +- src/util-pool.c | 13 + src/util-pool.h | 4 + src/util-time.c | 19 +- 25 files changed, 1493 insertions(+), 845 deletions(-) diff --git a/src/app-layer-detect-proto.c b/src/app-layer-detect-proto.c index c2137beec1..2dc4008435 100644 --- a/src/app-layer-detect-proto.c +++ b/src/app-layer-detect-proto.c @@ -34,7 +34,7 @@ #include "app-layer-protos.h" #include "app-layer-parser.h" -#define INSPECT_BYTES 32 +#define INSPECT_BYTES 32 #define ALP_DETECT_MAX 256 typedef struct AlpProtoDetectDirectionThread_ { @@ -61,25 +61,7 @@ typedef struct AlpProtoDetectCtx_ { static AlpProtoDetectCtx alp_proto_ctx; static AlpProtoDetectThreadCtx alp_proto_tctx; -static uint8_t al_proto_id = 0; -/** \brief data stored in the stream */ -typedef struct AppLayerDetectProtoData_ { - uint8_t proto; -} AppLayerDetectProtoData; - -static Pool *al_detect_proto_pool = NULL; - -void *AppLayerDetectProtoAlloc(void *null) { - AppLayerDetectProtoData *d = malloc(sizeof(AppLayerDetectProtoData)); - if (d == NULL) { - return NULL; - } - - d->proto = ALPROTO_UNKNOWN; - return d; -} -#define AppLayerDetectProtoFree free void AlpProtoInit(AlpProtoDetectCtx *ctx) { memset(ctx, 0x00, sizeof(AlpProtoDetectCtx)); @@ -108,7 +90,7 @@ void AlpProtoDestroy(AlpProtoDetectCtx *ctx) { * \param offset Offset setting for the content. E.g. 4 mean that the content has to match after the first 4 bytes of the stream. * \param flags Set STREAM_TOCLIENT or STREAM_TOSERVER for the direction in which to try to match the content. */ -void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint8_t al_proto, char *content, uint16_t depth, uint16_t offset, uint8_t flags) { +void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint16_t al_proto, char *content, uint16_t depth, uint16_t offset, uint8_t flags) { DetectContentData *cd = DetectContentParse(content); if (cd == NULL) { return; @@ -159,13 +141,6 @@ void AlpProtoFinalizeGlobal(AlpProtoDetectCtx *ctx) { } void AppLayerDetectProtoThreadInit(void) { - al_proto_id = StreamL7RegisterModule(); - - al_detect_proto_pool = PoolInit(262144, 32768, AppLayerDetectProtoAlloc, NULL, AppLayerDetectProtoFree); - if (al_detect_proto_pool == NULL) { - exit(1); - } - AlpProtoInit(&alp_proto_ctx); /** \todo register these in the protocol parser api */ @@ -267,12 +242,17 @@ end: if (dir->mpm_ctx.Cleanup != NULL) { dir->mpm_ctx.Cleanup(&tdir->mpm_ctx); } - +//#define DEBUG #ifdef DEBUG printf("AppLayerDetectGetProto: returning %" PRIu16 " (%s): ", proto, flags & STREAM_TOCLIENT ? "TOCLIENT" : "TOSERVER"); switch (proto) { case ALPROTO_HTTP: - printf("HTTP\n"); + printf("HTTP: "); + /* print the first 32 bytes */ + if (buflen > 0) { + PrintRawUriFp(stdout,buf,(buflen>32)?32:buflen); + } + printf("\n"); break; case ALPROTO_FTP: printf("FTP\n"); @@ -283,6 +263,9 @@ end: case ALPROTO_SSH: printf("SSH\n"); break; + case ALPROTO_TLS: + printf("TLS\n"); + break; case ALPROTO_IMAP: printf("IMAP\n"); break; @@ -297,8 +280,12 @@ end: break; case ALPROTO_UNKNOWN: default: - printf("UNKNOWN\n"); - PrintRawDataFp(stdout,buf,buflen); + printf("UNKNOWN (%u): cnt was %u (", proto, cnt); + /* print the first 32 bytes */ + if (buflen > 0) { + PrintRawUriFp(stdout,buf,(buflen>32)?32:buflen); + } + printf(")\n"); break; } #endif @@ -309,9 +296,7 @@ void *AppLayerDetectProtoThread(void *td) { ThreadVars *tv = (ThreadVars *)td; char run = TRUE; - AppLayerDetectProtoData *al_proto = NULL; - char store = 0; - void *al_data_ptr = NULL; + uint16_t alproto = ALPROTO_UNKNOWN; /* get the stream msg queue for this thread */ StreamMsgQueue *stream_q = StreamMsgQueueGetByPort(0); @@ -327,20 +312,13 @@ void *AppLayerDetectProtoThread(void *td) StreamMsg *smsg = StreamMsgGetFromQueue(stream_q); if (smsg != NULL) { mutex_lock(&smsg->flow->m); - TcpSession *ssn = smsg->flow->stream; + TcpSession *ssn = smsg->flow->protoctx; if (ssn != NULL) { - if (ssn->l7data == NULL) { - /* XXX we can use a pool here, - or make it part of the stream setup */ - StreamL7DataPtrInit(ssn,StreamL7GetStorageSize()); - } - if (ssn->l7data != NULL) { - al_data_ptr = ssn->l7data[al_proto_id]; - } + alproto = ssn->alproto; } mutex_unlock(&smsg->flow->m); - if (ssn != NULL && ssn->l7data != NULL) { + if (ssn != NULL) { if (smsg->flags & STREAM_START) { //printf("L7AppDetectThread: stream initializer (len %" PRIu32 " (%" PRIu32 "))\n", smsg->data.data_len, MSG_DATA_SIZE); @@ -348,17 +326,18 @@ void *AppLayerDetectProtoThread(void *td) //PrintRawDataFp(stdout, smsg->init.data, smsg->init.data_len); //printf("=> Init Stream Data -- end\n"); - if (al_data_ptr == NULL) { - al_proto = (AppLayerDetectProtoData *)PoolGet(al_detect_proto_pool); - if (al_proto != NULL) { - al_proto->proto = AppLayerDetectGetProto(&alp_proto_ctx, &alp_proto_tctx, smsg->data.data, smsg->data.data_len, smsg->flags); - store = 1; + alproto = AppLayerDetectGetProto(&alp_proto_ctx, &alp_proto_tctx, smsg->data.data, smsg->data.data_len, smsg->flags); + if (alproto != ALPROTO_UNKNOWN) { + /* store the proto and setup the L7 data array */ + mutex_lock(&smsg->flow->m); + StreamL7DataPtrInit(ssn,StreamL7GetStorageSize()); + ssn->alproto = alproto; + mutex_unlock(&smsg->flow->m); - AppLayerParse(smsg->flow, al_proto->proto, smsg->flags, smsg->data.data, smsg->data.data_len); - } + AppLayerParse(smsg->flow, alproto, smsg->flags, smsg->data.data, smsg->data.data_len); } } else { - //printf("AppLayerDetectThread: stream data (len %" PRIu32 " (%" PRIu32 "))\n", smsg->data.data_len, MSG_DATA_SIZE); + //printf("AppLayerDetectThread: stream data (len %" PRIu32 " (%" PRIu32 ")), alproto %"PRIu16"\n", smsg->data.data_len, MSG_DATA_SIZE, alproto); //printf("=> Stream Data -- start\n"); //PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); @@ -366,11 +345,8 @@ void *AppLayerDetectProtoThread(void *td) /* if we don't have a data object here we are not getting it * a start msg should have gotten us one */ - if (al_data_ptr != NULL) { - al_proto = (AppLayerDetectProtoData *)al_data_ptr; - //printf("AppLayerDetectThread: already established that the proto is %" PRIu32 "\n", al_proto->proto); - - AppLayerParse(smsg->flow, al_proto->proto, smsg->flags, smsg->data.data, smsg->data.data_len); + if (alproto != ALPROTO_UNKNOWN) { + AppLayerParse(smsg->flow, alproto, smsg->flags, smsg->data.data, smsg->data.data_len); } else { //printf("AppLayerDetectThread: smsg not start, but no l7 data? Weird\n"); } @@ -378,19 +354,9 @@ void *AppLayerDetectProtoThread(void *td) } mutex_lock(&smsg->flow->m); - if (store == 1) { - /* store */ - if (ssn != NULL && ssn->l7data != NULL) { - ssn->l7data[al_proto_id] = (void *)al_proto; - } else { - al_proto->proto = 0; - PoolReturn(al_detect_proto_pool,(void *)al_proto); - } - store = 0; - } - /* XXX we need to improve this logic */ smsg->flow->use_cnt--; mutex_unlock(&smsg->flow->m); + /* return the used message to the queue */ StreamMsgReturnToPool(smsg); } diff --git a/src/app-layer-http.c b/src/app-layer-http.c index 1eb7c685ba..f0a67be0c2 100644 --- a/src/app-layer-http.c +++ b/src/app-layer-http.c @@ -208,7 +208,8 @@ static int HTTPParseRequest(void *http_state, AppLayerParserState *pstate, uint8 } break; } - case 2: +#if 0 + case 2: /* BODY */ { //printf("HTTPParseRequest: request body\n"); @@ -223,6 +224,7 @@ static int HTTPParseRequest(void *http_state, AppLayerParserState *pstate, uint8 break; } +#endif } } @@ -347,6 +349,7 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint } break; } +#if 0 case 2: { //printf("HTTPParseResponse: response body\n"); @@ -362,6 +365,7 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint break; } +#endif } } @@ -370,17 +374,36 @@ static int HTTPParseResponse(void *http_state, AppLayerParserState *pstate, uint return 1; } +#ifdef DEBUG +static pthread_mutex_t http_state_mem_lock = PTHREAD_MUTEX_INITIALIZER; +static uint64_t http_state_memuse = 0; +static uint64_t http_state_memcnt = 0; +#endif + static void *HTTPStateAlloc(void) { void *s = malloc(sizeof(HttpState)); if (s == NULL) return NULL; memset(s, 0, sizeof(HttpState)); + +#ifdef DEBUG + mutex_lock(&http_state_mem_lock); + http_state_memcnt++; + http_state_memuse+=sizeof(HttpState); + mutex_unlock(&http_state_mem_lock); +#endif return s; } static void HTTPStateFree(void *s) { free(s); +#ifdef DEBUG + mutex_lock(&http_state_mem_lock); + http_state_memcnt--; + http_state_memuse-=sizeof(HttpState); + mutex_unlock(&http_state_mem_lock); +#endif } void RegisterHTTPParsers(void) { @@ -396,6 +419,14 @@ void RegisterHTTPParsers(void) { AppLayerRegisterStateFuncs(ALPROTO_HTTP, HTTPStateAlloc, HTTPStateFree); } +void HTTPAtExitPrintStats(void) { +#ifdef DEBUG + mutex_lock(&http_state_mem_lock); + printf("HTTPAtExitPrintStats: http_state_memcnt %"PRIu64", http_state_memuse %"PRIu64"\n", http_state_memcnt, http_state_memuse); + mutex_unlock(&http_state_mem_lock); +#endif +} + /* UNITTESTS */ #ifdef UNITTESTS @@ -410,7 +441,7 @@ int HTTPParserTest01(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen); if (r != 0) { @@ -419,7 +450,7 @@ int HTTPParserTest01(void) { goto end; } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; @@ -447,7 +478,7 @@ int HTTPParserTest02(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf, httplen); if (r != 0) { @@ -456,7 +487,7 @@ int HTTPParserTest02(void) { goto end; } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { result = 0; goto end; @@ -487,7 +518,7 @@ int HTTPParserTest03(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); if (r != 0) { @@ -510,7 +541,7 @@ int HTTPParserTest03(void) { goto end; } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; @@ -542,7 +573,7 @@ int HTTPParserTest04(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); if (r != 0) { @@ -565,7 +596,7 @@ int HTTPParserTest04(void) { goto end; } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; @@ -598,7 +629,7 @@ int HTTPParserTest05(void) { memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); if (r != 0) { @@ -621,7 +652,7 @@ int HTTPParserTest05(void) { goto end; } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; @@ -650,7 +681,7 @@ int HTTPParserTest06(void) { memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START|STREAM_EOF, httpbuf1, httplen1); if (r != 0) { @@ -659,7 +690,7 @@ int HTTPParserTest06(void) { goto end; } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; @@ -690,7 +721,7 @@ int HTTPParserTest07(void) { memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); if (r != 0) { @@ -706,7 +737,7 @@ int HTTPParserTest07(void) { goto end; } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; @@ -747,7 +778,7 @@ int HTTPParserTest08(void) { memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); if (r != 0) { @@ -791,7 +822,7 @@ int HTTPParserTest08(void) { goto end; } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; @@ -827,7 +858,7 @@ int HTTPParserTest09(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; uint32_t u; for (u = 0; u < httplen1; u++) { @@ -860,7 +891,7 @@ int HTTPParserTest09(void) { } } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; @@ -894,7 +925,7 @@ int HTTPParserTest10(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; uint32_t u; for (u = 0; u < httplen1; u++) { @@ -912,7 +943,7 @@ int HTTPParserTest10(void) { } } - HttpState *http_state = ssn.l7data[AlpGetStateIdx(ALPROTO_HTTP)]; + HttpState *http_state = ssn.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; if (http_state == NULL) { printf("no http state: "); result = 0; diff --git a/src/app-layer-http.h b/src/app-layer-http.h index fd4cd22ed9..21681b5797 100644 --- a/src/app-layer-http.h +++ b/src/app-layer-http.h @@ -3,6 +3,7 @@ void RegisterHTTPParsers(void); void HTTPParserRegisterTests(void); +void HTTPAtExitPrintStats(void); #endif /* __APP_LAYER_HTTP_H__ */ diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 17403c2b49..08a5a8d582 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -402,7 +402,7 @@ uint16_t AlpGetStateIdx(uint16_t proto) { return al_proto_table[proto].storage_id; } -AppLayerParserStateStore* AppLayerParserStateStoreAlloc(void) { +AppLayerParserStateStore *AppLayerParserStateStoreAlloc(void) { AppLayerParserStateStore *s = (AppLayerParserStateStore *)malloc(sizeof(AppLayerParserStateStore)); if (s == NULL) return NULL; @@ -411,6 +411,17 @@ AppLayerParserStateStore* AppLayerParserStateStoreAlloc(void) { return s; } +/** \brief free a AppLayerParserStateStore structure + * \param s AppLayerParserStateStore structure to free */ +void AppLayerParserStateStoreFree(AppLayerParserStateStore *s) { + if (s->to_server.store != NULL) + free(s->to_server.store); + if (s->to_client.store != NULL) + free(s->to_client.store); + + free(s); +} + static void AppLayerParserResultCleanup(AppLayerParserResult *result) { AppLayerParserResultElmt *e = result->head; while (e != NULL) { @@ -493,22 +504,21 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_ uint16_t parser_idx = 0; AppLayerProto *p = &al_proto_table[proto]; - TcpSession *ssn = f->stream; + TcpSession *ssn = f->protoctx; if (ssn == NULL) { - printf("AppLayerParse: no stream\n"); + printf("AppLayerParse: no session\n"); return -1; } /* Get the parser state (if any) */ - AppLayerParserStateStore *parser_state_store = (AppLayerParserStateStore *)ssn->l7data[app_layer_sid]; + AppLayerParserStateStore *parser_state_store = (AppLayerParserStateStore *)ssn->aldata[app_layer_sid]; if (parser_state_store == NULL) { parser_state_store = AppLayerParserStateStoreAlloc(); if (parser_state_store == NULL) return -1; mutex_lock(&f->m); - if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */ - ssn->l7data[app_layer_sid] = (void *)parser_state_store; + ssn->aldata[app_layer_sid] = (void *)parser_state_store; mutex_unlock(&f->m); } @@ -546,8 +556,7 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_ /* See if we already have a 'app layer' state */ void *app_layer_state = NULL; mutex_lock(&f->m); - if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */ - app_layer_state = ssn->l7data[p->storage_id]; + app_layer_state = ssn->aldata[p->storage_id]; mutex_unlock(&f->m); if (app_layer_state == NULL) { app_layer_state = p->StateAlloc(); @@ -555,8 +564,7 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_ return -1; mutex_lock(&f->m); - if (ssn->l7data != NULL) /** \todo remove once we fixed ssn timeouts */ - ssn->l7data[p->storage_id] = app_layer_state; + ssn->aldata[p->storage_id] = app_layer_state; mutex_unlock(&f->m); } @@ -580,6 +588,39 @@ void RegisterAppLayerParsers(void) { al_result_pool = PoolInit(100,10,AlpResultElmtPoolAlloc,NULL,AlpResultElmtPoolFree); } +void AppLayerParserCleanupState(TcpSession *ssn) { + if (ssn == NULL) { + //printf("AppLayerParserCleanupState: no ssn\n"); + return; + } + + AppLayerProto *p = &al_proto_table[ssn->alproto]; + if (p == NULL) { + //printf("AppLayerParserCleanupState: no parser state for %"PRIu16"\n", ssn->alproto); + return; + } + + /* free the parser protocol state */ + if (p->StateFree != NULL) { + if (ssn->aldata[p->storage_id] != NULL) { + //printf("AppLayerParserCleanupState: calling StateFree\n"); + p->StateFree(ssn->aldata[p->storage_id]); + ssn->aldata[p->storage_id] = NULL; + } + } + + if (ssn->aldata != NULL) { + if (ssn->aldata[app_layer_sid] != NULL) { + //printf("AppLayerParserCleanupState: calling AppLayerParserStateStoreFree\n"); + AppLayerParserStateStoreFree(ssn->aldata[app_layer_sid]); + ssn->aldata[app_layer_sid] = NULL; + } + + free(ssn->aldata); + ssn->aldata = NULL; + } +} + /** \brief Create a mapping between the individual parsers local field id's * and the global field parser id's. * diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index 264e6306d1..a28bb7316d 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -87,5 +87,8 @@ int AlpParseFieldByEOF(AppLayerParserResult *, AppLayerParserState *, uint16_t, int AlpParseFieldByDelimiter(AppLayerParserResult *, AppLayerParserState *, uint16_t, const uint8_t *, uint8_t, uint8_t *, uint32_t, uint32_t *); uint16_t AlpGetStateIdx(uint16_t); +#include "stream-tcp-private.h" +void AppLayerParserCleanupState(TcpSession *); + #endif /* __APP_LAYER_PARSER_H__ */ diff --git a/src/app-layer-tls.c b/src/app-layer-tls.c index 9f520751f2..f19d713ebe 100644 --- a/src/app-layer-tls.c +++ b/src/app-layer-tls.c @@ -149,7 +149,7 @@ static int TLSParserTest01(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER|STREAM_EOF, tlsbuf, tlslen); if (r != 0) { @@ -158,7 +158,7 @@ static int TLSParserTest01(void) { goto end; } - TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)]; + TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)]; if (tls_state == NULL) { printf("no tls state: "); result = 0; @@ -193,7 +193,7 @@ static int TLSParserTest02(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1); if (r != 0) { @@ -209,7 +209,7 @@ static int TLSParserTest02(void) { goto end; } - TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)]; + TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)]; if (tls_state == NULL) { printf("no tls state: "); result = 0; @@ -246,7 +246,7 @@ static int TLSParserTest03(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1); if (r != 0) { @@ -269,7 +269,7 @@ static int TLSParserTest03(void) { goto end; } - TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)]; + TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)]; if (tls_state == NULL) { printf("no tls state: "); result = 0; @@ -308,7 +308,7 @@ static int TLSParserTest04(void) { memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize()); - f.stream = (void *)&ssn; + f.protoctx = (void *)&ssn; int r = AppLayerParse(&f, ALPROTO_TLS, STREAM_TOSERVER, tlsbuf1, tlslen1); if (r != 0) { @@ -338,7 +338,7 @@ static int TLSParserTest04(void) { goto end; } - TlsState *tls_state = ssn.l7data[AlpGetStateIdx(ALPROTO_TLS)]; + TlsState *tls_state = ssn.aldata[AlpGetStateIdx(ALPROTO_TLS)]; if (tls_state == NULL) { printf("no tls state: "); result = 0; diff --git a/src/eidps-common.h b/src/eidps-common.h index 7b8574c3d8..b6919d5f01 100644 --- a/src/eidps-common.h +++ b/src/eidps-common.h @@ -13,5 +13,8 @@ #include #include +#include +#define BUG_ON(x) assert(!(x)) + #endif /* __EIDPS_COMMON_H__ */ diff --git a/src/eidps.c b/src/eidps.c index db4889c8f6..6fa677a768 100644 --- a/src/eidps.c +++ b/src/eidps.c @@ -624,6 +624,320 @@ int RunModeIdsPcap2(DetectEngineCtx *de_ctx, char *iface) { return 0; } +/** \brief Live pcap mode with 4 stream tracking and reassembly threads, testing the flow queuehandler */ +int RunModeIdsPcap3(DetectEngineCtx *de_ctx, char *iface) { + TimeModeSetLive(); + + /* create the threads */ + ThreadVars *tv_receivepcap = TmThreadCreatePacketHandler("ReceivePcap","packetpool","packetpool","pickup-queue","simple","1slot_noinout"); + if (tv_receivepcap == NULL) { + printf("ERROR: TmThreadsCreate failed\n"); + exit(EXIT_FAILURE); + } + TmModule *tm_module = TmModuleGetByName("ReceivePcap"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed for ReceivePcap\n"); + exit(EXIT_FAILURE); + } + Tm1SlotSetFunc(tv_receivepcap,tm_module,(void *)iface); + + if (TmThreadSpawn(tv_receivepcap) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + + ThreadVars *tv_decode1 = TmThreadCreatePacketHandler("Decode1","pickup-queue","simple","decode-queue1,decode-queue2,decode-queue3,decode-queue4","flow","1slot"); + if (tv_decode1 == NULL) { + printf("ERROR: TmThreadsCreate failed for Decode1\n"); + exit(EXIT_FAILURE); + } + tm_module = TmModuleGetByName("DecodePcap"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName DecodePcap failed\n"); + exit(EXIT_FAILURE); + } + Tm1SlotSetFunc(tv_decode1,tm_module,NULL); + + if (TmThreadSpawn(tv_decode1) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + + ThreadVars *tv; + tv = TmThreadCreatePacketHandler("Stream1","decode-queue1","simple","packetpool","packetpool","varslot"); + if (tv == NULL) { + printf("ERROR: TmThreadsCreate failed for Stream1\n"); + exit(EXIT_FAILURE); + } + tm_module = TmModuleGetByName("StreamTcp"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx); + + tm_module = TmModuleGetByName("RespondReject"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for RespondReject failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertFastlog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertFastlog failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("LogHttplog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertUnifiedLog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertUnifiedAlert"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertDebuglog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + TmThreadSetCPUAffinity(tv, 0); + + if (TmThreadSpawn(tv) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + + tv = TmThreadCreatePacketHandler("Stream2","decode-queue2","simple","packetpool","packetpool","varslot"); + if (tv == NULL) { + printf("ERROR: TmThreadsCreate failed for Stream1\n"); + exit(EXIT_FAILURE); + } + tm_module = TmModuleGetByName("StreamTcp"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx); + + tm_module = TmModuleGetByName("RespondReject"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for RespondReject failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertFastlog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertFastlog failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("LogHttplog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertUnifiedLog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertUnifiedAlert"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertDebuglog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + TmThreadSetCPUAffinity(tv, 0); + + if (TmThreadSpawn(tv) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + + tv = TmThreadCreatePacketHandler("Stream3","decode-queue3","simple","packetpool","packetpool","varslot"); + if (tv == NULL) { + printf("ERROR: TmThreadsCreate failed for Stream1\n"); + exit(EXIT_FAILURE); + } + tm_module = TmModuleGetByName("StreamTcp"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx); + + tm_module = TmModuleGetByName("RespondReject"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for RespondReject failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertFastlog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertFastlog failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("LogHttplog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertUnifiedLog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertUnifiedAlert"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertDebuglog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + TmThreadSetCPUAffinity(tv, 1); + + if (TmThreadSpawn(tv) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + + tv = TmThreadCreatePacketHandler("Stream4","decode-queue4","simple","packetpool","packetpool","varslot"); + if (tv == NULL) { + printf("ERROR: TmThreadsCreate failed for Stream1\n"); + exit(EXIT_FAILURE); + } + tm_module = TmModuleGetByName("StreamTcp"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("Detect"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName StreamTcp failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,(void *)de_ctx); + + tm_module = TmModuleGetByName("RespondReject"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for RespondReject failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertFastlog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertFastlog failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("LogHttplog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertUnifiedLog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedLog failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertUnifiedAlert"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName for AlertUnifiedAlert failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + tm_module = TmModuleGetByName("AlertDebuglog"); + if (tm_module == NULL) { + printf("ERROR: TmModuleGetByName failed\n"); + exit(EXIT_FAILURE); + } + TmVarSlotSetFuncAppend(tv,tm_module,NULL); + + TmThreadSetCPUAffinity(tv, 1); + + if (TmThreadSpawn(tv) != 0) { + printf("ERROR: TmThreadSpawn failed\n"); + exit(EXIT_FAILURE); + } + return 0; +} + int RunModeIpsNFQ(DetectEngineCtx *de_ctx) { TimeModeSetLive(); @@ -1256,7 +1570,8 @@ int main(int argc, char **argv) gettimeofday(&start_time, NULL); if (mode == MODE_PCAP_DEV) { - RunModeIdsPcap2(de_ctx, pcap_dev); + RunModeIdsPcap3(de_ctx, pcap_dev); + //RunModeIdsPcap2(de_ctx, pcap_dev); //RunModeIdsPcap(de_ctx, pcap_dev); } else if (mode == MODE_PCAP_FILE) { @@ -1337,6 +1652,8 @@ int main(int argc, char **argv) FlowShutdown(); FlowPrintQueueInfo(); + StreamTcpFreeConfig(STREAM_VERBOSE); + HTTPAtExitPrintStats(); /** \todo review whats needed here */ SigGroupCleanup(de_ctx); diff --git a/src/flow-hash.c b/src/flow-hash.c index 379e37d825..fafb64f59f 100644 --- a/src/flow-hash.c +++ b/src/flow-hash.c @@ -98,7 +98,7 @@ Flow *FlowGetFlowFromHash (Packet *p) /* got one, now lock, initialize and return */ mutex_lock(&f->m); FlowInit(f,p); - FlowRequeue(f, NULL, &flow_new_q); + FlowRequeue(f, NULL, &flow_new_q[f->protomap]); f->flags |= FLOW_NEW_LIST; f->fb = fb; @@ -139,14 +139,14 @@ Flow *FlowGetFlowFromHash (Packet *p) /* lock, initialize and return */ mutex_lock(&f->m); FlowInit(f,p); - FlowRequeue(f, NULL, &flow_new_q); + FlowRequeue(f, NULL, &flow_new_q[f->protomap]); f->flags |= FLOW_NEW_LIST; f->fb = fb; mutex_unlock(&fb->m); return f; - } + } mutex_lock(&f->m); diff --git a/src/flow-private.h b/src/flow-private.h index aed736558a..dc49dc43fa 100644 --- a/src/flow-private.h +++ b/src/flow-private.h @@ -9,10 +9,13 @@ /* per flow flags */ #define FLOW_TO_SRC_SEEN 0x01 #define FLOW_TO_DST_SEEN 0x02 + #define FLOW_NEW_LIST 0x04 #define FLOW_EST_LIST 0x08 -#define FLOW_TOSERVER_IPONLY_SET 0x10 -#define FLOW_TOCLIENT_IPONLY_SET 0x20 +#define FLOW_CLOSED_LIST 0x10 + +#define FLOW_TOSERVER_IPONLY_SET 0x20 +#define FLOW_TOCLIENT_IPONLY_SET 0x40 /* global flow flags */ #define FLOW_EMERGENCY 0x01 @@ -38,16 +41,41 @@ #define FLOW_IPPROTO_ICMP_EMERG_NEW_TIMEOUT 10 #define FLOW_IPPROTO_ICMP_EMERG_EST_TIMEOUT 100 +enum { + FLOW_PROTO_DEFAULT = 0, + FLOW_PROTO_TCP, + FLOW_PROTO_UDP, + FLOW_PROTO_ICMP, + + /* should be last */ + FLOW_PROTO_MAX, +}; + /* * Variables */ -FlowQueue flow_spare_q; /* Spare flow's. Prealloced flows in here */ -FlowQueue flow_new_q; /* Flows in the unreplied state live here */ -FlowQueue flow_est_q; /* All other flows live here, the top holds the - * last recently used (lru) flow, so we can remove - * that in case of memory problems and check it for - * timeouts. */ +/** FlowProto specific timeouts and free/state functions */ +FlowProto flow_proto[FLOW_PROTO_MAX]; + +/** spare/unused/prealloced flows live here */ +FlowQueue flow_spare_q; + +/** Flows in the new/unreplied state live here */ +FlowQueue flow_new_q[FLOW_PROTO_MAX]; + +/** All "established" flows live here, the top holds the + * last recently used (lru) flow, so we can remove + * that in case of memory problems and check it for + * timeouts. */ +FlowQueue flow_est_q[FLOW_PROTO_MAX]; + +/** All "closing" flows live here, the top holds the + * last recently used (lru) flow, so we can remove + * that in case of memory problems and check it for + * timeouts. */ +FlowQueue flow_close_q[FLOW_PROTO_MAX]; + FlowBucket *flow_hash; FlowConfig flow_config; diff --git a/src/flow-util.c b/src/flow-util.c index c43fc8d4fa..2bd80cdf32 100644 --- a/src/flow-util.c +++ b/src/flow-util.c @@ -49,6 +49,26 @@ void FlowFree(Flow *f) free(f); } +/** + * \brief Function to map the protocol to the defined FLOW_PROTO_* enumeration. + * + * \param proto protocol which is needed to be mapped + */ + +int FlowGetProtoMapping(uint8_t proto) { + + switch (proto) { + case IPPROTO_TCP: + return FLOW_PROTO_TCP; + case IPPROTO_UDP: + return FLOW_PROTO_UDP; + case IPPROTO_ICMP: + return FLOW_PROTO_ICMP; + default: + return FLOW_PROTO_DEFAULT; + } +} + /* initialize the flow from the first packet * we see from it. */ void FlowInit(Flow *f, Packet *p) @@ -81,5 +101,7 @@ void FlowInit(Flow *f, Packet *p) } COPY_TIMESTAMP(&p->ts, &f->startts); + + f->protomap = FlowGetProtoMapping(f->proto); } diff --git a/src/flow-util.h b/src/flow-util.h index 7642cac667..aafffeede2 100644 --- a/src/flow-util.h +++ b/src/flow-util.h @@ -3,9 +3,6 @@ #ifndef __FLOW_UTIL_H__ #define __FLOW_UTIL_H__ -/** FlowProto specific timeouts and free/state functions */ -FlowProto flow_proto[FLOW_PROTO_MAX]; - #define COPY_TIMESTAMP(src,dst) ((dst)->tv_sec = (src)->tv_sec, (dst)->tv_usec = (src)->tv_usec) /* only clear the parts that won't be overwritten @@ -19,12 +16,13 @@ FlowProto flow_proto[FLOW_PROTO_MAX]; (f)->lastts.tv_usec = 0; \ GenericVarFree((f)->flowvar); \ (f)->flowvar = NULL; \ - (f)->stream = NULL; \ + (f)->protoctx = NULL; \ (f)->use_cnt = 0; \ } Flow *FlowAlloc(void); void FlowFree(Flow *); +int FlowGetProtoMapping(uint8_t); void FlowInit(Flow *, Packet *); #endif /* __FLOW_UTIL_H__ */ diff --git a/src/flow.c b/src/flow.c index 9c0c419609..e110090a78 100644 --- a/src/flow.c +++ b/src/flow.c @@ -25,7 +25,6 @@ #include "flow-var.h" #include "flow-private.h" #include "util-unittest.h" -#include "stream-tcp-private.h" //#define FLOW_DEFAULT_HASHSIZE 262144 #define FLOW_DEFAULT_HASHSIZE 65536 @@ -40,32 +39,51 @@ static int FlowUpdateSpareFlows(void); int FlowSetProtoTimeout(uint8_t , uint32_t ,uint32_t ,uint32_t); int FlowSetProtoEmergencyTimeout(uint8_t , uint32_t ,uint32_t ,uint32_t); static int FlowClearMemory(Flow *,uint8_t ); -static int FlowGetProtoMapping(uint8_t); int FlowSetProtoFreeFunc(uint8_t, void (*Free)(void *)); int FlowSetFlowStateFunc (uint8_t , int (*GetProtoState)(void *)); /** \brief Update the flows position in the queue's * \param f Flow to requeue. + * \todo if we have a flow state func rely on that soly * - * In-use flows are either in the flow_new_q or flow_est_q lists. + * In-use flows are in the flow_new_q, flow_est_q lists or flow_close_q lists. */ -static void FlowUpdateQueue(Flow *f) +void FlowUpdateQueue(Flow *f) { if (f->flags & FLOW_NEW_LIST) { /* in the new list -- we consider a flow no longer * new if we have seen at least 2 pkts in both ways. */ if (f->todstpktcnt && f->tosrcpktcnt) { - FlowRequeue(f, &flow_new_q, &flow_est_q); + FlowRequeue(f, &flow_new_q[f->protomap], &flow_est_q[f->protomap]); f->flags |= FLOW_EST_LIST; /* transition */ f->flags &= ~FLOW_NEW_LIST; } else { - FlowRequeue(f, &flow_new_q, &flow_new_q); + FlowRequeue(f, &flow_new_q[f->protomap], &flow_new_q[f->protomap]); } } else if (f->flags & FLOW_EST_LIST) { + if (flow_proto[f->protomap].GetProtoState != NULL) { + uint8_t state = flow_proto[f->protomap].GetProtoState(f->protoctx); + if (state == FLOW_STATE_CLOSED) { + f->flags |= FLOW_CLOSED_LIST; /* transition */ + f->flags &= ~FLOW_EST_LIST; + + //printf("FlowUpdateQueue %p was put into closing queue ts %"PRIuMAX"\n", f, (uintmax_t)f->lastts.tv_sec); + FlowRequeue(f, &flow_est_q[f->protomap], &flow_close_q[f->protomap]); + } else { + /* Pull and put back -- this way the flows on + * top of the list are least recently used. */ + FlowRequeue(f, &flow_est_q[f->protomap], &flow_est_q[f->protomap]); + } + } else { + /* Pull and put back -- this way the flows on + * top of the list are least recently used. */ + FlowRequeue(f, &flow_est_q[f->protomap], &flow_est_q[f->protomap]); + } + } else if (f->flags & FLOW_CLOSED_LIST){ /* Pull and put back -- this way the flows on * top of the list are least recently used. */ - FlowRequeue(f, &flow_est_q, &flow_est_q); + FlowRequeue(f, &flow_close_q[f->protomap], &flow_close_q[f->protomap]); } } @@ -111,70 +129,69 @@ static int FlowPrune (FlowQueue *q, struct timeval *ts) /*set the timeout value according to the flow operating mode, flow's state and protocol.*/ uint32_t timeout = 0; - uint8_t proto_map; - proto_map = FlowGetProtoMapping(f->proto); if (flow_flags & FLOW_EMERGENCY) { - - if (flow_proto[proto_map].GetProtoState != NULL) { - switch(flow_proto[proto_map].GetProtoState(f->stream)) { + if (flow_proto[f->protomap].GetProtoState != NULL) { + switch(flow_proto[f->protomap].GetProtoState(f->protoctx)) { case FLOW_STATE_NEW: - timeout = flow_proto[proto_map].emerg_new_timeout; + timeout = flow_proto[f->protomap].emerg_new_timeout; break; case FLOW_STATE_ESTABLISHED: - timeout = flow_proto[proto_map].emerg_est_timeout; + timeout = flow_proto[f->protomap].emerg_est_timeout; break; case FLOW_STATE_CLOSED: - timeout = flow_proto[proto_map].emerg_closed_timeout; + timeout = flow_proto[f->protomap].emerg_closed_timeout; break; } } else { if (f->flags & FLOW_EST_LIST) - timeout = flow_proto[proto_map].emerg_est_timeout; + timeout = flow_proto[f->protomap].emerg_est_timeout; else - timeout = flow_proto[proto_map].emerg_new_timeout; + timeout = flow_proto[f->protomap].emerg_new_timeout; } - - } else { - - if (flow_proto[proto_map].GetProtoState != NULL) { - switch(flow_proto[proto_map].GetProtoState(f->stream)) { + } else { /* impliet not emergency */ + if (flow_proto[f->protomap].GetProtoState != NULL) { + switch(flow_proto[f->protomap].GetProtoState(f->protoctx)) { case FLOW_STATE_NEW: - timeout = flow_proto[proto_map].new_timeout; + timeout = flow_proto[f->protomap].new_timeout; break; case FLOW_STATE_ESTABLISHED: - timeout = flow_proto[proto_map].est_timeout; + timeout = flow_proto[f->protomap].est_timeout; break; case FLOW_STATE_CLOSED: - timeout = flow_proto[proto_map].closed_timeout; + timeout = flow_proto[f->protomap].closed_timeout; break; } } else { if (f->flags & FLOW_EST_LIST) - timeout = flow_proto[proto_map].est_timeout; + timeout = flow_proto[f->protomap].est_timeout; else - timeout = flow_proto[proto_map].new_timeout; + timeout = flow_proto[f->protomap].new_timeout; } } DEBUGPRINT("got lock, now check: %" PRId64 "+%" PRIu32 "=(%" PRId64 ") < %" PRId64 "", f->lastts.tv_sec, timeout, f->lastts.tv_sec + timeout, ts->tv_sec); - /** never prune a flow that is used by a packet or stream msg - * we are currently processing in one of the threads */ - if (f->use_cnt > 0) { + /* do the timeout check */ + if ((f->lastts.tv_sec + timeout) >= ts->tv_sec) { mutex_unlock(&f->fb->m); mutex_unlock(&f->m); return 0; } - /* do the timeout check */ - if ((f->lastts.tv_sec + timeout) >= ts->tv_sec) { + /** never prune a flow that is used by a packet or stream msg + * we are currently processing in one of the threads */ + if (f->use_cnt > 0) { + printf("FlowPrune: timed out but use_cnt > 0: %"PRIu16", %p, proto %"PRIu8"\n", f->use_cnt, f, f->proto); mutex_unlock(&f->fb->m); mutex_unlock(&f->m); return 0; } + //printf("timed out %" PRIuMAX "+%" PRIu32 "=(%" PRIuMAX ") < %" PRIuMAX ": %p, proto %"PRIu8"\n", (uintmax_t)f->lastts.tv_sec, + // timeout, (uintmax_t)(f->lastts.tv_sec + timeout), (uintmax_t)ts->tv_sec, f, f->proto); + /* remove from the hash */ if (f->hprev) f->hprev->hnext = f->hnext; @@ -189,7 +206,7 @@ static int FlowPrune (FlowQueue *q, struct timeval *ts) mutex_unlock(&f->fb->m); f->fb = NULL; - FlowClearMemory (f, proto_map); + FlowClearMemory (f, f->protomap); /* move to spare list */ FlowRequeue(f, q, &flow_spare_q); @@ -396,20 +413,26 @@ void FlowInitConfig (char quiet) * \warning Not thread safe */ void FlowPrintQueueInfo (void) { + int i; printf("* Flow Queue info:\n"); printf(" - SPARE %" PRIu32 " (", flow_spare_q.len); #ifdef DBG_PERF printf("flow_spare_q.dbg_maxlen %" PRIu32 ")\n", flow_spare_q.dbg_maxlen); #endif - printf(" - NEW %" PRIu32 " (", flow_new_q.len); + for (i = 0; i < FLOW_PROTO_MAX; i++) { + printf(" - NEW %" PRIu32 " (", flow_new_q[i].len); #ifdef DBG_PERF - printf("flow_new_q.dbg_maxlen %" PRIu32 ")\n", flow_new_q.dbg_maxlen); + printf("flow_new_q.dbg_maxlen %" PRIu32 ")\n", flow_new_q[i].dbg_maxlen); #endif - printf(" - ESTABLISHED %" PRIu32 " (", flow_est_q.len); + printf(" - ESTABLISHED %" PRIu32 " (", flow_est_q[i].len); #ifdef DBG_PERF - printf("flow_est_q.dbg_maxlen %" PRIu32 ")\n", flow_est_q.dbg_maxlen); + printf("flow_est_q.dbg_maxlen %" PRIu32 ")\n", flow_est_q[i].dbg_maxlen); #endif - + printf(" - CLOSING %" PRIu32 " (", flow_close_q[i].len); +#ifdef DBG_PERF + printf("flow_closing_q.dbg_maxlen %" PRIu32 ")\n", flow_close_q[i].dbg_maxlen); +#endif + } #ifdef FLOWBITS_STATS printf("* Flowbits added: %" PRIu32 ", removed: %" PRIu32 ", ", flowbits_added, flowbits_removed); printf("max memory usage: %" PRIu32 "\n", flowbits_memuse_max); @@ -420,19 +443,27 @@ void FlowPrintQueueInfo (void) * \warning Not thread safe */ void FlowShutdown(void) { Flow *f; + int i; while((f = FlowDequeue(&flow_spare_q))) { FlowFree(f); } - while((f = FlowDequeue(&flow_new_q))) { - uint8_t proto_map = FlowGetProtoMapping(f->proto); - FlowClearMemory(f, proto_map); - FlowFree(f); - } - while((f = FlowDequeue(&flow_est_q))) { - uint8_t proto_map = FlowGetProtoMapping(f->proto); - FlowClearMemory(f, proto_map); - FlowFree(f); + for (i = 0; i < FLOW_PROTO_MAX; i++) { + while((f = FlowDequeue(&flow_new_q[i]))) { + uint8_t proto_map = FlowGetProtoMapping(f->proto); + FlowClearMemory(f, proto_map); + FlowFree(f); + } + while((f = FlowDequeue(&flow_est_q[i]))) { + uint8_t proto_map = FlowGetProtoMapping(f->proto); + FlowClearMemory(f, proto_map); + FlowFree(f); + } + while((f = FlowDequeue(&flow_close_q[i]))) { + uint8_t proto_map = FlowGetProtoMapping(f->proto); + FlowClearMemory(f, proto_map); + FlowFree(f); + } } free(flow_hash); @@ -458,7 +489,7 @@ void *FlowManagerThread(void *td) { ThreadVars *th_v = (ThreadVars *)td; struct timeval ts; - uint32_t established_cnt = 0, new_cnt = 0, nowcnt; + uint32_t established_cnt = 0, new_cnt = 0, closing_cnt = 0, nowcnt; uint32_t sleeping = 0; uint8_t emerg = FALSE; @@ -484,23 +515,30 @@ void *FlowManagerThread(void *td) DEBUGPRINT("ts %" PRId64 "", ts.tv_sec); /* see if we still have enough spare flows */ - if (!(FlowUpdateSpareFlows()) && emerg == TRUE) { - /*timeout_new = flow_config.emerg_timeout_new; - timeout_est = flow_config.emerg_timeout_est;*/ - } - - /* prune new list */ - nowcnt = FlowPruneFlows(&flow_new_q, &ts); - if (nowcnt) { - DEBUGPRINT("Pruned %" PRIu32 " new flows...\n", nowcnt); - new_cnt += nowcnt; - } - - /* prune established list */ - nowcnt = FlowPruneFlows(&flow_est_q, &ts); - if (nowcnt) { - DEBUGPRINT("Pruned %" PRIu32 " established flows...\n", nowcnt); - established_cnt += nowcnt; + FlowUpdateSpareFlows(); + + int i; + for (i = 0; i < FLOW_PROTO_MAX; i++) { + /* prune closing list */ + nowcnt = FlowPruneFlows(&flow_close_q[i], &ts); + if (nowcnt) { + DEBUGPRINT("Pruned %" PRIu32 " closing flows...\n", nowcnt); + closing_cnt += nowcnt; + } + + /* prune new list */ + nowcnt = FlowPruneFlows(&flow_new_q[i], &ts); + if (nowcnt) { + DEBUGPRINT("Pruned %" PRIu32 " new flows...\n", nowcnt); + new_cnt += nowcnt; + } + + /* prune established list */ + nowcnt = FlowPruneFlows(&flow_est_q[i], &ts); + if (nowcnt) { + DEBUGPRINT("Pruned %" PRIu32 " established flows...\n", nowcnt); + established_cnt += nowcnt; + } } sleeping = 0; @@ -589,26 +627,6 @@ void FlowInitFlowProto(void) { flow_proto[FLOW_PROTO_ICMP].GetProtoState = NULL; } -/** - * \brief Function to map the protocol to the defined FLOW_PROTO_* enumeration. - * - * \param proto protocol which is needed to be mapped - */ - -static int FlowGetProtoMapping(uint8_t proto) { - - switch (proto) { - case IPPROTO_TCP: - return FLOW_PROTO_TCP; - case IPPROTO_UDP: - return FLOW_PROTO_UDP; - case IPPROTO_ICMP: - return FLOW_PROTO_ICMP; - default: - return FLOW_PROTO_DEFAULT; - } -} - /** * \brief Function clear the flow memory before queueing it to spare flow * queue. @@ -620,11 +638,10 @@ static int FlowGetProtoMapping(uint8_t proto) { static int FlowClearMemory(Flow* f, uint8_t proto_map) { /* call the protocol specific free function if we have one */ if (flow_proto[proto_map].Freefunc != NULL) { - flow_proto[proto_map].Freefunc(f->stream); + flow_proto[proto_map].Freefunc(f->protoctx); } - f->stream = NULL; + f->protoctx = NULL; - //memset(f, 0, sizeof(Flow)); CLEAR_FLOW(f); return 1; } @@ -705,6 +722,9 @@ int FlowSetProtoEmergencyTimeout(uint8_t proto, uint32_t emerg_new_timeout, uint return 1; } +#ifdef UNITTESTS +#include "stream-tcp-private.h" + /** * \test Test the setting of the per protocol timeouts. * @@ -813,7 +833,7 @@ static int FlowTestPrune(Flow *f, struct timeval *ts) { return 0; } - if (f->stream != NULL){ + if (f->protoctx != NULL){ printf("Failed in freeing the TcpSession\n"); return 0; } @@ -841,7 +861,7 @@ static int FlowTest03 (void) { TimeGet(&ts); f.lastts.tv_sec = ts.tv_sec - 5000; - f.stream = &ssn; + f.protoctx = &ssn; f.fb = &fb; f.proto = IPPROTO_TCP; @@ -885,7 +905,7 @@ static int FlowTest04 (void) { ssn.server = client; ssn.state = TCP_ESTABLISHED; f.lastts.tv_sec = ts.tv_sec - 5000; - f.stream = &ssn; + f.protoctx = &ssn; f.fb = &fb; f.proto = IPPROTO_TCP; @@ -918,7 +938,7 @@ static int FlowTest05 (void) { TimeGet(&ts); ssn.state = TCP_SYN_SENT; f.lastts.tv_sec = ts.tv_sec - 300; - f.stream = &ssn; + f.protoctx = &ssn; f.fb = &fb; f.proto = IPPROTO_TCP; f.flags = FLOW_EMERGENCY; @@ -963,7 +983,7 @@ static int FlowTest06 (void) { ssn.server = client; ssn.state = TCP_ESTABLISHED; f.lastts.tv_sec = ts.tv_sec - 5000; - f.stream = &ssn; + f.protoctx = &ssn; f.fb = &fb; f.proto = IPPROTO_TCP; f.flags = FLOW_EMERGENCY; @@ -974,15 +994,18 @@ static int FlowTest06 (void) { return 1; } +#endif /* UNITTESTS */ + /** * \brief Function to register the Flow Unitests. */ - void FlowRegisterTests (void) { +#ifdef UNITTESTS UtRegisterTest("FlowTest01 -- Protocol Specific Timeouts", FlowTest01, 1); UtRegisterTest("FlowTest02 -- Setting Protocol Specific Free Function", FlowTest02, 1); UtRegisterTest("FlowTest03 -- Timeout a flow having fresh TcpSession", FlowTest03, 1); UtRegisterTest("FlowTest04 -- Timeout a flow having TcpSession with segments", FlowTest04, 1); UtRegisterTest("FlowTest05 -- Timeout a flow in emergency having fresh TcpSession", FlowTest05, 1); UtRegisterTest("FlowTest06 -- Timeout a flow in emergency having TcpSession with segments", FlowTest06, 1); +#endif /* UNITTESTS */ } diff --git a/src/flow.h b/src/flow.h index 29167e0c17..f88336916a 100644 --- a/src/flow.h +++ b/src/flow.h @@ -67,9 +67,15 @@ typedef struct Flow_ uint32_t tosrcpktcnt; uint64_t bytecnt; - void *stream; - uint16_t use_cnt; /** how many pkts and stream msgs are - using the flow *right now* */ + /** mapping to Flow's protocol specific protocols for timeouts + and state and free functions. */ + uint8_t protomap; + + /** protocol specific data pointer, e.g. for TcpSession */ + void *protoctx; + + /** how many pkts and stream msgs are using the flow *right now* */ + uint16_t use_cnt; pthread_mutex_t m; @@ -84,16 +90,6 @@ typedef struct Flow_ struct FlowBucket_ *fb; } Flow; -enum { - FLOW_PROTO_DEFAULT = 0, - FLOW_PROTO_TCP, - FLOW_PROTO_UDP, - FLOW_PROTO_ICMP, - - /* should be last */ - FLOW_PROTO_MAX, -}; - enum { FLOW_STATE_NEW = 0, FLOW_STATE_ESTABLISHED, @@ -126,6 +122,7 @@ int FlowSetProtoTimeout(uint8_t ,uint32_t ,uint32_t ,uint32_t); int FlowSetProtoEmergencyTimeout(uint8_t ,uint32_t ,uint32_t ,uint32_t); int FlowSetProtoFreeFunc (uint8_t , void (*Free)(void *)); int FlowSetFlowStateFunc (uint8_t , int (*GetProtoState)(void *)); +void FlowUpdateQueue(Flow *); #endif /* __FLOW_H__ */ diff --git a/src/stream-tcp-private.h b/src/stream-tcp-private.h index 33b62e113d..04ade39f16 100644 --- a/src/stream-tcp-private.h +++ b/src/stream-tcp-private.h @@ -11,33 +11,34 @@ typedef struct TcpSegment_ { } TcpSegment; typedef struct TcpStream_ { - uint32_t isn; /* initial sequence number */ - uint32_t next_seq; /* next expected sequence number */ - uint32_t last_ack; /* last ack'd sequence number */ - uint32_t next_win; /* next max seq within window */ - uint8_t wscale; - uint16_t window; + uint32_t isn; /**< initial sequence number */ + uint32_t next_seq; /**< next expected sequence number */ + uint32_t last_ack; /**< last ack'd sequence number in this stream */ + uint32_t next_win; /**< next max seq within window */ + uint32_t window; /**< current window setting */ + uint8_t wscale; /**< wscale setting in this direction */ /* reassembly */ - uint32_t ra_base_seq; /* reassembled seq. We've reassembled up to this point. */ - TcpSegment *seg_list; - uint8_t os_policy; /* target based OS policy used for reassembly and handling packets*/ + uint32_t ra_base_seq; /**< reassembled seq. We've reassembled up to this point. */ + TcpSegment *seg_list; /**< list of TCP segments that are not yet (fully) used in reassembly */ + uint8_t os_policy; /**< target based OS policy used for reassembly and handling packets*/ } TcpStream; /* from /usr/include/netinet/tcp.h */ enum { - TCP_ESTABLISHED = 1, + TCP_NONE, + TCP_LISTEN, TCP_SYN_SENT, TCP_SYN_RECV, + TCP_ESTABLISHED, TCP_FIN_WAIT1, TCP_FIN_WAIT2, TCP_TIME_WAIT, - TCP_CLOSED, - TCP_CLOSE_WAIT, TCP_LAST_ACK, - TCP_LISTEN, - TCP_CLOSING /* now a valid state */ + TCP_CLOSE_WAIT, + TCP_CLOSING, + TCP_CLOSED, }; #define STREAMTCP_FLAG_MIDSTREAM 0x01 /*Flag for mid stream session*/ @@ -53,9 +54,10 @@ enum typedef struct TcpSession_ { uint8_t state; + uint8_t flags; + uint16_t alproto; /**< application level protocol */ TcpStream server; TcpStream client; - void **l7data; - u_int8_t flags; + void **aldata; /**< application level storage ptrs */ } TcpSession; #endif /* __STREAM_TCP_PRIVATE_H__ */ diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index a0b0b0837b..84a766a6cb 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -1,6 +1,7 @@ /** Copyright (c) 2008 Victor Julien * Copyright (c) 2009 Open Information Security Foundation * + * \file * \author Gurvinder Singh * \author Victor Julien * @@ -12,7 +13,6 @@ #include "eidps.h" - #include #include #include @@ -44,6 +44,12 @@ //#define DEBUG +#ifdef DEBUG +static pthread_mutex_t segment_pool_memuse_mutex; +static uint64_t segment_pool_memuse = 0; +static uint64_t segment_pool_memcnt = 0; +#endif + /* prototypes */ static int HandleSegmentStartsBeforeListSegment(TcpStream *, TcpSegment *, TcpSegment *, uint8_t); static int HandleSegmentStartsAtSameListSegment(TcpStream *, TcpSegment *, TcpSegment *, uint8_t); @@ -54,12 +60,8 @@ TcpSegment* StreamTcpGetSegment(uint16_t); void StreamTcpSegmentReturntoPool(TcpSegment *); void StreamTcpCreateTestPacket(uint8_t *, uint8_t, uint8_t); -/** \todo XXX Never defined */ -#if 0 -static int StreamTcpCheckStreamContents(uint8_t *, TcpStream *); -#endif - -void *TcpSegmentAlloc(void *payload_len) { +/** \brief alloc a tcp segment pool entry */ +void *TcpSegmentPoolAlloc(void *payload_len) { TcpSegment *seg = malloc(sizeof (TcpSegment)); if (seg == NULL) return NULL; @@ -75,10 +77,17 @@ void *TcpSegmentAlloc(void *payload_len) { return NULL; } +#ifdef DEBUG + mutex_lock(&segment_pool_memuse_mutex); + segment_pool_memuse += seg->payload_len; + segment_pool_memcnt ++; + mutex_unlock(&segment_pool_memuse_mutex); +#endif return seg; } -void TcpSegmentFree(void *ptr) { +/** \brief free a tcp segment pool entry */ +void TcpSegmentPoolFree(void *ptr) { if (ptr == NULL) return; @@ -94,18 +103,24 @@ void TcpSegmentFree(void *ptr) { * The cost is in memory of course. */ #define segment_pool_num 8 static uint16_t segment_pool_pktsizes[segment_pool_num] = {4, 16, 112, 248, 512, 768, 1448, 0xffff}; -static uint16_t segment_pool_poolsizes[segment_pool_num] = {1024, 1024, 1024, 1024, 4096, 4096, 1024, 128}; +static uint16_t segment_pool_poolsizes[segment_pool_num] = {2048, 3072, 3072, 3072, 3072, 4096, 8192, 512}; static Pool *segment_pool[segment_pool_num]; static pthread_mutex_t segment_pool_mutex[segment_pool_num]; +#ifdef DEBUG +static pthread_mutex_t segment_pool_cnt_mutex; +static uint64_t segment_pool_cnt = 0; +#endif /* index to the right pool for all packet sizes. */ static uint16_t segment_pool_idx[65536]; /* O(1) lookups of the pool */ int StreamTcpReassembleInit(void) { StreamMsgQueuesInit(); - +#ifdef DEBUG + pthread_mutex_init(&segment_pool_memuse_mutex, NULL); +#endif uint16_t u16 = 0; for (u16 = 0; u16 < segment_pool_num; u16++) { - segment_pool[u16] = PoolInit(segment_pool_poolsizes[u16], segment_pool_poolsizes[u16] / 2, TcpSegmentAlloc, (void *) & segment_pool_pktsizes[u16], TcpSegmentFree); + segment_pool[u16] = PoolInit(segment_pool_poolsizes[u16], segment_pool_poolsizes[u16] / 8, TcpSegmentPoolAlloc, (void *) & segment_pool_pktsizes[u16], TcpSegmentPoolFree); pthread_mutex_init(&segment_pool_mutex[u16], NULL); } @@ -123,23 +138,67 @@ int StreamTcpReassembleInit(void) { idx++; } +#ifdef DEBUG + pthread_mutex_init(&segment_pool_cnt_mutex, NULL); +#endif + return 0; +} + +void StreamTcpReassembleFree(void) { + uint16_t u16 = 0; + for (u16 = 0; u16 < segment_pool_num; u16++) { + PoolPrintSaturation(segment_pool[u16]); - /* - printf("pkt 0 : idx %" PRIu32 "\n", segment_pool_idx[0]); - printf("pkt 1 : idx %" PRIu32 "\n", segment_pool_idx[1]); - printf("pkt 1200 : idx %" PRIu32 "\n", segment_pool_idx[1200]); - printf("pkt 32 : idx %" PRIu32 "\n", segment_pool_idx[32]); - printf("pkt 1448 : idx %" PRIu32 "\n", segment_pool_idx[1448]); - printf("pkt 1449 : idx %" PRIu32 "\n", segment_pool_idx[1449]); - printf("pkt 65534: idx %" PRIu32 "\n", segment_pool_idx[65534]); - printf("pkt 65535: idx %" PRIu32 "\n", segment_pool_idx[65535]); - */ + printf("segment_pool[u16]->empty_list_size %"PRIu32", segment_pool[u16]->alloc_list_size %"PRIu32", alloced %"PRIu32"\n", segment_pool[u16]->empty_list_size, segment_pool[u16]->alloc_list_size, segment_pool[u16]->allocated); + PoolFree(segment_pool[u16]); + } - return 0; +#ifdef DEBUG + printf("segment_pool_cnt %"PRIu64"\n", segment_pool_cnt); + printf("segment_pool_memuse %"PRIu64"\n", segment_pool_memuse); + printf("segment_pool_memcnt %"PRIu64"\n", segment_pool_memcnt); +#endif + + StreamMsgQueuesDeinit(); +} + +void PrintList2(TcpSegment *seg) { + TcpSegment *prev_seg = NULL; + + if (seg == NULL) + return; + + uint32_t next_seq = seg->seq; + + while (seg != NULL) { + if (SEQ_LT(next_seq,seg->seq)) { + printf("PrintList2: missing segment(s) for %" PRIu32 " bytes of data\n", (seg->seq - next_seq)); + } + + printf("PrintList2: seg %10"PRIu32" len %" PRIu16 ", seg %p, prev %p, next %p\n", seg->seq, seg->payload_len, seg, seg->prev, seg->next); + + if (seg->prev != NULL && SEQ_LT(seg->seq,seg->prev->seq)) { + printf("PrintList2: inconsistant list: SEQ_LT(seg->seq,seg->prev->seq)) == TRUE, seg->seq %" PRIu32 ", seg->prev->seq %" PRIu32 "\n", seg->seq, seg->prev->seq); + } + + if (SEQ_LT(seg->seq,next_seq)) { + printf("PrintList2: inconsistant list: SEQ_LT(seg->seq,next_seq)) == TRUE, seg->seq %" PRIu32 ", next_seq %" PRIu32 "\n", seg->seq, next_seq); + } + + if (prev_seg != seg->prev) { + printf("PrintList2: inconsistant list: prev_seg %p != seg->prev %p\n", prev_seg, seg->prev); + } + + next_seq = seg->seq + seg->payload_len; + printf("PrintList2: next_seq is now %"PRIu32"\n", next_seq); + prev_seg = seg; + seg = seg->next; + } } -static void PrintList(TcpSegment *seg) { +void PrintList(TcpSegment *seg) { TcpSegment *prev_seg = NULL; + TcpSegment *head_seg = seg; if (seg == NULL) return; @@ -159,16 +218,19 @@ static void PrintList(TcpSegment *seg) { if (seg->prev != NULL && SEQ_LT(seg->seq,seg->prev->seq)) { printf("PrintList: inconsistant list: SEQ_LT(seg->seq,seg->prev->seq)) == TRUE, seg->seq %" PRIu32 ", seg->prev->seq %" PRIu32 "\n", seg->seq, seg->prev->seq); + PrintList2(head_seg); abort(); } if (SEQ_LT(seg->seq,next_seq)) { printf("PrintList: inconsistant list: SEQ_LT(seg->seq,next_seq)) == TRUE, seg->seq %" PRIu32 ", next_seq %" PRIu32 "\n", seg->seq, next_seq); + PrintList2(head_seg); abort(); } if (prev_seg != seg->prev) { printf("PrintList: inconsistant list: prev_seg %p != seg->prev %p\n", prev_seg, seg->prev); + PrintList2(head_seg); abort(); } @@ -194,8 +256,7 @@ static int ReassembleInsertSegment(TcpStream *stream, TcpSegment *seg) { uint8_t os_policy = stream->os_policy; int ret_value = 0; - - //printf("ReassembleInsertSegment start seg %p\n", seg); + char return_seg = FALSE; if (list_seg == NULL) { #ifdef DEBUG @@ -210,12 +271,10 @@ static int ReassembleInsertSegment(TcpStream *stream, TcpSegment *seg) { for (; list_seg != NULL; list_seg = list_seg->next) { #ifdef DEBUG printf("ReassembleInsertSegment: seg %p, list_seg %p, list_prev %p list_seg->next %p, segment length %" PRIu32 "\n", seg, list_seg, list_seg->prev, list_seg->next, seg->payload_len); - PrintRawDataFp(stdout, seg->payload, seg->payload_len); - PrintRawDataFp(stdout, list_seg->payload, list_seg->payload_len); #endif /* segment starts before list */ if (SEQ_LT(seg->seq, list_seg->seq)) { - /*seg is entirely before list_seg*/ + /* seg is entirely before list_seg */ if (SEQ_LEQ((seg->seq + seg->payload_len), list_seg->seq)) { #ifdef DEBUG printf("ReassembleInsertSegment: before list seg: seg->seq %" PRIu32 ", list_seg->seq %" PRIu32 ", list_seg->payload_len %" PRIu32 ", list_seg->prev %p\n", seg->seq, list_seg->seq, list_seg->payload_len, list_seg->prev); @@ -235,24 +294,28 @@ static int ReassembleInsertSegment(TcpStream *stream, TcpSegment *seg) { ret_value = HandleSegmentStartsBeforeListSegment(stream, list_seg, seg, os_policy); if (ret_value == 1) { ret_value = 0; + return_seg = TRUE; goto end; } else if (ret_value == -1) { - ret_value = 0; - return -1; + printf("ERROR: HandleSegmentStartsBeforeListSegment failed\n"); + ret_value = -1; + return_seg = TRUE; + goto end; } } - /* seg starts at same sequence number as list_seg */ } else if (SEQ_EQ(seg->seq, list_seg->seq)) { ret_value = HandleSegmentStartsAtSameListSegment(stream, list_seg, seg, os_policy); if (ret_value == 1) { ret_value = 0; + return_seg = TRUE; goto end; } else if (ret_value == -1) { - ret_value = 0; - return -1; + printf("ERROR: HandleSegmentStartsAtSameListSegment failed\n"); + ret_value = -1; + return_seg = TRUE; + goto end; } - /* seg starts at sequence number higher than list_seg */ } else if (SEQ_GT(seg->seq, list_seg->seq)) { if (((SEQ_GEQ(seg->seq, (list_seg->seq + list_seg->payload_len)))) && @@ -271,19 +334,24 @@ static int ReassembleInsertSegment(TcpStream *stream, TcpSegment *seg) { ret_value = HandleSegmentStartsAfterListSegment(stream, list_seg, seg, os_policy); if (ret_value == 1) { ret_value = 0; + return_seg = TRUE; goto end; } else if (ret_value == -1) { - ret_value = 0; - return -1; + printf("ERROR: HandleSegmentStartsAfterListSegment failed\n"); + ret_value = -1; + return_seg = TRUE; + goto end; } } } } end: - //printf("\nReassembleInsertSegment: @exit\n"); - PrintList(stream->seg_list); - //printf("\n"); + if (return_seg == TRUE && seg != NULL) { + StreamTcpSegmentReturntoPool(seg); + } + + //PrintList(stream->seg_list); return 0; } @@ -308,8 +376,8 @@ static int HandleSegmentStartsBeforeListSegment(TcpStream *stream, TcpSegment *l char end_same = FALSE; #ifdef DEBUG printf("\nHandleSegmentStartsBeforeListSegment: seg->seq %" PRIu32 ", seg->payload_len %" PRIu32 "\n", seg->seq, seg->payload_len); -#endif PrintList(stream->seg_list); +#endif if (SEQ_GT((seg->seq + seg->payload_len), list_seg->seq) && SEQ_LT((seg->seq + seg->payload_len),(list_seg->seq + list_seg->payload_len))) { @@ -464,21 +532,6 @@ static int HandleSegmentStartsBeforeListSegment(TcpStream *stream, TcpSegment *l } switch (os_policy) { - case OS_POLICY_BSD: - case OS_POLICY_HPUX10: - case OS_POLICY_IRIX: - case OS_POLICY_WINDOWS: - case OS_POLICY_WINDOWS2K3: - case OS_POLICY_OLD_LINUX: - case OS_POLICY_LINUX: - case OS_POLICY_MACOS: - case OS_POLICY_LAST: -#ifdef DEBUG - printf("Replacing Old Data in starts before list seg list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); -#endif - StreamTcpSegmentDataReplace(list_seg, seg, overlap_point, overlap); - //PrintRawDataFp(stdout, list_seg->payload, list_seg->payload_len); - break; case OS_POLICY_SOLARIS: case OS_POLICY_HPUX11: if (end_after == TRUE || end_same == TRUE) { @@ -496,21 +549,31 @@ static int HandleSegmentStartsBeforeListSegment(TcpStream *stream, TcpSegment *l printf("Using Old Data in starts before list case, list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); #endif break; + case OS_POLICY_BSD: + case OS_POLICY_HPUX10: + case OS_POLICY_IRIX: + case OS_POLICY_WINDOWS: + case OS_POLICY_WINDOWS2K3: + case OS_POLICY_OLD_LINUX: + case OS_POLICY_LINUX: + case OS_POLICY_MACOS: + case OS_POLICY_LAST: default: +#ifdef DEBUG + printf("Replacing Old Data in starts before list seg list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); +#endif + StreamTcpSegmentDataReplace(list_seg, seg, overlap_point, overlap); + //PrintRawDataFp(stdout, list_seg->payload, list_seg->payload_len); break; } /*To return from for loop as seg is finished with current list_seg no need to check further (improve performance)*/ //PrintRawDataFp(stdout, list_seg->payload, list_seg->payload_len); if (end_before == TRUE || end_same == TRUE) { - end_before = FALSE; - end_same = FALSE; - //printf("\nHandleSegmentStartsBeforeListSegment: @exit (return 1)\n"); - //PrintList(stream->seg_list); return 1; } } - //printf("\nHandleSegmentStartsBeforeListSegment: @exit (return 0)\n"); + //PrintList(stream->seg_list); return 0; } @@ -554,7 +617,7 @@ static int HandleSegmentStartsAtSameListSegment(TcpStream *stream, TcpSegment *l overlap = seg->payload_len; end_same = TRUE; #ifdef DEBUG - printf("HandleSegmentStartsAtSameListSegment: (retransmission) starts at list seq, ends at list end: seg->seq %" PRIu32 ", list_seg->seq %" PRIu32 ", list_seg->payload_len %" PRIu32 " overlap is%" PRIu32 "\n", seg->seq, list_seg->seq, list_seg->payload_len, overlap); + printf("HandleSegmentStartsAtSameListSegment: (retransmission) starts at list seq, ends at list end: seg->seq %" PRIu32 ", list_seg->seq %" PRIu32 ", list_seg->payload_len %" PRIu32 " overlap is %"PRIu32"\n", seg->seq, list_seg->seq, list_seg->payload_len, overlap); #endif } else if (SEQ_GT((seg->seq + seg->payload_len), (list_seg->seq + list_seg->payload_len))) { @@ -565,7 +628,7 @@ static int HandleSegmentStartsAtSameListSegment(TcpStream *stream, TcpSegment *l overlap = list_seg->payload_len; end_after = TRUE; #ifdef DEBUG - printf("HandleSegmentStartsAtSameListSegment: starts at list seq, ends beyond list end: seg->seq %" PRIu32 ", list_seg->seq %" PRIu32 ", list_seg->payload_len %" PRIu32 " overlap is%" PRIu32 "\n", seg->seq, list_seg->seq, list_seg->payload_len, overlap); + printf("HandleSegmentStartsAtSameListSegment: starts at list seq, ends beyond list end: seg->seq %" PRIu32 ", list_seg->seq %" PRIu32 ", list_seg->payload_len %" PRIu32 " overlap is %" PRIu32 "\n", seg->seq, list_seg->seq, list_seg->payload_len, overlap); #endif } if (overlap > 0) { @@ -573,62 +636,48 @@ static int HandleSegmentStartsAtSameListSegment(TcpStream *stream, TcpSegment *l segment and original segment is the last segment in the list or the next segment in the list starts after the end of new segment*/ if (end_after == TRUE) { - if (SEQ_GT((seg->seq + seg->payload_len), (list_seg->seq + list_seg->payload_len))) { - if (list_seg->next == NULL) - handle_beyond = TRUE; - else if (SEQ_GT(list_seg->next->seq, (seg->seq + seg->payload_len))) + char fill_gap = FALSE; + + if (list_seg->next != NULL) { + /* first see if we have space left to fill up */ + if (SEQ_LT((list_seg->seq + list_seg->payload_len), list_seg->next->seq)) { + fill_gap = TRUE; + } + + /* then see if we overlap (partly) with the next seg */ + if (SEQ_GT((seg->seq + seg->payload_len), list_seg->next->seq)) { handle_beyond = TRUE; + } } - if (handle_beyond == TRUE) { - packet_length = seg->payload_len; + printf("XXXXXX: fill_gap %s, handle_beyond %s\n", fill_gap?"TRUE":"FALSE", handle_beyond?"TRUE":"FALSE"); + + if (fill_gap == TRUE) { + /* if there is a gap after this list_seg we fill it now with a new seg */ +#ifdef DEBUG + printf("HandleSegmentStartsAtSameListSegment: filling gap: list_seg->next->seq %"PRIu32"\n", list_seg->next?list_seg->next->seq:0); +#endif + packet_length = seg->payload_len - list_seg->payload_len; +#ifdef DEBUG + printf("HandleSegmentStartsAtSameListSegment: packet_length %"PRIu16"\n", packet_length); +#endif TcpSegment *new_seg = StreamTcpGetSegment(packet_length); if (new_seg == NULL) { return -1; } new_seg->payload_len = packet_length; - new_seg->seq = list_seg->seq; + new_seg->seq = list_seg->seq + list_seg->payload_len; new_seg->next = list_seg->next; - new_seg->prev = list_seg->prev; - - StreamTcpSegmentDataCopy(new_seg, list_seg); - StreamTcpSegmentDataReplace(new_seg, seg, (list_seg->seq + list_seg->payload_len), (uint16_t) ((seg->seq + seg->payload_len) - (list_seg->seq + list_seg->payload_len))); - - - StreamTcpSegmentReturntoPool(list_seg); + if (new_seg->next != NULL) + new_seg->next->prev = new_seg; + new_seg->prev = list_seg; + list_seg->next = new_seg; + printf("HandleSegmentStartsAtSameListSegment: new_seg %p, new_seg->next %p, new_seg->prev %p, list_seg->next %p\n", new_seg, new_seg->next, new_seg->prev, list_seg->next); - list_seg = new_seg; - if (list_seg->next != NULL) { - list_seg->next->prev = new_seg; - } - if (list_seg->prev != NULL) { - list_seg->prev->next = new_seg; - } else { - stream->seg_list = new_seg; - } + StreamTcpSegmentDataReplace(new_seg, seg, new_seg->seq, new_seg->payload_len); } } switch (os_policy) { - case OS_POLICY_BSD: - case OS_POLICY_HPUX10: - case OS_POLICY_IRIX: - case OS_POLICY_WINDOWS: - case OS_POLICY_WINDOWS2K3: - case OS_POLICY_VISTA: - case OS_POLICY_MACOS: - case OS_POLICY_FIRST: -#ifdef DEBUG - printf("Using Old Data in starts at list case, list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); -#endif - break; - case OS_POLICY_LINUX: - if (end_after == TRUE) { - StreamTcpSegmentDataReplace(list_seg, seg, seg->seq, overlap); - } else -#ifdef DEBUG - printf("Using Old Data in starts at list case, list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); -#endif - break; case OS_POLICY_OLD_LINUX: case OS_POLICY_SOLARIS: case OS_POLICY_HPUX11: @@ -644,13 +693,32 @@ static int HandleSegmentStartsAtSameListSegment(TcpStream *stream, TcpSegment *l case OS_POLICY_LAST: StreamTcpSegmentDataReplace(list_seg, seg, seg->seq, overlap); break; + case OS_POLICY_LINUX: + if (end_after == TRUE) { + StreamTcpSegmentDataReplace(list_seg, seg, seg->seq, overlap); + } else { +#ifdef DEBUG + printf("Using Old Data in starts at list case, list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); +#endif + } + break; + case OS_POLICY_BSD: + case OS_POLICY_HPUX10: + case OS_POLICY_IRIX: + case OS_POLICY_WINDOWS: + case OS_POLICY_WINDOWS2K3: + case OS_POLICY_VISTA: + case OS_POLICY_MACOS: + case OS_POLICY_FIRST: default: +#ifdef DEBUG + printf("Using Old Data in starts at list case, list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); +#endif break; } - if (end_before == TRUE || end_same == TRUE || handle_beyond == TRUE) { - end_before = FALSE; - end_same = FALSE; - handle_beyond = FALSE; + + /* return 1 if we're done */ + if (end_before == TRUE || end_same == TRUE || handle_beyond == FALSE) { return 1; } } @@ -706,67 +774,59 @@ static int HandleSegmentStartsAfterListSegment(TcpStream *stream, TcpSegment *li overlap = (list_seg->seq + list_seg->payload_len) - seg->seq; end_after = TRUE; #ifdef DEBUG - printf("HandleSegmentStartsAfterListSegment: starts beyond list seq, before list end, ends at list end: seg->seq %" PRIu32 ", list_seg->seq %" PRIu32 ", list_seg->payload_len %" PRIu32 " overlap is %" PRIu32 "\n", seg->seq, list_seg->seq, list_seg->payload_len, overlap); + printf("HandleSegmentStartsAfterListSegment: starts beyond list seq, before list end, ends at list end: seg->seq %" PRIu32 ", seg->payload_len %"PRIu16" (%"PRIu32") list_seg->seq %" PRIu32 ", list_seg->payload_len %" PRIu32 " (%"PRIu32") overlap is %" PRIu32 "\n", seg->seq, seg->payload_len, seg->seq + seg->payload_len, list_seg->seq, list_seg->payload_len, list_seg->seq + list_seg->payload_len, overlap); #endif } if (overlap > 0) { /*Handle the case when newly arrived segment ends after original segment and original segment is the last segment in the list*/ if (end_after == TRUE) { - if (SEQ_GT((seg->seq + seg->payload_len), (list_seg->seq + list_seg->payload_len))) { - if (list_seg->next == NULL) - handle_beyond = TRUE; - else if (SEQ_GT(list_seg->next->seq, (seg->seq + seg->payload_len))) + char fill_gap = FALSE; + + if (list_seg->next != NULL) { + /* first see if we have space left to fill up */ + if (SEQ_LT((list_seg->seq + list_seg->payload_len), list_seg->next->seq)) { + fill_gap = TRUE; + } + + /* then see if we overlap (partly) with the next seg */ + if (SEQ_GT((seg->seq + seg->payload_len), list_seg->next->seq)) { handle_beyond = TRUE; + } } - if (handle_beyond == TRUE) { - packet_length = (list_seg->payload_len + seg->payload_len) - overlap; +#ifdef DEBUG + printf("HandleSegmentStartsAfterListSegment: fill_gap %s, handle_beyond %s\n", fill_gap?"TRUE":"FALSE", handle_beyond?"TRUE":"FALSE"); +#endif + + if (fill_gap == TRUE) { + /* if there is a gap after this list_seg we fill it now with a new seg */ +#ifdef DEBUG + printf("HandleSegmentStartsAfterListSegment: filling gap: list_seg->next->seq %"PRIu32"\n", list_seg->next?list_seg->next->seq:0); +#endif + packet_length = seg->payload_len - overlap; +#ifdef DEBUG + printf("HandleSegmentStartsAfterListSegment: packet_length %"PRIu16"\n", packet_length); +#endif TcpSegment *new_seg = StreamTcpGetSegment(packet_length); if (new_seg == NULL) { return -1; } - new_seg->payload_len = packet_length; - if (SEQ_LT(list_seg->seq, seg->seq)) - new_seg->seq = list_seg->seq; - else - new_seg->seq = seg->seq; + new_seg->seq = list_seg->seq + list_seg->payload_len; new_seg->next = list_seg->next; - new_seg->prev = list_seg->prev; - - StreamTcpSegmentDataCopy(new_seg, list_seg); - StreamTcpSegmentDataReplace(new_seg, seg, (list_seg->seq + list_seg->payload_len), (uint16_t) ((seg->seq + seg->payload_len) - (list_seg->seq + list_seg->payload_len))); - - StreamTcpSegmentReturntoPool(list_seg); - - list_seg = new_seg; + if (new_seg->next != NULL) + new_seg->next->prev = new_seg; + new_seg->prev = list_seg; + list_seg->next = new_seg; +#ifdef DEBUG + printf("HandleSegmentStartsAfterListSegment: new_seg %p, new_seg->next %p, new_seg->prev %p, list_seg->next %p\n", new_seg, new_seg->next, new_seg->prev, list_seg->next); +#endif - if (list_seg->next != NULL) { - list_seg->next->prev = new_seg; - } - if (list_seg->prev != NULL) { - list_seg->prev->next = list_seg; - } else { - stream->seg_list = list_seg; - } + StreamTcpSegmentDataReplace(new_seg, seg, new_seg->seq, new_seg->payload_len); } } switch (os_policy) { - case OS_POLICY_BSD: - case OS_POLICY_HPUX10: - case OS_POLICY_IRIX: - case OS_POLICY_WINDOWS: - case OS_POLICY_WINDOWS2K3: - case OS_POLICY_VISTA: - case OS_POLICY_OLD_LINUX: - case OS_POLICY_LINUX: - case OS_POLICY_MACOS: - case OS_POLICY_FIRST: -#ifdef DEBUG - printf("Using Old Data in starts beyond list case, list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); -#endif - break; case OS_POLICY_SOLARIS: case OS_POLICY_HPUX11: if (end_after == TRUE) { @@ -781,13 +841,23 @@ static int HandleSegmentStartsAfterListSegment(TcpStream *stream, TcpSegment *li case OS_POLICY_LAST: StreamTcpSegmentDataReplace(list_seg, seg, seg->seq, overlap); break; - default: + case OS_POLICY_BSD: + case OS_POLICY_HPUX10: + case OS_POLICY_IRIX: + case OS_POLICY_WINDOWS: + case OS_POLICY_WINDOWS2K3: + case OS_POLICY_VISTA: + case OS_POLICY_OLD_LINUX: + case OS_POLICY_LINUX: + case OS_POLICY_MACOS: + case OS_POLICY_FIRST: + default: /* DEFAULT POLICY */ +#ifdef DEBUG + printf("Using Old Data in starts beyond list case, list_seg->seq %" PRIu32 " policy %" PRIu32 " overlap %" PRIu32 "\n", list_seg->seq, os_policy, overlap); +#endif break; } if (end_before == TRUE || end_same == TRUE || handle_beyond == TRUE) { - end_before = FALSE; - end_same = FALSE; - handle_beyond = FALSE; return 1; } } @@ -797,10 +867,12 @@ static int HandleSegmentStartsAfterListSegment(TcpStream *stream, TcpSegment *li int StreamTcpReassembleHandleSegmentHandleData(TcpSession *ssn, TcpStream *stream, Packet *p) { TcpSegment *seg = StreamTcpGetSegment(p->payload_len); - if (seg == NULL) + if (seg == NULL) { +#ifdef DEBUG + printf("StreamTcpGetSegment returned NULL\n"); +#endif return -1; - - //printf("StreamTcpReassembleHandleSegmentHandleData: seg %p, seg->pool_size %" PRIu32 "\n", seg, seg->pool_size); + } memcpy(seg->payload, p->payload, p->payload_len); seg->payload_len = p->payload_len; @@ -808,35 +880,24 @@ int StreamTcpReassembleHandleSegmentHandleData(TcpSession *ssn, TcpStream *strea seg->next = NULL; seg->prev = NULL; - if (ReassembleInsertSegment(stream, seg) != 0) + if (ReassembleInsertSegment(stream, seg) != 0) { +#ifdef DEBUG + printf("ReassembleInsertSegment failed\n"); +#endif return -1; + } return 0; } -/* initialize the first msg */ -static void StreamTcpSetupInitMsg(Packet *p, StreamMsg *smsg) { - smsg->flags |= STREAM_START; - - if (p->flowflags & FLOW_PKT_TOSERVER) { - COPY_ADDRESS(&p->flow->src, &smsg->data.src_ip); - COPY_ADDRESS(&p->flow->dst, &smsg->data.dst_ip); - COPY_PORT(p->flow->sp, smsg->data.src_port); - COPY_PORT(p->flow->dp, smsg->data.dst_port); - - smsg->flags |= STREAM_TOSERVER; +static void StreamTcpSetupMsg(TcpSession *ssn, TcpStream *stream, Packet *p, StreamMsg *smsg) { + if (stream->ra_base_seq == stream->isn) { + smsg->flags = STREAM_START; + } else if (ssn->state > TCP_ESTABLISHED) { + smsg->flags = STREAM_EOF; } else { - COPY_ADDRESS(&p->flow->dst, &smsg->data.src_ip); - COPY_ADDRESS(&p->flow->src, &smsg->data.dst_ip); - COPY_PORT(p->flow->dp, smsg->data.src_port); - COPY_PORT(p->flow->sp, smsg->data.dst_port); - - smsg->flags |= STREAM_TOCLIENT; + smsg->flags = 0; } -} - -static void StreamTcpSetupMsg(Packet *p, StreamMsg *smsg) { - smsg->flags = 0; if (p->flowflags & FLOW_PKT_TOSERVER) { COPY_ADDRESS(&p->flow->src,&smsg->data.src_ip); @@ -853,6 +914,64 @@ static void StreamTcpSetupMsg(Packet *p, StreamMsg *smsg) { smsg->flags |= STREAM_TOCLIENT; } + + smsg->data.data_len = 0; + smsg->flow = p->flow; + if (smsg->flow != NULL) { + smsg->flow->use_cnt++; + } +} + +//#define DEBUG +/** \brief Check the minimum size limits for reassembly. + * \retval 0 don't reassemble yet + * \retval 1 do reassemble */ +static int StreamTcpReassembleCheckLimit(TcpSession *ssn, TcpStream *stream, Packet *p) { + + /* some states mean we reassemble no matter how much data we have */ + if (ssn->state == TCP_TIME_WAIT) + return 1; + + /* check if we have enough data to send to L7 */ + if (p->flowflags & FLOW_PKT_TOSERVER) { + if (stream->ra_base_seq == stream->isn) { + if (StreamMsgQueueGetMinInitChunkLen(STREAM_TOSERVER) > + (stream->last_ack - stream->ra_base_seq)) { +#ifdef DEBUG + printf("StreamTcpReassembleCheckLimit: toserver min init chunk len not yet reached: last_ack %"PRIu32", ra_base_seq %"PRIu32", len %"PRIu32"\n", stream->last_ack, stream->ra_base_seq, StreamMsgQueueGetMinInitChunkLen(STREAM_TOSERVER)); +#endif + return 0; + } + } else { + if (StreamMsgQueueGetMinChunkLen(STREAM_TOSERVER) > + (stream->last_ack - stream->ra_base_seq)) { +#ifdef DEBUG + printf("StreamTcpReassembleCheckLimit: toserver min chunk len not yet reached\n"); +#endif + return 0; + } + } + } else { + if (stream->ra_base_seq == stream->isn) { + if (StreamMsgQueueGetMinInitChunkLen(STREAM_TOCLIENT) > + (stream->last_ack - stream->ra_base_seq)) { +#ifdef DEBUG + printf("StreamTcpReassembleCheckLimit: tosclient min init chunk len not yet reached\n"); +#endif + return 0; + } + } else { + if (StreamMsgQueueGetMinChunkLen(STREAM_TOCLIENT) > + (stream->last_ack - stream->ra_base_seq)) { +#ifdef DEBUG + printf("StreamTcpReassembleCheckLimit: toclient min chunk len not yet reached\n"); +#endif + return 0; + } + } + } + + return 1; } int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *stream, Packet *p) { @@ -860,98 +979,75 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea return 0; #ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: start\n"); + printf("StreamTcpReassembleHandleSegmentUpdateACK: start p %p\n", p); #endif StreamMsg *smsg = NULL; - char remove = FALSE; uint16_t smsg_offset = 0; uint16_t payload_offset = 0; uint16_t payload_len = 0; TcpSegment *seg = stream->seg_list; uint32_t next_seq = stream->ra_base_seq + 1; - uint32_t gap_len = 0; /* check if we have enough data to send to L7 */ - /*XXX GS we need to send data if the data length is less than - * min ispected length and connection is closed already. - * more states need to be checked according to TCP state transition !!*/ - if (ssn->state != TCP_TIME_WAIT) { - - if (p->flowflags & FLOW_PKT_TOSERVER) { - if (stream->ra_base_seq == stream->isn) { - if (StreamMsgQueueGetMinInitChunkLen(STREAM_TOSERVER) > - (stream->last_ack - stream->ra_base_seq)) - return 0; - } else { - if (StreamMsgQueueGetMinChunkLen(STREAM_TOSERVER) > - (stream->last_ack - stream->ra_base_seq)) - return 0; - } - } else { - if (stream->ra_base_seq == stream->isn) { - if (StreamMsgQueueGetMinInitChunkLen(STREAM_TOCLIENT) > - (stream->last_ack - stream->ra_base_seq)) - return 0; - } else { - if (StreamMsgQueueGetMinChunkLen(STREAM_TOCLIENT) > - (stream->last_ack - stream->ra_base_seq)) - return 0; - } - } + if (StreamTcpReassembleCheckLimit(ssn,stream,p) == 0) { +#ifdef DEBUG + printf("StreamTcpReassembleHandleSegmentUpdateACK: not yet reassembling\n"); +#endif + return 0; } - -//#ifdef DEBUG - PrintList(seg); -//#endif - /* loop through the segments and fill one or more msgs */ for (; seg != NULL && SEQ_LT(seg->seq, stream->last_ack);) { +#ifdef DEBUG + printf("StreamTcpReassembleHandleSegmentUpdateACK: seg %p\n", seg); +#endif + /* If packets are fully before ra_base_seq, skip them. We do this * because we've reassembled up to the ra_base_seq point already, * so we won't do anything with segments before it anyway. */ - if (SEQ_LT((seg->seq + seg->payload_len), stream->ra_base_seq)) { +#ifdef DEBUG + printf("StreamTcpReassembleHandleSegmentUpdateACK: checking for pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32" len %"PRIu16", combined %"PRIu32"\n", stream->ra_base_seq, seg, seg->seq, seg->payload_len, seg->seq+seg->payload_len); +#endif + + /** \todo we should probably not even insert them into the seglist */ + if (SEQ_LEQ((seg->seq + seg->payload_len), (stream->ra_base_seq+1))) { +#ifdef DEBUG printf("StreamTcpReassembleHandleSegmentUpdateACK: removing pre ra_base_seq %"PRIu32" seg %p seq %"PRIu32" len %"PRIu16"\n", stream->ra_base_seq, seg, seg->seq, seg->payload_len); +#endif TcpSegment *next_seg = seg->next; - stream->seg_list = seg->next; - if (stream->seg_list != NULL) - stream->seg_list->prev = NULL; + if (seg->prev == NULL) { + stream->seg_list = seg->next; + if (stream->seg_list != NULL) + stream->seg_list->prev = NULL; + } else { + seg->prev->next = seg->next; + if (seg->next != NULL) + seg->next->prev = seg->prev; + } StreamTcpSegmentReturntoPool(seg); seg = next_seg; continue; } - PrintList(stream->seg_list); -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: seg %p\n", seg); -#endif - /* we've run into a sequence gap */ - if (next_seq != seg->seq) { - /* next_seq should never be smaller than seg->seq */ - if (SEQ_GT(next_seq, seg->seq)) { - PrintList(seg); - PrintList(stream->seg_list); - abort(); - } + if (SEQ_GT(seg->seq, next_seq)) { /* see what the length of the gap is, gap length is seg->seq - (ra_base_seq +1) */ - gap_len = seg->seq - next_seq; + uint32_t gap_len = seg->seq - next_seq; +#ifdef DEBUG printf("StreamTcpReassembleHandleSegmentUpdateACK: expected next_seq %" PRIu32 ", got %" PRIu32 " , stream->last_ack %" PRIu32 ". Seq gap %" PRIu32"\n", next_seq, seg->seq, stream->last_ack, gap_len); +#endif + next_seq = seg->seq; - /* pass on pre existing smsgs if any */ + /* pass on pre existing smsg (if any) */ if (smsg != NULL && smsg->data.data_len > 0) { -#ifdef DEBUG - PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); -#endif StreamMsgPutInQueue(smsg); smsg = NULL; } - if (smsg == NULL) { smsg = StreamMsgGetFromPool(); if (smsg == NULL) { @@ -960,25 +1056,16 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea return -1; } } + StreamTcpSetupMsg(ssn, stream, p, smsg); - if (stream->ra_base_seq == stream->isn) { - StreamTcpSetupInitMsg(p, smsg); - } else { - StreamTcpSetupMsg(p, smsg); - } - - smsg->data.data_len = 0; - smsg->flow = p->flow; - if (smsg->flow) - smsg->flow->use_cnt++; - - /* As IDS has missed the packet and end host has ack'd it, so + /* We have missed the packet and end host has ack'd it, so IDS should advance it's ra_base_seq and should not consider this packet any longer, even if it is retransmitted, as end host will drop it anyway */ stream->ra_base_seq = seg->seq - 1; smsg->flags |= STREAM_GAP; smsg->gap.gap_size = gap_len; + StreamMsgPutInQueue(smsg); smsg = NULL; smsg_offset = 0; @@ -990,27 +1077,15 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea printf("StreamTcpReassembleHandleSegmentUpdateACK: seg->seq %" PRIu32 ", seg->payload_len %" PRIu32 ", stream->ra_base_seq %" PRIu32 "\n", seg->seq, seg->payload_len, stream->ra_base_seq); #endif - - /* get a message - XXX we need a setup function */ if (smsg == NULL) { smsg = StreamMsgGetFromPool(); if (smsg == NULL) { - printf("StreamTcpReassembleHandleSegmentUpdateACK: couldn't " - "get a stream msg from the pool\n"); + printf("StreamTcpReassembleHandleSegmentUpdateACK: couldn't get a stream msg from the pool\n"); return -1; } - smsg_offset = 0; - if (stream->ra_base_seq == stream->isn) { - StreamTcpSetupInitMsg(p, smsg); - } else { - StreamTcpSetupMsg(p, smsg); - } - smsg->data.data_len = 0; - smsg->flow = p->flow; - if (smsg->flow) - smsg->flow->use_cnt++; + smsg_offset = 0; + StreamTcpSetupMsg(ssn, stream, p, smsg); } /* handle segments partly before ra_base_seq */ @@ -1019,71 +1094,43 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea if (SEQ_LT(stream->last_ack, (seg->seq + seg->payload_len))) { payload_len = ((seg->seq + seg->payload_len) - stream->last_ack) - payload_offset; -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: starts " - "before ra_base, ends beyond last_ack, payload_offset %" PRIu32 ", " - "payload_len %" PRIu32 "\n", payload_offset, payload_len); -#endif } else { payload_len = seg->payload_len - payload_offset; -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: starts " - "before ra_base, ends normal, payload_offset %" PRIu32 ", " - "payload_len %" PRIu32 "\n", payload_offset, payload_len); -#endif } - - if (payload_offset > seg->payload_len) { - printf("BUG(%" PRIu32 "): payload_offset %" PRIu32 " > seg->payload_len %" PRIu32 ". seg->seq %" PRIu32 ", stream->ra_base_seq %" PRIu32 "\n", - __LINE__, payload_offset, seg->payload_len, seg->seq, stream->ra_base_seq); - abort(); - } - /* handle segments after ra_base_seq */ + BUG_ON(payload_offset > seg->payload_len); } else { payload_offset = 0; if (SEQ_LT(stream->last_ack, (seg->seq + seg->payload_len))) { payload_len = stream->last_ack - seg->seq; -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: start " - "fine, ends beyond last_ack, payload_offset %" PRIu32 ", " - "payload_len %" PRIu32 "\n", payload_offset, payload_len); -#endif } else { payload_len = seg->payload_len; -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: normal " - "(smsg_offset %" PRIu32 "), payload_offset %" PRIu32 ", payload_len %" PRIu32 "\n", - smsg_offset, payload_offset, payload_len); -#endif } } + /* copy the data into the smsg */ uint16_t copy_size = sizeof (smsg->data.data) - smsg_offset; if (copy_size > payload_len) { copy_size = payload_len; } - if (copy_size > sizeof(smsg->data.data)) { - printf("BUG(%" PRIu32 "): copy_size %" PRIu32 " > sizeof(smsg->data.data) %" PRIuMAX "\n", __LINE__, copy_size, (uintmax_t)sizeof(smsg->data.data)); - abort(); - } -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: copy_size %" PRIu32 " " - "(payload_len %" PRIu32 ", payload_offset %" PRIu32 ")\n", copy_size, payload_len, payload_offset); -#endif - + BUG_ON(copy_size > sizeof(smsg->data.data)); memcpy(smsg->data.data + smsg_offset, seg->payload + payload_offset, copy_size); - smsg_offset += copy_size; stream->ra_base_seq += copy_size; +#ifdef DEBUG + printf("StreamTcpReassembleHandleSegmentUpdateACK: stream->ra_base_seq %"PRIu32"\n", stream->ra_base_seq); +#endif smsg->data.data_len += copy_size; + /* queue the smsg if it's full */ if (smsg->data.data_len == sizeof (smsg->data.data)) { StreamMsgPutInQueue(smsg); smsg = NULL; } + /* if the payload len is bigger than what we copied, we handle the rest of the + payload next... */ if (copy_size < payload_len) { #ifdef DEBUG printf("StreamTcpReassembleHandleSegmentUpdateACK: " @@ -1092,18 +1139,11 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea payload_offset += copy_size; payload_len -= copy_size; - if (payload_offset > seg->payload_len) { - printf("BUG(%" PRIu32 "): payload_offset %" PRIu32 " > seg->payload_len %" PRIu32 "\n", __LINE__, payload_offset, seg->payload_len); - abort(); - } -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: " - "payload_offset %" PRIu32 "\n", payload_offset); -#endif + BUG_ON(payload_offset > seg->payload_len); /* we need a while loop here as the packets theoretically can be 64k */ - - while (remove == FALSE) { + char segment_done = FALSE; + while (segment_done == FALSE) { #ifdef DEBUG printf("StreamTcpReassembleHandleSegmentUpdateACK: " "new msg at offset %" PRIu32 ", payload_len %" PRIu32 "\n", payload_offset, payload_len); @@ -1118,21 +1158,15 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea return -1; } smsg_offset = 0; - smsg->data.data_len = 0; - smsg->flow = p->flow; - if (smsg->flow) - smsg->flow->use_cnt++; - StreamTcpSetupMsg(p,smsg); + StreamTcpSetupMsg(ssn, stream,p,smsg); copy_size = sizeof(smsg->data.data) - smsg_offset; if (copy_size > (seg->payload_len - payload_offset)) { copy_size = (seg->payload_len - payload_offset); } - if (copy_size > sizeof(smsg->data.data)) { - printf("BUG(%" PRIu32 "): copy_size %" PRIu32 " > sizeof(smsg->data.data) %" PRIuMAX "\n", __LINE__, copy_size, (uintmax_t)sizeof(smsg->data.data)); - abort(); - } + BUG_ON(copy_size > sizeof(smsg->data.data)); + #ifdef DEBUG printf("StreamTcpReassembleHandleSegmentUpdateACK: copy " "payload_offset %" PRIu32 ", smsg_offset %" PRIu32 ", copy_size %" PRIu32 "\n", @@ -1141,6 +1175,9 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea memcpy(smsg->data.data + smsg_offset, seg->payload + payload_offset, copy_size); smsg_offset += copy_size; stream->ra_base_seq += copy_size; +#ifdef DEBUG + printf("StreamTcpReassembleHandleSegmentUpdateACK: stream->ra_base_seq %"PRIu32"\n", stream->ra_base_seq); +#endif smsg->data.data_len += copy_size; #ifdef DEBUG printf("StreamTcpReassembleHandleSegmentUpdateACK: copied " @@ -1152,57 +1189,41 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea smsg = NULL; } + /* see if we have segment payload left to process */ if ((copy_size + payload_offset) < seg->payload_len) { payload_offset += copy_size; payload_len -= copy_size; - if (payload_offset > seg->payload_len) { - printf("BUG(%" PRIu32 "): payload_offset %" PRIu32 " > seg->payload_len %" PRIu32 "\n", __LINE__, payload_offset, seg->payload_len); - abort(); - } -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: loop not done\n"); -#endif + BUG_ON(payload_offset > seg->payload_len); } else { -#ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: loop done\n"); -#endif payload_offset = 0; - remove = TRUE; + segment_done = TRUE; } } } else { payload_offset = 0; - remove = TRUE; } } - TcpSegment *next_seg = seg->next; - /* done with this segment, return it to the pool */ - if (remove == TRUE) { - next_seq = seg->seq + seg->payload_len; + TcpSegment *next_seg = seg->next; + next_seq = seg->seq + seg->payload_len; #ifdef DEBUG - printf("StreamTcpReassembleHandleSegmentUpdateACK: removing seg %p, " - "seg->next %p\n", seg, seg->next); + printf("StreamTcpReassembleHandleSegmentUpdateACK: removing seg %p, " + "seg->next %p\n", seg, seg->next); #endif - stream->seg_list = seg->next; - if (stream->seg_list != NULL) - stream->seg_list->prev = NULL; + BUG_ON(seg->prev != NULL); /**< BUG if we aren't the top of the list */ - StreamTcpSegmentReturntoPool(seg); - - remove = FALSE; - } + stream->seg_list = seg->next; + if (stream->seg_list != NULL) + stream->seg_list->prev = NULL; + StreamTcpSegmentReturntoPool(seg); seg = next_seg; } /* put the partly filled smsg in the queue to the l7 handler */ if (smsg != NULL) { -#ifdef DEBUG - PrintRawDataFp(stdout, smsg->data.data, smsg->data.data_len); -#endif StreamMsgPutInQueue(smsg); smsg = NULL; } @@ -1211,35 +1232,47 @@ int StreamTcpReassembleHandleSegmentUpdateACK (TcpSession *ssn, TcpStream *strea } int StreamTcpReassembleHandleSegment(TcpSession *ssn, TcpStream *stream, Packet *p) { + //printf("StreamTcpReassembleHandleSegment: ssn %p, stream %p, p %p, p->payload_len %"PRIu16"\n", ssn, stream, p, p->payload_len); + /* handle ack received */ - if (StreamTcpReassembleHandleSegmentUpdateACK(ssn, stream, p) != 0) + if (StreamTcpReassembleHandleSegmentUpdateACK(ssn, stream, p) != 0) { +#ifdef DEBUG + printf("StreamTcpReassembleHandleSegment: StreamTcpReassembleHandleSegmentUpdateACK error\n"); +#endif return -1; + } if (p->payload_len > 0) { - if (StreamTcpReassembleHandleSegmentHandleData(ssn, stream, p) != 0) + if (StreamTcpReassembleHandleSegmentHandleData(ssn, stream, p) != 0) { +#ifdef DEBUG + printf("StreamTcpReassembleHandleSegment: StreamTcpReassembleHandleSegmentHandleData error\n"); +#endif return -1; + } } return 0; } -/* Initialize the l7data ptr in the TCP session used - * by the L7 Modules for data storage. +/** \brief Initialize the l7data ptr in the TCP session used by the L7 Modules for data storage. * - * ssn = TcpSesssion - * cnt = number of items in the array + * \param ssn TcpSesssion to init the ptrs for + * \param cnt number of items in the array * - * XXX use a pool? + * \todo VJ use a pool? */ void StreamL7DataPtrInit(TcpSession *ssn, uint8_t cnt) { if (cnt == 0) return; - ssn->l7data = (void **) malloc(sizeof (void *) * cnt); - if (ssn->l7data != NULL) { + if (ssn->aldata != NULL) + return; + + ssn->aldata = (void **) malloc(sizeof (void *) * cnt); + if (ssn->aldata != NULL) { uint8_t u; for (u = 0; u < cnt; u++) { - ssn->l7data[u] = NULL; + ssn->aldata[u] = NULL; } } } @@ -1254,7 +1287,6 @@ void StreamL7DataPtrInit(TcpSession *ssn, uint8_t cnt) { * * \todo VJ We can remove the abort()s later. */ - void StreamTcpSegmentDataReplace(TcpSegment *dst_seg, TcpSegment *src_seg, uint32_t start_point, uint16_t len) { uint32_t seq; uint16_t s_cnt = 0; @@ -1266,24 +1298,17 @@ void StreamTcpSegmentDataReplace(TcpSegment *dst_seg, TcpSegment *src_seg, uint3 dst_pos = dst_seg->seq - start_point; } - if (len + dst_pos > dst_seg->payload_len) { - printf("ERROR: trying to replace more data than we have\n"); - abort(); - } + BUG_ON(len + dst_pos > dst_seg->payload_len); for (seq = start_point; SEQ_LT(seq, (start_point + len)); seq++) { if (dst_pos >= dst_seg->payload_len) abort(); - //printf("StreamTcpSegmentDataReplace: seq %" PRIu32 ", start point %" PRIu32 ", len %" PRIu32 ", dst_seg len %" PRIu32 ", dst_pos %" PRIu32 ", s_cnt %" PRIu32 "\n", seq, start_point, len, dst_seg->payload_len, dst_pos, s_cnt); - dst_seg->payload[dst_pos] = src_seg->payload[s_cnt]; dst_pos++; s_cnt++; } - //printf("print in data replace\n"); - //PrintRawDataFp(stdout, list_seg->payload, list_seg->payload_len); } /** @@ -1328,13 +1353,34 @@ void StreamTcpSegmentDataCopy(TcpSegment *dst_seg, TcpSegment *src_seg) { TcpSegment* StreamTcpGetSegment(uint16_t len) { uint16_t idx = segment_pool_idx[len]; - //printf("StreamTcpReassembleHandleSegmentHandleData: idx %" PRIu32 " for payload_len %" PRIu32 "\n", idx, p->payload_len); - +#ifdef DEBUG + printf("StreamTcpGetSegment: %" PRIu32 " for payload_len %" PRIu32 "\n", idx, len); +#endif mutex_lock(&segment_pool_mutex[idx]); - //printf("StreamTcpReassembleHandleSegmentHandleData: mutex locked, getting data from pool %p\n", segment_pool[idx]); +#ifdef DEBUG + printf("StreamTcpGetSegment: mutex locked, getting data from pool %p\n", segment_pool[idx]); +#endif TcpSegment *seg = (TcpSegment *) PoolGet(segment_pool[idx]); +#ifdef DEBUG + printf("StreamTcpGetSegment: segment_pool[%u]->empty_list_size %u, segment_pool[%u]->alloc_list_size %u, alloc %u\n", idx, segment_pool[idx]->empty_list_size, idx, segment_pool[idx]->alloc_list_size, segment_pool[idx]->allocated); +#endif + //PoolPrintSaturation(segment_pool[idx]); mutex_unlock(&segment_pool_mutex[idx]); +#ifdef DEBUG + printf("StreamTcpGetSegment: seg we return is %p\n", seg); +#endif + if (seg == NULL) { +#ifdef DEBUG + printf("StreamTcpGetSegment: segment_pool[%u]->empty_list_size %u, alloc %u\n", idx, segment_pool[idx]->empty_list_size, segment_pool[idx]->allocated); +#endif + } else { +#ifdef DEBUG + mutex_lock(&segment_pool_cnt_mutex); + segment_pool_cnt++; + mutex_unlock(&segment_pool_cnt_mutex); +#endif + } return seg; } @@ -1350,9 +1396,16 @@ void StreamTcpSegmentReturntoPool(TcpSegment *seg) { uint16_t idx = segment_pool_idx[seg->pool_size]; mutex_lock(&segment_pool_mutex[idx]); - //printf("StreamTcpReassembleHandleSegmentHandleData: mutex locked, getting data from pool %p\n", segment_pool[idx]); PoolReturn(segment_pool[idx], (void *) seg); +#ifdef DEBUG + printf("StreamTcpSegmentReturntoPool: segment_pool[%u]->empty_list_size %u\n", idx,segment_pool[idx]->empty_list_size); +#endif mutex_unlock(&segment_pool_mutex[idx]); +#ifdef DEBUG + mutex_lock(&segment_pool_cnt_mutex); + segment_pool_cnt--; + mutex_unlock(&segment_pool_cnt_mutex); +#endif } #ifdef UNITTESTS @@ -1383,7 +1436,7 @@ static int StreamTcpReassembleStreamTest(TcpStream *stream) { memset(&p, 0, sizeof (Packet)); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.src.family = AF_INET; p.dst.family = AF_INET; p.proto = IPPROTO_TCP; @@ -1656,7 +1709,7 @@ static int StreamTcpTestStartsBeforeListSegment(TcpStream *stream) { memset(&p, 0, sizeof (Packet)); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.src.family = AF_INET; p.dst.family = AF_INET; p.proto = IPPROTO_TCP; @@ -1743,7 +1796,7 @@ static int StreamTcpTestStartsAtSameListSegment(TcpStream *stream) { memset(&p, 0, sizeof (Packet)); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.src.family = AF_INET; p.dst.family = AF_INET; p.proto = IPPROTO_TCP; @@ -1831,7 +1884,7 @@ static int StreamTcpTestStartsAfterListSegment(TcpStream *stream) { memset(&p, 0, sizeof (Packet)); memset(&f, 0, sizeof (Flow)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.src.family = AF_INET; p.dst.family = AF_INET; p.proto = IPPROTO_TCP; diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h index fa4b5a85b0..e80c228234 100644 --- a/src/stream-tcp-reassemble.h +++ b/src/stream-tcp-reassemble.h @@ -31,6 +31,7 @@ enum int StreamTcpReassembleHandleSegment(TcpSession *, TcpStream *, Packet *); int StreamTcpReassembleInit(void); +void StreamTcpReassembleFree(void); void StreamTcpReassembleRegisterTests(void); void StreamTcpCreateTestPacket(u_int8_t *, u_int8_t, u_int8_t); diff --git a/src/stream-tcp.c b/src/stream-tcp.c index dc8a94984d..60cf111ffb 100644 --- a/src/stream-tcp.c +++ b/src/stream-tcp.c @@ -2,7 +2,6 @@ /* 2009 Gurvinder Singh */ -#include "decode.h" #include #include #include @@ -16,6 +15,7 @@ #include #include "eidps-common.h" +#include "decode.h" #include "debug.h" #include "detect.h" #include "flow.h" @@ -34,6 +34,8 @@ #include "stream.h" #include "stream-tcp.h" +#include "app-layer-parser.h" + //#define DEBUG int StreamTcp (ThreadVars *, Packet *, void *, PacketQueue *); @@ -42,55 +44,87 @@ int StreamTcpThreadDeinit(ThreadVars *, void *); void StreamTcpExitPrintStats(ThreadVars *, void *); static int ValidReset(TcpSession * , Packet *); static int StreamTcpHandleFin(TcpSession *, Packet *); -void StreamTcpSessionPktFree (Packet *); void StreamTcpRegisterTests (void); void StreamTcpReturnStreamSegments (TcpStream *); void StreamTcpInitConfig(char); extern void StreamTcpSegmentReturntoPool(TcpSegment *); int StreamTcpGetFlowState(void *); -#define STREAMTCP_DEFAULT_SESSIONS 262144 -#define STREAMTCP_DEFAULT_PREALLOC 32768 -#define STREAMTCP_NEW_TIMEOUT 60 -#define STREAMTCP_EST_TIMEOUT 3600 -#define STREAMTCP_CLOSED_TIMEOUT 120 -#define STREAMTCP_EMERG_NEW_TIMEOUT 10 -#define STREAMTCP_EMERG_EST_TIMEOUT 300 -#define STREAMTCP_EMERG_CLOSED_TIMEOUT 20 +#define STREAMTCP_DEFAULT_SESSIONS 262144 +#define STREAMTCP_DEFAULT_PREALLOC 32768 + +#define STREAMTCP_NEW_TIMEOUT 60 +#define STREAMTCP_EST_TIMEOUT 3600 +#define STREAMTCP_CLOSED_TIMEOUT 120 + +#define STREAMTCP_EMERG_NEW_TIMEOUT 10 +#define STREAMTCP_EMERG_EST_TIMEOUT 300 +#define STREAMTCP_EMERG_CLOSED_TIMEOUT 20 static Pool *ssn_pool; static pthread_mutex_t ssn_pool_mutex; +#ifdef DEBUG +static uint64_t ssn_pool_cnt; +static pthread_mutex_t ssn_pool_cnt_mutex; +#endif + typedef struct StreamTcpThread_ { - u_int64_t pkts; + uint64_t pkts; - u_int64_t counter_tcp_streams; + uint16_t counter_tcp_sessions; } StreamTcpThread; void TmModuleStreamTcpRegister (void) { - tmm_modules[TMM_STREAMTCP].name = "StreamTcp"; tmm_modules[TMM_STREAMTCP].Init = StreamTcpThreadInit; tmm_modules[TMM_STREAMTCP].Func = StreamTcp; tmm_modules[TMM_STREAMTCP].ExitPrintStats = StreamTcpExitPrintStats; tmm_modules[TMM_STREAMTCP].Deinit = StreamTcpThreadDeinit; tmm_modules[TMM_STREAMTCP].RegisterTests = StreamTcpRegisterTests; - //StreamTcpInitConfig(STREAM_VERBOSE); } void StreamTcpReturnStreamSegments (TcpStream *stream) { - TcpSegment *temp = stream->seg_list; - TcpSegment *prev; - for (; temp!=NULL; ) { - if (temp->next != NULL) { - prev = temp; - temp = temp->next; - StreamTcpSegmentReturntoPool(prev); - } else { - StreamTcpSegmentReturntoPool(temp); - temp = temp->next; - } + TcpSegment *seg = stream->seg_list; + TcpSegment *next_seg; + + if (seg == NULL) + return; + + while (seg != NULL) { + next_seg = seg->next; + StreamTcpSegmentReturntoPool(seg); + seg = next_seg; } + + stream->seg_list = NULL; +} + +/** \brief Function to return the stream back to the pool. It returns the + * segments in the stream to the segment pool. + * + * \param ssn Void ptr to the ssn. + */ +void StreamTcpSessionClear(void *ssnptr) { + TcpSession *ssn = (TcpSession *)ssnptr; + if (ssn == NULL) + return; + + StreamTcpReturnStreamSegments(&ssn->client); + StreamTcpReturnStreamSegments(&ssn->server); + + AppLayerParserCleanupState(ssn); + + memset(ssn, 0, sizeof(TcpSession)); + mutex_lock(&ssn_pool_mutex); + PoolReturn(ssn_pool, ssn); + mutex_unlock(&ssn_pool_mutex); + +#ifdef DEBUG + mutex_lock(&ssn_pool_cnt_mutex); + ssn_pool_cnt--; + mutex_unlock(&ssn_pool_cnt_mutex); +#endif } /** \brief Function to return the stream back to the pool. It returns the @@ -98,20 +132,28 @@ void StreamTcpReturnStreamSegments (TcpStream *stream) { * * \param p Packet used to identify the stream. */ -void StreamTcpSessionPktFree (Packet *p) { - TcpSession *ssn = (TcpSession *)p->flow->stream; +static void StreamTcpSessionPktFree (Packet *p) { + TcpSession *ssn = (TcpSession *)p->flow->protoctx; if (ssn == NULL) return; StreamTcpReturnStreamSegments(&ssn->client); StreamTcpReturnStreamSegments(&ssn->server); + AppLayerParserCleanupState(ssn); + memset(ssn, 0, sizeof(TcpSession)); mutex_lock(&ssn_pool_mutex); - PoolReturn(ssn_pool, p->flow->stream); + PoolReturn(ssn_pool, p->flow->protoctx); mutex_unlock(&ssn_pool_mutex); - p->flow->stream = NULL; + p->flow->protoctx = NULL; + +#ifdef DEBUG + mutex_lock(&ssn_pool_cnt_mutex); + ssn_pool_cnt--; + mutex_unlock(&ssn_pool_cnt_mutex); +#endif } /** \brief Stream alloc function for the Pool @@ -166,13 +208,24 @@ void StreamTcpInitConfig(char quiet) { pthread_mutex_init(&ssn_pool_mutex, NULL); StreamTcpReassembleInit(); - /*set the default TCP timeout, free function and flow state function values.*/ + + /* set the default TCP timeout, free function and flow state function values. */ FlowSetProtoTimeout(IPPROTO_TCP, STREAMTCP_NEW_TIMEOUT, STREAMTCP_EST_TIMEOUT, STREAMTCP_CLOSED_TIMEOUT); FlowSetProtoEmergencyTimeout(IPPROTO_TCP, STREAMTCP_EMERG_NEW_TIMEOUT, STREAMTCP_EMERG_EST_TIMEOUT, STREAMTCP_EMERG_CLOSED_TIMEOUT); - FlowSetProtoFreeFunc(IPPROTO_TCP, StreamTcpSessionPoolFree); + + FlowSetProtoFreeFunc(IPPROTO_TCP, StreamTcpSessionClear); FlowSetFlowStateFunc(IPPROTO_TCP, StreamTcpGetFlowState); } +void StreamTcpFreeConfig(char quiet) { + StreamTcpReassembleFree(); + + PoolFree(ssn_pool); +#ifdef DEBUG + printf("ssn_pool_cnt %"PRIu64"\n", ssn_pool_cnt); +#endif +} + /** \brief The function is used to to fetch a TCP session from the * ssn_pool, when a TCP SYN is received. * @@ -181,24 +234,39 @@ void StreamTcpInitConfig(char quiet) { * \retval TcpSession A new TCP session with field initilaized to 0/NULL. */ TcpSession *StreamTcpNewSession (Packet *p) { - TcpSession *ssn = (TcpSession *)p->flow->stream; + TcpSession *ssn = (TcpSession *)p->flow->protoctx; if (ssn == NULL) { mutex_lock(&ssn_pool_mutex); - p->flow->stream = PoolGet(ssn_pool); + p->flow->protoctx = PoolGet(ssn_pool); mutex_unlock(&ssn_pool_mutex); - ssn = (TcpSession *)p->flow->stream; + ssn = (TcpSession *)p->flow->protoctx; if (ssn == NULL) return NULL; - ssn->state = 0; - ssn->l7data = NULL; + ssn->state = TCP_NONE; + ssn->aldata = NULL; + +#ifdef DEBUG + mutex_lock(&ssn_pool_cnt_mutex); + ssn_pool_cnt++; + mutex_unlock(&ssn_pool_cnt_mutex); +#endif } return ssn; } +static inline void StreamTcpPacketSetState(Packet *p, TcpSession *ssn, uint8_t state) { + if (state == ssn->state) + return; + + ssn->state = state; + + FlowUpdateQueue(p->flow); +} + /** * \brief Function to handle the TCP_CLOSED or NONE state. The function handles * packets while the session state is None which means a newly @@ -208,7 +276,6 @@ TcpSession *StreamTcpNewSession (Packet *p) { * \param p Packet which has to be handled in this TCP state. * \param stt Strean Thread module registered to handle the stream handling */ - static int StreamTcpPacketStateNone(ThreadVars *tv, Packet *p, StreamTcpThread *stt, TcpSession *ssn) { switch (p->tcph->th_flags) { case TH_SYN: @@ -217,46 +284,46 @@ static int StreamTcpPacketStateNone(ThreadVars *tv, Packet *p, StreamTcpThread * ssn = StreamTcpNewSession(p); if (ssn == NULL) return -1; + + PerfCounterIncr(stt->counter_tcp_sessions, tv->pca); } /* set the state */ - ssn->state = TCP_SYN_SENT; + StreamTcpPacketSetState(p, ssn, TCP_SYN_SENT); #ifdef DEBUG printf("StreamTcpPacketStateNone (%p): =~ ssn state is now TCP_SYN_SENT\n", ssn); #endif - /* set the sequence numbers and window */ ssn->client.isn = TCP_GET_SEQ(p); ssn->client.ra_base_seq = ssn->client.isn; ssn->client.next_seq = ssn->client.isn + 1; - ssn->client.window = TCP_GET_WINDOW(p); - - //ssn->server.last_ack = ssn->client.isn + 1; - //ssn->server.last_ack = TCP_GET_ACK(p); + ssn->server.window = TCP_GET_WINDOW(p); #ifdef DEBUG - printf("StreamTcpPacketStateNone (%p): ssn->client.isn %" PRIu32 ", ssn->client.next_seq %" PRIu32 "\n", - ssn, ssn->client.isn, ssn->client.next_seq); + printf("StreamTcpPacketStateNone (%p): ssn->client.isn %" PRIu32 ", ssn->client.next_seq %" PRIu32 ", ssn->client.last_ack %"PRIu32"\n", + ssn, ssn->client.isn, ssn->client.next_seq, ssn->client.last_ack); #endif if (p->tcpvars.ws != NULL) { #ifdef DEBUG printf("StreamTcpPacketStateNone (%p): p->tcpvars.ws %p, %02x\n", ssn, p->tcpvars.ws, *p->tcpvars.ws->data); #endif - ssn->client.wscale = *p->tcpvars.ws->data; + ssn->server.wscale = *p->tcpvars.ws->data; } break; } case TH_SYN|TH_ACK: if (stream_config.midstream == FALSE) break; + if (ssn == NULL) { ssn = StreamTcpNewSession(p); if (ssn == NULL) return -1; + PerfCounterIncr(stt->counter_tcp_sessions, tv->pca); } /* set the state */ - ssn->state = TCP_SYN_RECV; + StreamTcpPacketSetState(p, ssn, TCP_SYN_RECV); #ifdef DEBUG printf("StreamTcpPacketStateNone (%p): =~ midstream picked ssn state is now TCP_SYN_RECV\n", ssn); #endif @@ -277,21 +344,21 @@ static int StreamTcpPacketStateNone(ThreadVars *tv, Packet *p, StreamTcpThread * ssn->client.last_ack = TCP_GET_ACK(p); #ifdef DEBUG - printf("StreamTcpPacketStateNone (%p): ssn->client.isn %u, ssn->client.next_seq %u\n", - ssn, ssn->client.isn, ssn->client.next_seq); + printf("StreamTcpPacketStateNone (%p): ssn->client.isn %"PRIu32", ssn->client.next_seq %"PRIu32", ssn->client.last_ack %"PRIu32"\n", + ssn, ssn->client.isn, ssn->client.next_seq, ssn->client.last_ack); #endif if (p->tcpvars.ws != NULL) { #ifdef DEBUG printf("StreamTcpPacketStateNone (%p): p->tcpvars.ws %p, %02x\n", ssn, p->tcpvars.ws, *p->tcpvars.ws->data); #endif - ssn->server.wscale = *p->tcpvars.ws->data; + ssn->client.wscale = *p->tcpvars.ws->data; } #ifdef DEBUG - printf("StreamTcpPacketStateNone (%p): ssn->server.isn %u, ssn->server.next_seq %u, ssn->CLIENT.last_ack %u\n", - ssn, ssn->server.isn, ssn->server.next_seq, ssn->client.last_ack); + printf("StreamTcpPacketStateNone (%p): ssn->server.isn %"PRIu32", ssn->server.next_seq %"PRIu32", ssn->server.last_ack %"PRIu32"\n", + ssn, ssn->server.isn, ssn->server.next_seq, ssn->server.last_ack); #endif break; - /*Handle SYN/ACK and 3WHS shake missed together as it is almost similar. ryt ?*/ + /* Handle SYN/ACK and 3WHS shake missed together as it is almost similar. */ case TH_ACK: case TH_ACK|TH_PUSH: if (stream_config.midstream == FALSE) @@ -300,9 +367,10 @@ static int StreamTcpPacketStateNone(ThreadVars *tv, Packet *p, StreamTcpThread * ssn = StreamTcpNewSession(p); if (ssn == NULL) return -1; + PerfCounterIncr(stt->counter_tcp_sessions, tv->pca); } /* set the state */ - ssn->state = TCP_ESTABLISHED; + StreamTcpPacketSetState(p, ssn, TCP_ESTABLISHED); #ifdef DEBUG printf("StreamTcpPacketStateNone (%p): =~ midstream picked ssn state is now TCP_ESTABLISHED\n", ssn); #endif @@ -324,6 +392,14 @@ static int StreamTcpPacketStateNone(ThreadVars *tv, Packet *p, StreamTcpThread * ssn->server.ra_base_seq = ssn->server.isn; ssn->server.next_seq = ssn->server.isn + 1; ssn->server.last_ack = TCP_GET_ACK(p); + ssn->server.next_win = ssn->server.last_ack; + +#ifdef DEBUG + printf("StreamTcpPacketStateNone (%p): ssn->client.next_win %"PRIu32", ssn->server.next_win %"PRIu32"\n", + ssn, ssn->client.next_win, ssn->server.next_win); + printf("StreamTcpPacketStateNone (%p): ssn->client.last_ack %"PRIu32", ssn->server.last_ack %"PRIu32"\n", + ssn, ssn->client.last_ack, ssn->server.last_ack); +#endif /** \todo window scaling for midstream pickups */ ssn->client.wscale = 0; @@ -337,7 +413,10 @@ static int StreamTcpPacketStateNone(ThreadVars *tv, Packet *p, StreamTcpThread * case TH_FIN: case TH_FIN|TH_ACK: case TH_FIN|TH_ACK|TH_PUSH: - p->flow->stream = NULL; + if (p->flow->protoctx != NULL) { + printf("p->flow->protoctx %p\n", p->flow->protoctx); + } + BUG_ON(p->flow->protoctx != NULL); #ifdef DEBUG printf ("StreamTcpPacketStateNone: FIN or RST packet received, no session setup\n"); #endif @@ -390,7 +469,7 @@ static int StreamTcpPacketStateSynSent(ThreadVars *tv, Packet *p, StreamTcpThrea } /* update state */ - ssn->state = TCP_SYN_RECV; + StreamTcpPacketSetState(p, ssn, TCP_SYN_RECV); #ifdef DEBUG printf("StreamTcpPacketStateSynSent (%p): =~ ssn state is now TCP_SYN_RECV\n", ssn); #endif @@ -398,37 +477,37 @@ static int StreamTcpPacketStateSynSent(ThreadVars *tv, Packet *p, StreamTcpThrea ssn->server.isn = TCP_GET_SEQ(p); ssn->server.ra_base_seq = ssn->server.isn; ssn->server.next_seq = ssn->server.isn + 1; - ssn->server.window = TCP_GET_WINDOW(p); + + ssn->client.window = TCP_GET_WINDOW(p); #ifdef DEBUG - printf("StreamTcpPacketStateSynSent: (%p): window %" PRIu32 "\n", ssn, ssn->server.window); + printf("StreamTcpPacketStateSynSent (%p): window %" PRIu32 "\n", ssn, ssn->server.window); #endif ssn->client.last_ack = TCP_GET_ACK(p); ssn->server.last_ack = ssn->server.isn + 1; - if (ssn->client.wscale != 0 && p->tcpvars.ws != NULL) { + if (ssn->server.wscale != 0 && p->tcpvars.ws != NULL) { #ifdef DEBUG printf("StreamTcpPacketStateSynSent (%p): p->tcpvars.ws %p, %02x\n", ssn, p->tcpvars.ws, *p->tcpvars.ws->data); #endif - ssn->server.wscale = *p->tcpvars.ws->data; + ssn->client.wscale = *p->tcpvars.ws->data; } else { ssn->client.wscale = 0; } ssn->server.next_win = ssn->server.last_ack + ssn->server.window; + ssn->client.next_win = ssn->client.last_ack + ssn->client.window; #ifdef DEBUG - printf("StreamTcpPacketStateSynSent (%p): next_win %" PRIu32 "\n", ssn, ssn->server.next_win); - printf("StreamTcpPacketStateSynSent (%p): ssn->server.isn %" PRIu32 ", ssn->server.next_seq %" PRIu32 ", ssn->CLIENT.last_ack %" PRIu32 "\n", - ssn, ssn->server.isn, ssn->server.next_seq, ssn->client.last_ack); + printf("StreamTcpPacketStateSynSent (%p): ssn->server.next_win %" PRIu32 "\n", ssn, ssn->server.next_win); + printf("StreamTcpPacketStateSynSent (%p): ssn->client.next_win %" PRIu32 "\n", ssn, ssn->client.next_win); + printf("StreamTcpPacketStateSynSent (%p): ssn->server.isn %" PRIu32 ", ssn->server.next_seq %" PRIu32 ", ssn->server.last_ack %" PRIu32 " (ssn->client.last_ack %" PRIu32 ")\n", + ssn, ssn->server.isn, ssn->server.next_seq, ssn->server.last_ack, ssn->client.last_ack); #endif break; case TH_RST: case TH_RST|TH_ACK: - /* seq should be 0, win should be 0, ack should be isn +1. - * check Snort's stream4/5 for more security*/ if(ValidReset(ssn, p)){ if(SEQ_EQ(TCP_GET_SEQ(p), ssn->client.isn) && SEQ_EQ(TCP_GET_WINDOW(p), 0) && SEQ_EQ(TCP_GET_ACK(p), (ssn->client.isn + 1))) { - ssn->state = TCP_CLOSED; - //StreamTcpSessionPktFree(p); + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); } } else return -1; @@ -495,23 +574,22 @@ static int StreamTcpPacketStateSynRecv(ThreadVars *tv, Packet *p, StreamTcpThrea printf("StreamTcpPacketStateSynRecv (%p): pkt (%" PRIu32 ") is to server: SEQ %" PRIu32 ", ACK %" PRIu32 "\n", ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p)); #endif - ssn->state = TCP_ESTABLISHED; + StreamTcpPacketSetState(p, ssn, TCP_ESTABLISHED); #ifdef DEBUG printf("StreamTcpPacketStateSynRecv (%p): =~ ssn state is now TCP_ESTABLISHED\n", ssn); #endif ssn->client.next_seq += p->payload_len; - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; - ssn->client.next_win = ssn->client.last_ack + ssn->client.window; + ssn->server.next_win = ssn->server.last_ack + ssn->server.window; #ifdef DEBUG - printf("StreamTcpPacketStateSynRecv (%p): next_win %" PRIu32 "\n", ssn, ssn->client.next_win); + printf("StreamTcpPacketStateSynRecv (%p): ssn->server.next_win %" PRIu32 ", ssn->server.last_ack %"PRIu32"\n", ssn, ssn->server.next_win, ssn->server.last_ack); #endif break; case TH_RST: case TH_RST|TH_ACK: if(ValidReset(ssn, p)) { - ssn->state = TCP_CLOSED; - //StreamTcpSessionPktFree(p); + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); } else return -1; break; @@ -560,8 +638,8 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, StreamTcpT case TH_ACK|TH_PUSH: if (PKT_IS_TOSERVER(p)) { #ifdef DEBUG - printf("StreamTcpPacketStateEstablished (%p): =+ pkt (%" PRIu32 ") is to server: SEQ %" PRIu32 ", ACK %" PRIu32 "\n", - ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p)); + printf("StreamTcpPacketStateEstablished (%p): =+ pkt (%" PRIu32 ") is to server: SEQ %" PRIu32 ", ACK %" PRIu32 ", WIN %"PRIu16"\n", + ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p), TCP_GET_WINDOW(p)); #endif if (SEQ_EQ(ssn->client.next_seq, TCP_GET_SEQ(p))) { ssn->client.next_seq += p->payload_len; @@ -570,27 +648,40 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, StreamTcpT #endif } - if (SEQ_GEQ(TCP_GET_SEQ(p), ssn->client.last_ack) && - SEQ_LEQ(TCP_GET_SEQ(p) + p->payload_len, ssn->client.next_win)) { + if (SEQ_GEQ(TCP_GET_SEQ(p), ssn->client.last_ack)) { + if (SEQ_LEQ(TCP_GET_SEQ(p) + p->payload_len, ssn->client.next_win) || + ssn->flags & STREAMTCP_FLAG_MIDSTREAM) { +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): seq %"PRIu32" in window, ssn->client.next_win %" PRIu32 "\n", ssn, TCP_GET_SEQ(p), ssn->client.next_win); +#endif + + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): ssn->server.window %"PRIu32"\n", ssn, ssn->server.window); +#endif - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) + ssn->server.last_ack = TCP_GET_ACK(p); - if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) - ssn->server.last_ack = TCP_GET_ACK(p); + if (SEQ_GT((ssn->server.last_ack + ssn->server.window), ssn->server.next_win)) { + ssn->server.next_win = ssn->server.last_ack + ssn->server.window; +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): seq %"PRIu32", updated ssn->server.next_win %" PRIu32 " (win %"PRIu32")\n", ssn, TCP_GET_SEQ(p), ssn->server.next_win, ssn->server.window); +#endif + } - if (SEQ_GT(ssn->client.last_ack + ssn->client.window, ssn->client.next_win)) { - ssn->client.next_win = ssn->client.last_ack + ssn->client.window; + StreamTcpReassembleHandleSegment(ssn, &ssn->client, p); + } else { +//#define DEBUG #ifdef DEBUG - printf("StreamTcpPacketStateEstablished (%p): ssn->client.next_win %" PRIu32 "\n", ssn, ssn->client.next_win); + printf("StreamTcpPacketStateEstablished (%p): server => SEQ out of window, packet SEQ %" PRIu32 ", payload size %" PRIu32 " (%" PRIu32 "), ssn->client.last_ack %" PRIu32 ", ssn->client.next_win %" PRIu32 "(%"PRIu32") (ssn->client.ra_base_seq %"PRIu32")\n", ssn, TCP_GET_SEQ(p), p->payload_len, TCP_GET_SEQ(p) + p->payload_len, ssn->client.last_ack, ssn->client.next_win, TCP_GET_SEQ(p) + p->payload_len - ssn->client.next_win, ssn->client.ra_base_seq); #endif } - - StreamTcpReassembleHandleSegment(ssn, &ssn->client, p); } else { #ifdef DEBUG - printf("StreamTcpPacketStateEstablished (%p): server !!!!! => SEQ mismatch, packet SEQ %" PRIu32 ", payload size %" PRIu32 " (%" PRIu32 "), last_ack %" PRIu32 ", next_win %" PRIu32 "\n", - ssn, TCP_GET_SEQ(p), p->payload_len, TCP_GET_SEQ(p) + p->payload_len, ssn->client.last_ack, ssn->client.next_win); + printf("StreamTcpPacketStateEstablished (%p): server => SEQ before last_ack, packet SEQ %" PRIu32 ", payload size %" PRIu32 " (%" PRIu32 "), ssn->client.last_ack %" PRIu32 ", ssn->client.next_win %" PRIu32 "(%"PRIu32") (ssn->client.ra_base_seq %"PRIu32")\n", ssn, TCP_GET_SEQ(p), p->payload_len, TCP_GET_SEQ(p) + p->payload_len, ssn->client.last_ack, ssn->client.next_win, TCP_GET_SEQ(p) + p->payload_len - ssn->client.next_win, ssn->client.ra_base_seq); #endif +//#undef DEBUG } #ifdef DEBUG @@ -599,16 +690,19 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, StreamTcpT #endif } else { /* implied to client */ #ifdef DEBUG - printf("StreamTcpPacketStateEstablished (%p): =+ pkt (%" PRIu32 ") is to client: SEQ %" PRIu32 ", ACK %" PRIu32 "\n", - ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p)); + printf("StreamTcpPacketStateEstablished (%p): =+ pkt (%" PRIu32 ") is to client: SEQ %" PRIu32 ", ACK %" PRIu32 ", WIN %"PRIu16"\n", + ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p), TCP_GET_WINDOW(p)); #endif - /*To get the server window value from the servers packet, when connection - is picked up as midstream*/ + /* To get the server window value from the servers packet, when connection + is picked up as midstream */ if ((ssn->flags & STREAMTCP_FLAG_MIDSTREAM) && (ssn->flags & STREAMTCP_FLAG_MIDSTREAM_ESTABLISHED)) { ssn->server.window = TCP_GET_WINDOW(p); ssn->server.next_win = ssn->server.last_ack + ssn->server.window; - ssn->flags = STREAMTCP_FLAG_MIDSTREAM; + ssn->flags &= ~STREAMTCP_FLAG_MIDSTREAM_ESTABLISHED; +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): adjusted midstream ssn->server.next_win to %" PRIu32 "\n", ssn, ssn->server.next_win); +#endif } if (SEQ_EQ(ssn->server.next_seq, TCP_GET_SEQ(p))) { @@ -618,26 +712,40 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, StreamTcpT #endif } - if (SEQ_GEQ(TCP_GET_SEQ(p), ssn->server.last_ack) && - SEQ_LEQ(TCP_GET_SEQ(p) + p->payload_len, ssn->server.next_win)) { + if (SEQ_GEQ(TCP_GET_SEQ(p), ssn->server.last_ack)) { + if (SEQ_LEQ(TCP_GET_SEQ(p) + p->payload_len, ssn->server.next_win) || + ssn->flags & STREAMTCP_FLAG_MIDSTREAM) { +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): seq %"PRIu32" in window, ssn->server.next_win %" PRIu32 "\n", ssn, TCP_GET_SEQ(p), ssn->server.next_win); +#endif + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): ssn->client.window %"PRIu32"\n", ssn, ssn->client.window); +#endif - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) + ssn->client.last_ack = TCP_GET_ACK(p); - if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) - ssn->client.last_ack = TCP_GET_ACK(p); + if (SEQ_GT((ssn->client.last_ack + ssn->client.window), ssn->client.next_win)) { + ssn->client.next_win = ssn->client.last_ack + ssn->client.window; +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): seq %"PRIu32", updated ssn->client.next_win %" PRIu32 " (win %"PRIu32")\n", ssn, TCP_GET_SEQ(p), ssn->client.next_win, ssn->client.window); +#endif + } else { +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): seq %"PRIu32", keeping ssn->client.next_win %" PRIu32 " the same (win %"PRIu32")\n", ssn, TCP_GET_SEQ(p), ssn->client.next_win, ssn->client.window); +#endif + } - if (SEQ_GT(ssn->server.last_ack + ssn->server.window, ssn->server.next_win)) { - ssn->server.next_win = ssn->server.last_ack + ssn->server.window; + StreamTcpReassembleHandleSegment(ssn, &ssn->server, p); + } else { #ifdef DEBUG - printf("StreamTcpPacketStateEstablished (%p): ssn->server.next_win %" PRIu32 "\n", ssn, ssn->server.next_win); + printf("StreamTcpPacketStateEstablished (%p): client => SEQ out of window, packet SEQ %" PRIu32 ", payload size %" PRIu32 " (%" PRIu32 "), ssn->server.last_ack %" PRIu32 ", ssn->server.next_win %" PRIu32 "(%"PRIu32") (ssn->server.ra_base_seq %"PRIu32")\n", ssn, TCP_GET_SEQ(p), p->payload_len, TCP_GET_SEQ(p) + p->payload_len, ssn->server.last_ack, ssn->server.next_win, TCP_GET_SEQ(p) + p->payload_len - ssn->server.next_win, ssn->server.ra_base_seq); #endif } - - StreamTcpReassembleHandleSegment(ssn, &ssn->server, p); } else { #ifdef DEBUG - printf("StreamTcpPacketStateEstablished (%p): client !!!!! => SEQ mismatch, packet SEQ %" PRIu32 ", payload size %" PRIu32 " (%" PRIu32 "), last_ack %" PRIu32 ", next_win %" PRIu32 "\n", - ssn, TCP_GET_SEQ(p), p->payload_len, TCP_GET_SEQ(p) + p->payload_len, ssn->server.last_ack, ssn->server.next_win); + printf("StreamTcpPacketStateEstablished (%p): client => SEQ before last ack, packet SEQ %" PRIu32 ", payload size %" PRIu32 " (%" PRIu32 "), ssn->server.last_ack %" PRIu32 ", ssn->server.next_win %" PRIu32 "(%"PRIu32") (ssn->server.ra_base_seq %"PRIu32")\n", ssn, TCP_GET_SEQ(p), p->payload_len, TCP_GET_SEQ(p) + p->payload_len, ssn->server.last_ack, ssn->server.next_win, TCP_GET_SEQ(p) + p->payload_len - ssn->server.next_win, ssn->server.ra_base_seq); #endif } #ifdef DEBUG @@ -649,6 +757,10 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, StreamTcpT case TH_FIN: case TH_FIN|TH_ACK: case TH_FIN|TH_ACK|TH_PUSH: +#ifdef DEBUG + printf("StreamTcpPacketStateEstablished (%p): FIN received SEQ %" PRIu32 ", last ACK %" PRIu32 ", next win %" PRIu32 ", win %" PRIu32 "\n", + ssn, ssn->server.next_seq, ssn->client.last_ack, ssn->server.next_win, ssn->server.window); +#endif if((StreamTcpHandleFin(ssn, p)) == -1) return -1; break; @@ -659,7 +771,7 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, StreamTcpT #ifdef DEBUG printf("StreamTcpPacketStateEstablished (%p): Reset received and state changed to TCP_CLOSED\n", ssn); #endif - ssn->state = TCP_CLOSED; + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); /*Similar remote application is closed, so jump to CLOSE_WAIT*/ ssn->client.next_seq = TCP_GET_ACK(p); ssn->server.next_seq = TCP_GET_SEQ(p) + p->payload_len + 1; @@ -676,12 +788,11 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, StreamTcpT printf("StreamTcpPacketStateEstablished (%p): =+ next SEQ %" PRIu32 ", last ACK %" PRIu32 "\n", ssn, ssn->client.next_seq, ssn->server.last_ack); #endif - //StreamTcpSessionPktFree(p); } else { #ifdef DEBUG printf("StreamTcpPacketStateEstablished (%p): Reset received and state changed to TCP_CLOSED\n", ssn); #endif - ssn->state = TCP_CLOSED; + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); /*Similar remote application is closed, so jump to CLOSE_WAIT*/ ssn->server.next_seq = TCP_GET_SEQ(p) + p->payload_len + 1; ssn->client.next_seq = TCP_GET_ACK(p); @@ -698,7 +809,6 @@ static int StreamTcpPacketStateEstablished(ThreadVars *tv, Packet *p, StreamTcpT printf("StreamTcpPacketStateEstablished (%p): =+ next SEQ %" PRIu32 ", last ACK %" PRIu32 "\n", ssn, ssn->server.next_seq, ssn->client.last_ack); #endif - //StreamTcpSessionPktFree(p); } } else return -1; @@ -725,28 +835,28 @@ static int StreamTcpHandleFin(TcpSession *ssn, Packet *p) { if (PKT_IS_TOSERVER(p)) { #ifdef DEBUG - printf("StreamTcpPacket (%p): pkt (%" PRIu32 ") is to server: SEQ %" PRIu32 ", ACK %" PRIu32 "\n", + printf("StreamTcpHandleFin (%p): pkt (%" PRIu32 ") is to server: SEQ %" PRIu32 ", ACK %" PRIu32 "\n", ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p)); #endif if (SEQ_LT(TCP_GET_SEQ(p), ssn->client.next_seq) || SEQ_GT(TCP_GET_SEQ(p), (ssn->client.last_ack + ssn->client.window))) { #ifdef DEBUG - printf("StreamTcpPacket (%p): -> SEQ mismatch, packet SEQ %" PRIu32 " != %" PRIu32 " from stream\n", + printf("StreamTcpHandleFin (%p): -> SEQ mismatch, packet SEQ %" PRIu32 " != %" PRIu32 " from stream\n", ssn, TCP_GET_SEQ(p), ssn->client.next_seq); #endif return -1; } #ifdef DEBUG - printf("StreamTcpPacket (%p): state changed to TCP_CLOSE_WAIT\n", ssn); + printf("StreamTcpHandleFin (%p): state changed to TCP_CLOSE_WAIT\n", ssn); #endif - ssn->state = TCP_CLOSE_WAIT; + StreamTcpPacketSetState(p, ssn, TCP_CLOSE_WAIT); ssn->client.next_seq = TCP_GET_ACK(p); ssn->server.next_seq = TCP_GET_SEQ(p) + p->payload_len + 1; #ifdef DEBUG - printf("StreamTcpPacket (%p): ssn->server.next_seq %" PRIu32 "\n", ssn, ssn->server.next_seq); + printf("StreamTcpHandleFin (%p): ssn->server.next_seq %" PRIu32 "\n", ssn, ssn->server.next_seq); #endif - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) ssn->server.last_ack = TCP_GET_ACK(p); @@ -754,40 +864,40 @@ static int StreamTcpHandleFin(TcpSession *ssn, Packet *p) { StreamTcpReassembleHandleSegment(ssn, &ssn->client, p); #ifdef DEBUG - printf("StreamTcpPacket (%p): =+ next SEQ %" PRIu32 ", last ACK %" PRIu32 "\n", + printf("StreamTcpHandleFin (%p): =+ next SEQ %" PRIu32 ", last ACK %" PRIu32 "\n", ssn, ssn->client.next_seq, ssn->server.last_ack); #endif } else { /* implied to client */ #ifdef DEBUG - printf("StreamTcpPacket (%p): pkt (%" PRIu32 ") is to client: SEQ %" PRIu32 ", ACK %" PRIu32 "\n", + printf("StreamTcpHandleFin (%p): pkt (%" PRIu32 ") is to client: SEQ %" PRIu32 ", ACK %" PRIu32 "\n", ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p)); #endif if (SEQ_LT(TCP_GET_SEQ(p), ssn->server.next_seq) || SEQ_GT(TCP_GET_SEQ(p), (ssn->server.last_ack + ssn->server.window))) { #ifdef DEBUG - printf("StreamTcpPacket (%p): -> SEQ mismatch, packet SEQ %" PRIu32 " != %" PRIu32 " from stream\n", + printf("StreamTcpHandleFin (%p): -> SEQ mismatch, packet SEQ %" PRIu32 " != %" PRIu32 " from stream\n", ssn, TCP_GET_SEQ(p), ssn->server.next_seq); #endif return -1; } #ifdef DEBUG - printf("StreamTcpPacket (%p): state changed to TCP_FIN_WAIT1\n", ssn); + printf("StreamTcpHandleFin (%p): state changed to TCP_FIN_WAIT1\n", ssn); #endif - ssn->state = TCP_FIN_WAIT1; + StreamTcpPacketSetState(p, ssn, TCP_FIN_WAIT1); ssn->server.next_seq = TCP_GET_SEQ(p) + p->payload_len + 1; ssn->client.next_seq = TCP_GET_ACK(p); #ifdef DEBUG - printf("StreamTcpPacket (%p): ssn->server.next_seq %" PRIu32 "\n", ssn, ssn->server.next_seq); + printf("StreamTcpHandleFin (%p): ssn->server.next_seq %" PRIu32 "\n", ssn, ssn->server.next_seq); #endif - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) - ssn->client.last_ack = TCP_GET_ACK(p); + ssn->client.last_ack = TCP_GET_ACK(p); StreamTcpReassembleHandleSegment(ssn, &ssn->server, p); #ifdef DEBUG - printf("StreamTcpPacket (%p): =+ next SEQ %" PRIu32 ", last ACK %" PRIu32 "\n", + printf("StreamTcpHandleFin (%p): =+ next SEQ %" PRIu32 ", last ACK %" PRIu32 "\n", ssn, ssn->server.next_seq, ssn->client.last_ack); #endif } @@ -816,9 +926,9 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p, StreamTcpThre ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p)); printf("StreamTcpPacketStateFinWait1 (%p): state changed to TCP_FIN_WAIT2\n", ssn); #endif - ssn->state = TCP_FIN_WAIT2; + StreamTcpPacketSetState(p, ssn, TCP_FIN_WAIT2); - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) ssn->server.last_ack = TCP_GET_ACK(p); @@ -834,8 +944,8 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p, StreamTcpThre ssn, p->payload_len, TCP_GET_SEQ(p), TCP_GET_ACK(p)); printf("StreamTcpPacketStateFinWait1 (%p): state changed to TCP_FIN_WAIT2\n", ssn); #endif - ssn->state = TCP_FIN_WAIT2; - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + StreamTcpPacketSetState(p, ssn, TCP_FIN_WAIT2); + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) ssn->client.last_ack = TCP_GET_ACK(p); @@ -865,9 +975,9 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateFinWait1 (%p): state changed to TCP_TIME_WAIT\n", ssn); #endif - ssn->state = TCP_TIME_WAIT; + StreamTcpPacketSetState(p, ssn, TCP_TIME_WAIT); - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) ssn->server.last_ack = TCP_GET_ACK(p); @@ -893,8 +1003,8 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateFinWait1 (%p): state changed to TCP_TIME_WAIT\n", ssn); #endif - ssn->state = TCP_TIME_WAIT; - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + StreamTcpPacketSetState(p, ssn, TCP_TIME_WAIT); + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) ssn->client.last_ack = TCP_GET_ACK(p); @@ -912,7 +1022,7 @@ static int StreamTcpPacketStateFinWait1(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateFinWait1 (%p): Reset received state changed to TCP_CLOSED\n", ssn); #endif - ssn->state = TCP_CLOSED; + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); //StreamTcpSessionPktFree(p); } else @@ -960,8 +1070,8 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateFinWait2 (%p): state changed to TCP_TIME_WAIT\n", ssn); #endif - ssn->state = TCP_TIME_WAIT; - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + StreamTcpPacketSetState(p, ssn, TCP_TIME_WAIT); + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) ssn->server.last_ack = TCP_GET_ACK(p); @@ -986,8 +1096,8 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateFinWait2 (%p): state changed to TCP_TIME_WAIT\n", ssn); #endif - ssn->state = TCP_TIME_WAIT; - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + StreamTcpPacketSetState(p, ssn, TCP_TIME_WAIT); + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) ssn->client.last_ack = TCP_GET_ACK(p); @@ -1005,7 +1115,7 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateFinWait2 (%p): Reset received state changed to TCP_CLOSED\n", ssn); #endif - ssn->state = TCP_CLOSED; + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); //StreamTcpSessionPktFree(p); } else @@ -1027,8 +1137,8 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateFinWait2 (%p): state changed to TCP_TIME_WAIT\n", ssn); #endif - ssn->state = TCP_TIME_WAIT; - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + StreamTcpPacketSetState(p, ssn, TCP_TIME_WAIT); + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) ssn->server.last_ack = TCP_GET_ACK(p); @@ -1055,8 +1165,8 @@ static int StreamTcpPacketStateFinWait2(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateFinWait2 (%p): state changed to TCP_TIME_WAIT\n", ssn); #endif - ssn->state = TCP_TIME_WAIT; - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + StreamTcpPacketSetState(p, ssn, TCP_TIME_WAIT); + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) ssn->client.last_ack = TCP_GET_ACK(p); @@ -1111,7 +1221,7 @@ static int StreamTcpPacketStateClosing(ThreadVars *tv, Packet *p, StreamTcpThrea #ifdef DEBUG printf("StreamTcpPacketStateClosing (%p): state changed to TCP_TIME_WAIT\n", ssn); #endif - ssn->state = TCP_TIME_WAIT; + StreamTcpPacketSetState(p, ssn, TCP_TIME_WAIT); ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) @@ -1139,8 +1249,8 @@ static int StreamTcpPacketStateClosing(ThreadVars *tv, Packet *p, StreamTcpThrea #ifdef DEBUG printf("StreamTcpPacketStateClosing (%p): state changed to TCP_TIME_WAIT\n", ssn); #endif - ssn->state = TCP_TIME_WAIT; - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + StreamTcpPacketSetState(p, ssn, TCP_TIME_WAIT); + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) ssn->client.last_ack = TCP_GET_ACK(p); @@ -1194,8 +1304,8 @@ static int StreamTcpPacketStateCloseWait(ThreadVars *tv, Packet *p, StreamTcpThr #ifdef DEBUG printf("StreamTcpPacketStateCloseWait (%p): state changed to TCP_LAST_ACK\n", ssn); #endif - ssn->state = TCP_LAST_ACK; - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + StreamTcpPacketSetState(p, ssn, TCP_LAST_ACK); + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) ssn->client.last_ack = TCP_GET_ACK(p); @@ -1247,8 +1357,8 @@ static int StreamTcpPakcetStateLastAck(ThreadVars *tv, Packet *p, StreamTcpThrea #ifdef DEBUG printf("StreamTcpPacketStateLastAck (%p): state changed to TCP_CLOSED\n", ssn); #endif - ssn->state = TCP_CLOSED; - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) ssn->server.last_ack = TCP_GET_ACK(p); @@ -1258,7 +1368,6 @@ static int StreamTcpPakcetStateLastAck(ThreadVars *tv, Packet *p, StreamTcpThrea printf("StreamTcpPacketStateLastAck (%p): =+ next SEQ %" PRIu32 ", last ACK %" PRIu32 "\n", ssn, ssn->client.next_seq, ssn->server.last_ack); #endif - //StreamTcpSessionPktFree(p); } break; default: @@ -1299,8 +1408,9 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateTimeWait (%p): state changed to TCP_CLOSED\n", ssn); #endif - ssn->state = TCP_CLOSED; - ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); + + ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->server.last_ack)) ssn->server.last_ack = TCP_GET_ACK(p); @@ -1326,9 +1436,9 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p, StreamTcpThre #ifdef DEBUG printf("StreamTcpPacketStateTimeWait (%p): state changed to TCP_CLOSED\n", ssn); #endif - ssn->state = TCP_CLOSED; - ssn->server.window = TCP_GET_WINDOW(p) << ssn->server.wscale; + StreamTcpPacketSetState(p, ssn, TCP_CLOSED); + ssn->client.window = TCP_GET_WINDOW(p) << ssn->client.wscale; if (SEQ_GT(TCP_GET_ACK(p),ssn->client.last_ack)) ssn->client.last_ack = TCP_GET_ACK(p); @@ -1352,9 +1462,9 @@ static int StreamTcpPacketStateTimeWait(ThreadVars *tv, Packet *p, StreamTcpThre /* flow is and stays locked */ static int StreamTcpPacket (ThreadVars *tv, Packet *p, StreamTcpThread *stt) { - TcpSession *ssn = (TcpSession *)p->flow->stream; + TcpSession *ssn = (TcpSession *)p->flow->protoctx; - if (ssn == NULL || ssn->state == 0 || ssn->state == TCP_CLOSED) { + if (ssn == NULL || ssn->state == TCP_NONE) { if (StreamTcpPacketStateNone(tv, p, stt, ssn) == -1) return -1; } else { @@ -1395,6 +1505,12 @@ static int StreamTcpPacket (ThreadVars *tv, Packet *p, StreamTcpThread *stt) { if(StreamTcpPacketStateTimeWait(tv, p, stt, ssn)) return -1; break; + case TCP_CLOSED: + //printf("StreamTcpPacket: packet received on closed state\n"); + break; + default: + //printf("StreamTcpPacket: packet received on default state\n"); + break; } } @@ -1405,8 +1521,6 @@ int StreamTcp (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq) { StreamTcpThread *stt = (StreamTcpThread *)data; - // PerfCounterAddUI64(stt->counter_tcp_streams, tv->pca, a); - if (!(PKT_IS_TCP(p))) return 0; @@ -1433,7 +1547,7 @@ int StreamTcp (ThreadVars *tv, Packet *p, void *data, PacketQueue *pq) return 0; } -int StreamTcpThreadInit(ThreadVars *t, void *initdata, void **data) +int StreamTcpThreadInit(ThreadVars *tv, void *initdata, void **data) { StreamTcpThread *stt = malloc(sizeof(StreamTcpThread)); if (stt == NULL) { @@ -1445,17 +1559,14 @@ int StreamTcpThreadInit(ThreadVars *t, void *initdata, void **data) *data = (void *)stt; - stt->counter_tcp_streams = PerfTVRegisterAvgCounter("streamtcp.tcp_streams", - t, TYPE_DOUBLE, "NULL"); - - t->pca = PerfGetAllCountersArray(&t->pctx); - - PerfAddToClubbedTMTable(t->name, &t->pctx); + stt->counter_tcp_sessions = PerfTVRegisterCounter("tcp.sessions", tv, TYPE_UINT64, "NULL"); + tv->pca = PerfGetAllCountersArray(&tv->pctx); + PerfAddToClubbedTMTable(tv->name, &tv->pctx); return 0; } -int StreamTcpThreadDeinit(ThreadVars *t, void *data) +int StreamTcpThreadDeinit(ThreadVars *tv, void *data) { StreamTcpThread *stt = (StreamTcpThread *)data; if (stt == NULL) { @@ -1491,22 +1602,16 @@ void StreamTcpExitPrintStats(ThreadVars *tv, void *data) { static int ValidReset(TcpSession *ssn, Packet *p) { uint8_t os_policy; + if (PKT_IS_TOSERVER(p)) os_policy = ssn->server.os_policy; else os_policy = ssn->client.os_policy; + switch(os_policy) { - case OS_POLICY_BSD: - case OS_POLICY_FIRST: - case OS_POLICY_HPUX10: - case OS_POLICY_IRIX: - case OS_POLICY_MACOS: - case OS_POLICY_LAST: - case OS_POLICY_WINDOWS: - case OS_POLICY_WINDOWS2K3: - case OS_POLICY_VISTA: + case OS_POLICY_HPUX11: if(PKT_IS_TOSERVER(p)){ - if(SEQ_EQ(TCP_GET_SEQ(p), ssn->client.next_seq)) { + if(SEQ_GEQ(TCP_GET_SEQ(p), ssn->client.next_seq)) { #ifdef DEBUG printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); #endif @@ -1518,7 +1623,7 @@ static int ValidReset(TcpSession *ssn, Packet *p) { return 0; } } else { /* implied to client */ - if(SEQ_EQ(TCP_GET_SEQ(p), ssn->server.next_seq)) { + if(SEQ_GEQ(TCP_GET_SEQ(p), ssn->server.next_seq)) { #ifdef DEBUG printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); #endif @@ -1531,13 +1636,17 @@ static int ValidReset(TcpSession *ssn, Packet *p) { } } break; - case OS_POLICY_HPUX11: + case OS_POLICY_OLD_LINUX: + case OS_POLICY_LINUX: + case OS_POLICY_SOLARIS: if(PKT_IS_TOSERVER(p)){ - if(SEQ_GEQ(TCP_GET_SEQ(p), ssn->client.next_seq)) { + if(SEQ_GEQ((TCP_GET_SEQ(p)+p->payload_len), ssn->client.last_ack)) { /*window base is needed !!*/ + if(SEQ_LT(TCP_GET_SEQ(p), (ssn->client.next_seq + ssn->client.window))) { #ifdef DEBUG - printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); + printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); #endif - return 1; + return 1; + } } else { #ifdef DEBUG printf("Reset is not Valid! Packet SEQ: %" PRIu32 " and server SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p), ssn->client.next_seq); @@ -1545,11 +1654,13 @@ static int ValidReset(TcpSession *ssn, Packet *p) { return 0; } } else { /* implied to client */ - if(SEQ_GEQ(TCP_GET_SEQ(p), ssn->server.next_seq)) { + if(SEQ_GEQ((TCP_GET_SEQ(p) + p->payload_len), ssn->server.last_ack)) { /*window base is needed !!*/ + if(SEQ_LT(TCP_GET_SEQ(p), (ssn->server.next_seq + ssn->server.window))) { #ifdef DEBUG - printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); + printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); #endif - return 1; + return 1; + } } else { #ifdef DEBUG printf("Reset is not Valid! Packet SEQ: %" PRIu32 " and client SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p), ssn->server.next_seq); @@ -1558,17 +1669,22 @@ static int ValidReset(TcpSession *ssn, Packet *p) { } } break; - case OS_POLICY_OLD_LINUX: - case OS_POLICY_LINUX: - case OS_POLICY_SOLARIS: + default: + case OS_POLICY_BSD: + case OS_POLICY_FIRST: + case OS_POLICY_HPUX10: + case OS_POLICY_IRIX: + case OS_POLICY_MACOS: + case OS_POLICY_LAST: + case OS_POLICY_WINDOWS: + case OS_POLICY_WINDOWS2K3: + case OS_POLICY_VISTA: if(PKT_IS_TOSERVER(p)){ - if(SEQ_GEQ((TCP_GET_SEQ(p)+p->payload_len), ssn->client.last_ack)) { /*window base is needed !!*/ - if(SEQ_LT(TCP_GET_SEQ(p), (ssn->client.next_seq + ssn->client.window))) { + if(SEQ_EQ(TCP_GET_SEQ(p), ssn->client.next_seq)) { #ifdef DEBUG - printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); + printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); #endif - return 1; - } + return 1; } else { #ifdef DEBUG printf("Reset is not Valid! Packet SEQ: %" PRIu32 " and server SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p), ssn->client.next_seq); @@ -1576,13 +1692,11 @@ static int ValidReset(TcpSession *ssn, Packet *p) { return 0; } } else { /* implied to client */ - if(SEQ_GEQ((TCP_GET_SEQ(p) + p->payload_len), ssn->server.last_ack)) { /*window base is needed !!*/ - if(SEQ_LT(TCP_GET_SEQ(p), (ssn->server.next_seq + ssn->server.window))) { + if(SEQ_EQ(TCP_GET_SEQ(p), ssn->server.next_seq)) { #ifdef DEBUG - printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); + printf("Reset is Valid! Pakcet SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p)); #endif - return 1; - } + return 1; } else { #ifdef DEBUG printf("Reset is not Valid! Packet SEQ: %" PRIu32 " and client SEQ: %" PRIu32 "\n", TCP_GET_SEQ(p), ssn->server.next_seq); @@ -1591,7 +1705,6 @@ static int ValidReset(TcpSession *ssn, Packet *p) { } } break; - default: #ifdef DEBUG printf("Reset is not Valid! Packet SEQ: %" PRIu32 " & os_policy default case\n", TCP_GET_SEQ(p)); #endif @@ -1606,7 +1719,7 @@ int StreamTcpGetFlowState(void *s) { return FLOW_STATE_CLOSED; switch(ssn->state) { - case 0: + case TCP_NONE: case TCP_SYN_SENT: case TCP_SYN_RECV: case TCP_LISTEN: @@ -1642,7 +1755,7 @@ static int StreamTcpTest01 (void) { memset (&p, 0, sizeof(Packet)); memset (&f, 0, sizeof(Flow)); memset(&ssn1, 0, sizeof (TcpSession)); - f.stream = &ssn1; + f.protoctx = &ssn1; p.flow = &f; StreamTcpInitConfig(TRUE); @@ -1651,8 +1764,8 @@ static int StreamTcpTest01 (void) { printf("Session can not be allocated \n"); return 0; } - if (ssn->l7data != NULL) { - printf("Layer 7 field not set to NULL \n"); + if (ssn->aldata != NULL) { + printf("AppLayer field not set to NULL \n"); return 0; } if (ssn->state != 0) { @@ -1686,7 +1799,7 @@ static int StreamTcpTest02 (void) { memset(&tv, 0, sizeof (ThreadVars)); memset(&stt, 0, sizeof (StreamTcpThread)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.flow = &f; tcph.th_win = htons(5480); @@ -1745,7 +1858,7 @@ static int StreamTcpTest02 (void) { return 0; StreamTcpSessionPktFree(&p); - if (p.flow->stream != NULL) + if (p.flow->protoctx != NULL) return 0; return 1; } @@ -1771,7 +1884,7 @@ static int StreamTcpTest03 (void) { memset(&tv, 0, sizeof (ThreadVars)); memset(&stt, 0, sizeof (StreamTcpThread)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.flow = &f; tcph.th_win = htons(5480); @@ -1801,11 +1914,11 @@ static int StreamTcpTest03 (void) { if (stream_config.midstream != TRUE) return 1; - if (((TcpSession *)(p.flow->stream))->state != TCP_ESTABLISHED) + if (((TcpSession *)(p.flow->protoctx))->state != TCP_ESTABLISHED) return 0; - if (((TcpSession *)(p.flow->stream))->client.next_seq != 20 || - ((TcpSession *)(p.flow->stream))->server.next_seq != 11) + if (((TcpSession *)(p.flow->protoctx))->client.next_seq != 20 || + ((TcpSession *)(p.flow->protoctx))->server.next_seq != 11) return 0; StreamTcpSessionPktFree(&p); @@ -1833,7 +1946,7 @@ static int StreamTcpTest04 (void) { memset(&tv, 0, sizeof (ThreadVars)); memset(&stt, 0, sizeof (StreamTcpThread)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.flow = &f; tcph.th_win = htons(5480); @@ -1855,11 +1968,11 @@ static int StreamTcpTest04 (void) { if (stream_config.midstream != TRUE) return 1; - if (((TcpSession *)(p.flow->stream))->state != TCP_ESTABLISHED) + if (((TcpSession *)(p.flow->protoctx))->state != TCP_ESTABLISHED) return 0; - if (((TcpSession *)(p.flow->stream))->client.next_seq != 10 || - ((TcpSession *)(p.flow->stream))->server.next_seq != 20) + if (((TcpSession *)(p.flow->protoctx))->client.next_seq != 10 || + ((TcpSession *)(p.flow->protoctx))->server.next_seq != 20) return 0; StreamTcpSessionPktFree(&p); @@ -1888,7 +2001,7 @@ static int StreamTcpTest05 (void) { memset(&tv, 0, sizeof (ThreadVars)); memset(&stt, 0, sizeof (StreamTcpThread)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.flow = &f; /* prevent L7 from kicking in */ @@ -1948,11 +2061,11 @@ static int StreamTcpTest05 (void) { if (stream_config.midstream != TRUE) return 1; - if (((TcpSession *)(p.flow->stream))->state != TCP_ESTABLISHED) + if (((TcpSession *)(p.flow->protoctx))->state != TCP_ESTABLISHED) return 0; - if (((TcpSession *)(p.flow->stream))->client.next_seq != 16 || - ((TcpSession *)(p.flow->stream))->server.next_seq != 23) + if (((TcpSession *)(p.flow->protoctx))->client.next_seq != 16 || + ((TcpSession *)(p.flow->protoctx))->server.next_seq != 23) return 0; StreamTcpSessionPktFree(&p); return 1; @@ -1979,7 +2092,7 @@ static int StreamTcpTest06 (void) { memset(&tv, 0, sizeof (ThreadVars)); memset(&stt, 0, sizeof (StreamTcpThread)); memset(&tcph, 0, sizeof (TCPHdr)); - f.stream = &ssn; + f.protoctx = &ssn; p.flow = &f; tcph.th_flags = TH_FIN; p.tcph = &tcph; @@ -1987,14 +2100,14 @@ static int StreamTcpTest06 (void) { if (StreamTcpPacket(&tv, &p, &stt) == -1) return 0; - if (((TcpSession *)(p.flow->stream)) != NULL) + if (((TcpSession *)(p.flow->protoctx)) != NULL) return 0; p.tcph->th_flags = TH_RST; if (StreamTcpPacket(&tv, &p, &stt) == -1) return 0; - if (((TcpSession *)(p.flow->stream)) != NULL) + if (((TcpSession *)(p.flow->protoctx)) != NULL) return 0; return 1; } diff --git a/src/stream-tcp.h b/src/stream-tcp.h index aa99225c24..5aafbd2d36 100644 --- a/src/stream-tcp.h +++ b/src/stream-tcp.h @@ -16,6 +16,7 @@ typedef struct TcpStreamCnf_ { TcpStreamCnf stream_config; void TmModuleStreamTcpRegister (void); void StreamTcpInitConfig (char); +void StreamTcpFreeConfig(char); void StreamTcpRegisterTests (void); #endif /* __STREAM_TCP_H__ */ diff --git a/src/stream.c b/src/stream.c index be2887378f..ca2fe78631 100644 --- a/src/stream.c +++ b/src/stream.c @@ -8,6 +8,10 @@ #include "util-pool.h" +static pthread_mutex_t stream_pool_memuse_mutex; +static uint64_t stream_pool_memuse = 0; +static uint64_t stream_pool_memcnt = 0; + static StreamMsgQueue stream_q; /* per queue setting */ @@ -25,6 +29,11 @@ void *StreamMsgAlloc(void *null) { return NULL; memset(s, 0, sizeof(StreamMsg)); + + mutex_lock(&stream_pool_memuse_mutex); + stream_pool_memuse += sizeof(StreamMsg); + stream_pool_memcnt ++; + mutex_unlock(&stream_pool_memuse_mutex); return s; } @@ -140,6 +149,14 @@ void StreamMsgQueuesInit(void) { stream_msg_pool = PoolInit(5000,250,StreamMsgAlloc,NULL,StreamMsgFree); if (stream_msg_pool == NULL) exit(1); /* XXX */ + + pthread_mutex_init(&stream_pool_memuse_mutex, NULL); +} + +void StreamMsgQueuesDeinit(void) { + PoolFree(stream_msg_pool); + + printf("StreamMsgQueuesDeinit: stream_pool_memuse %"PRIu64", stream_pool_memcnt %"PRIu64"\n", stream_pool_memuse, stream_pool_memcnt); } StreamMsgQueue *StreamMsgQueueGetByPort(uint16_t port) { diff --git a/src/stream.h b/src/stream.h index 3dce3cae67..285b5f5031 100644 --- a/src/stream.h +++ b/src/stream.h @@ -49,6 +49,7 @@ typedef struct StreamMsgQueue_ { /* prototypes */ void StreamMsgQueuesInit(void); +void StreamMsgQueuesDeinit(void); StreamMsg *StreamMsgGetFromPool(void); void StreamMsgReturnToPool(StreamMsg *); diff --git a/src/tmqh-flow.c b/src/tmqh-flow.c index 12dad79717..3e5ddec3a1 100644 --- a/src/tmqh-flow.c +++ b/src/tmqh-flow.c @@ -27,6 +27,7 @@ typedef struct TmqhFlowCtx_ { uint16_t size; uint16_t *queues; + uint16_t last; } TmqhFlowCtx; Packet *TmqhInputFlow(ThreadVars *t); @@ -167,7 +168,12 @@ void TmqhOutputFlow(ThreadVars *tv, Packet *p) uint16_t idx = addr % ctx->size; qid = ctx->queues[idx]; } else { - qid = ctx->queues[0]; + ctx->last++; + + if (ctx->last == ctx->size) + ctx->last = 0; + + qid = ctx->queues[ctx->last]; } PacketQueue *q = &trans_q[qid]; diff --git a/src/util-pool.c b/src/util-pool.c index 406fc57287..4302761ab7 100644 --- a/src/util-pool.c +++ b/src/util-pool.c @@ -113,6 +113,11 @@ void *PoolGet(Pool *p) { } else { if (p->allocated < p->max_buckets) { p->allocated++; + + p->outstanding++; + if (p->outstanding > p->max_outstanding) + p->max_outstanding = p->outstanding; + return p->Alloc(p->AllocData); } else { return NULL; @@ -121,6 +126,9 @@ void *PoolGet(Pool *p) { void *ptr = pb->data; pb->data = NULL; + p->outstanding++; + if (p->outstanding > p->max_outstanding) + p->max_outstanding = p->outstanding; return ptr; } @@ -141,9 +149,14 @@ void PoolReturn(Pool *p, void *data) { p->alloc_list_size++; pb->data = data; + p->outstanding--; return; } +void PoolPrintSaturation(Pool *p) { + printf("PoolPrintSaturation: Pool %p is using %"PRIu32" out of %"PRIu32" items (%02.1f%%), max %"PRIu32" (%02.1f%%): pool struct memory %"PRIu64".\n", p, p->outstanding, p->max_buckets, (float)(p->outstanding/(float)(p->max_buckets))*100, p->max_outstanding, (float)(p->max_outstanding/(float)(p->max_buckets))*100, (uint64_t)(p->max_buckets * sizeof(PoolBucket))); +} + /* * ONLY TESTS BELOW THIS COMMENT */ diff --git a/src/util-pool.h b/src/util-pool.h index 0393c76144..45950ae7eb 100644 --- a/src/util-pool.h +++ b/src/util-pool.h @@ -23,12 +23,16 @@ typedef struct Pool_ { void *(*Alloc)(void *); void *AllocData; void (*Free)(void *); + + uint32_t outstanding; + uint32_t max_outstanding; } Pool; /* prototypes */ Pool* PoolInit(uint32_t, uint32_t, void *(*Alloc)(void *), void *, void (*Free)(void *)); void PoolFree(Pool *); void PoolPrint(Pool *); +void PoolPrintSaturation(Pool *p); void *PoolGet(Pool *); void PoolReturn(Pool *, void *); diff --git a/src/util-time.c b/src/util-time.c index 23b332aebb..46485ae9de 100644 --- a/src/util-time.c +++ b/src/util-time.c @@ -4,6 +4,8 @@ #include "detect.h" #include "threads.h" +//#define DEBUG + static struct timeval current_time = { 0,0 }; static pthread_mutex_t current_time_mutex = PTHREAD_MUTEX_INITIALIZER; static char live = TRUE; @@ -24,10 +26,13 @@ void TimeSet(struct timeval *tv) { return; mutex_lock(¤t_time_mutex); - current_time.tv_sec = tv->tv_sec; + current_time.tv_sec = tv->tv_sec; current_time.tv_usec = tv->tv_usec; - //printf("TimeSet: time set to %" PRIu64 " sec, %" PRIu64 " usec\n", - // current_time.tv_sec, current_time.tv_usec); + +#ifdef DEBUG + printf("TimeSet: time set to %" PRIuMAX " sec, %" PRIuMAX " usec\n", + (uintmax_t)current_time.tv_sec, (uintmax_t)current_time.tv_usec); +#endif mutex_unlock(¤t_time_mutex); } @@ -40,12 +45,14 @@ void TimeGet(struct timeval *tv) { gettimeofday(tv, NULL); } else { mutex_lock(¤t_time_mutex); - tv->tv_sec = current_time.tv_sec; + tv->tv_sec = current_time.tv_sec; tv->tv_usec = current_time.tv_usec; mutex_unlock(¤t_time_mutex); } - //printf("TimeGet: time we got is %" PRIu64 " sec, %" PRIu64 " usec\n", - // tv->tv_sec, tv->tv_usec); +#ifdef DEBUG + printf("TimeGet: time we got is %" PRIuMAX " sec, %" PRIuMAX " usec\n", + (uintmax_t)tv->tv_sec, (uintmax_t)tv->tv_usec); +#endif }