Improvements to content keyword memory handling.

First version of a simple pattern based L7 proto detection engine. Currently just works by matching a single pattern in the initial data. Implemented HTTP, SSL, MSN, JABBER, SMTP and a few more.

Couple of pattern matcher cleanups.

src/app-layer-detect-proto.c

@@ -1,5 +1,16 @@
 /* Copyright (c) 2009 Victor Julien */
 
+/** \file
+ *  \author Victor Julien <victor@inliniac.net>
+ *
+ *  A simple application layer (L7) protocol detector. It works by allowing
+ *  developers to set a series of patterns that if exactly matching indicate
+ *  that the session is a certain protocol.
+ *
+ *  \todo More advanced detection methods, regex maybe.
+ *  \todo Fall back to port based classification if other detection fails.
+ */
+
 #include "eidps-common.h"
 #include "debug.h"
 #include "decode.h"
@@ -8,8 +19,13 @@
 #include "threadvars.h"
 #include "tm-threads.h"
 
+#include "detect.h"
+#include "detect-engine.h"
+#include "detect-content.h"
+
 #include "util-print.h"
 #include "util-pool.h"
+#include "util-unittest.h"
 
 #include "stream-tcp-private.h"
 #include "stream-tcp-reassemble.h"
@@ -19,9 +35,35 @@
 #include "app-layer-parser.h"
 
 #define INSPECT_BYTES   32
-
+#define ALP_DETECT_MAX 256
+
+typedef struct AlpProtoDetectDirectionThread_ {
+    MpmThreadCtx mpm_ctx;
+    PatternMatcherQueue pmq;
+} AlpProtoDetectDirectionThread;
+
+typedef struct AlpProtoDetectDirection_ {
+    MpmCtx mpm_ctx;
+    uint32_t id;
+    /** a mapping between condition id's and protocol */
+    uint16_t map[ALP_DETECT_MAX];
+} AlpProtoDetectDirection;
+
+typedef struct AlpProtoDetectThreadCtx_ {
+    AlpProtoDetectDirectionThread toserver;
+    AlpProtoDetectDirectionThread toclient;
+} AlpProtoDetectThreadCtx;
+
+typedef struct AlpProtoDetectCtx_ {
+    AlpProtoDetectDirection toserver;
+    AlpProtoDetectDirection toclient;
+} AlpProtoDetectCtx;
+
+static AlpProtoDetectCtx alp_proto_ctx;
+static AlpProtoDetectThreadCtx alp_proto_tctx;
 static uint8_t al_proto_id = 0;
 
+/** \brief data stored in the stream */
 typedef struct AppLayerDetectProtoData_ {
     uint8_t proto;
 } AppLayerDetectProtoData;
@@ -39,6 +81,83 @@ void *AppLayerDetectProtoAlloc(void *null) {
 }
 #define AppLayerDetectProtoFree free
 
+void AlpProtoInit(AlpProtoDetectCtx *ctx) {
+    memset(ctx, 0x00, sizeof(AlpProtoDetectCtx));
+
+    MpmInitCtx(&ctx->toserver.mpm_ctx, MPM_B2G);
+    MpmInitCtx(&ctx->toclient.mpm_ctx, MPM_B2G);
+
+    memset(&ctx->toserver.map, 0x00, sizeof(ctx->toserver.map));
+    memset(&ctx->toclient.map, 0x00, sizeof(ctx->toclient.map));
+
+    ctx->toserver.id = 0;
+    ctx->toclient.id = 0;
+}
+
+void AlpProtoDestroy(AlpProtoDetectCtx *ctx) {
+    ctx->toserver.mpm_ctx.DestroyCtx(&ctx->toserver.mpm_ctx);
+    ctx->toclient.mpm_ctx.DestroyCtx(&ctx->toclient.mpm_ctx);
+}
+
+/** \brief Add a proto detection string to the detection ctx.
+ *  \param ctx The detection ctx
+ *  \param ip_proto The IP proto (TCP, UDP, etc)
+ *  \param al_proto Application layer proto
+ *  \param content A content string in the 'content:"some|20|string"' format.
+ *  \param depth Depth setting for the content. E.g. 4 means that the content has to match in the first 4 bytes of the stream.
+ *  \param offset Offset setting for the content. E.g. 4 mean that the content has to match after the first 4 bytes of the stream.
+ *  \param flags Set STREAM_TOCLIENT or STREAM_TOSERVER for the direction in which to try to match the content.
+ */
+void AlpProtoAdd(AlpProtoDetectCtx *ctx, uint16_t ip_proto, uint8_t al_proto, char *content, uint16_t depth, uint16_t offset, uint8_t flags) {
+    DetectContentData *cd = DetectContentParse(content);
+    if (cd == NULL) {
+        return;
+    }
+    cd->depth = depth;
+    cd->offset = offset;
+
+    PrintRawDataFp(stdout,cd->content,cd->content_len);
+
+    AlpProtoDetectDirection *dir;
+    if (flags & STREAM_TOCLIENT) {
+        dir = &ctx->toclient;
+    } else {
+        dir = &ctx->toserver;
+    }
+
+    dir->mpm_ctx.AddScanPattern(&dir->mpm_ctx, cd->content, cd->content_len,
+                                cd->offset, cd->depth, dir->id, dir->id, 0);
+    dir->map[dir->id] = al_proto;
+    dir->id++;
+
+    /* no longer need the cd */
+    DetectContentFree(cd);
+}
+
+void AlpProtoFinalizeThread(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx *tctx) {
+    uint32_t maxid;
+    memset(tctx, 0x00, sizeof(AlpProtoDetectThreadCtx));
+
+    if (ctx->toclient.id > 0) {
+        maxid = ctx->toclient.id;
+        ctx->toclient.mpm_ctx.InitThreadCtx(&ctx->toclient.mpm_ctx, &tctx->toclient.mpm_ctx, maxid);
+        PmqSetup(&tctx->toclient.pmq, maxid);
+    }
+    if (ctx->toserver.id > 0) {
+        maxid = ctx->toserver.id;
+        ctx->toserver.mpm_ctx.InitThreadCtx(&ctx->toserver.mpm_ctx, &tctx->toserver.mpm_ctx, maxid);
+        PmqSetup(&tctx->toserver.pmq, maxid);
+    }
+}
+
+void AlpProtoFinalizeGlobal(AlpProtoDetectCtx *ctx) {
+    if (ctx == NULL)
+        return;
+
+    ctx->toclient.mpm_ctx.Prepare(&ctx->toclient.mpm_ctx);
+    ctx->toserver.mpm_ctx.Prepare(&ctx->toserver.mpm_ctx);
+}
+
 void AppLayerDetectProtoThreadInit(void) {
     al_proto_id = StreamL7RegisterModule();
 
@@ -46,15 +165,133 @@ void AppLayerDetectProtoThreadInit(void) {
     if (al_detect_proto_pool == NULL) {
         exit(1);
     }
+
+    AlpProtoInit(&alp_proto_ctx);
+
+    /** \todo register these in the protocol parser api */
+
+    /** HTTP */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_HTTP, "GET", 3, 0, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_HTTP, "PUT", 3, 0, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_HTTP, "POST", 4, 0, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_HTTP, "TRACE", 5, 0, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_HTTP, "OPTIONS", 7, 0, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_HTTP, "HTTP", 4, 0, STREAM_TOCLIENT);
+
+    /** SSH */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSH, "SSH-", 4, 0, STREAM_TOCLIENT);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSH, "SSH-", 4, 0, STREAM_TOSERVER);
+
+    /** SSLv3 */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|01 03 00|", 5, 2, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|16 03 00|", 5, 2, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|16 03 00|", 3, 0, STREAM_TOCLIENT);
+
+    /** TLSv1 */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|01 03 01|", 3, 0, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|01 03 01|", 5, 2, STREAM_TOSERVER); /** midstream */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|01 03 01|", 5, 2, STREAM_TOCLIENT); /** midstream */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|16 03 01|", 3, 0, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|16 03 01|", 3, 0, STREAM_TOCLIENT);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|17 03 01|", 3, 0, STREAM_TOSERVER); /** midstream */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SSL, "|17 03 01|", 3, 0, STREAM_TOCLIENT); /** midstream */
+
+    /** IMAP */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_IMAP, "|2A 20|OK|20|", 5, 0, STREAM_TOCLIENT);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_IMAP, "1|20|capability", 12, 0, STREAM_TOSERVER);
+
+    /** SMTP */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SMTP, "EHLO ", 5, 0, STREAM_TOCLIENT);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SMTP, "HELO ", 5, 0, STREAM_TOCLIENT);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SMTP, "ESMTP ", 64, 4, STREAM_TOSERVER);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_SMTP, "SMTP ", 64, 4, STREAM_TOSERVER);
+
+    /** MSN Messenger */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_MSN, "MSNP", 10, 6, STREAM_TOCLIENT);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_MSN, "MSNP", 10, 6, STREAM_TOSERVER);
+
+    /** Jabber */
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_JABBER, "xmlns='jabber|3A|client'", 74, 53, STREAM_TOCLIENT);
+    AlpProtoAdd(&alp_proto_ctx, IPPROTO_TCP, ALPROTO_JABBER, "xmlns='jabber|3A|client'", 74, 53, STREAM_TOSERVER);
+
+    AlpProtoFinalizeGlobal(&alp_proto_ctx);
+    AlpProtoFinalizeThread(&alp_proto_ctx, &alp_proto_tctx);
 }
 
-uint8_t AppLayerDetectGetProto(uint8_t *buf, uint16_t buflen) {
-    if (buflen < INSPECT_BYTES)
+uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *ctx, AlpProtoDetectThreadCtx *tctx, uint8_t *buf, uint16_t buflen, uint8_t flags) {
+    //printf("AppLayerDetectGetProto: start\n");
+    //PrintRawDataFp(stdout, buf, buflen);
+
+    //if (buflen < INSPECT_BYTES)
+    //    return ALPROTO_UNKNOWN;
+
+    AlpProtoDetectDirection *dir;
+    AlpProtoDetectDirectionThread *tdir;
+    if (flags & STREAM_TOSERVER) {
+        dir = &ctx->toserver;
+        tdir = &tctx->toserver;
+    } else {
+        dir = &ctx->toclient;
+        tdir = &tctx->toclient;
+    }
+
+    if (dir->id == 0)
         return ALPROTO_UNKNOWN;
 
-    /* XXX do actual detect */
-    //printf("AppLayerDetectGetProto: protocol detection goes here.\n");
-    return ALPROTO_HTTP;
+    uint16_t proto;
+    uint32_t cnt = dir->mpm_ctx.Scan(&dir->mpm_ctx, &tdir->mpm_ctx, &tdir->pmq, buf, buflen);
+    //printf("AppLayerDetectGetProto: scan cnt %" PRIu32 "\n", cnt);
+    if (cnt == 0) {
+        proto = ALPROTO_UNKNOWN;
+        goto end;
+    }
+
+    /** We just return the first match
+     *  \todo what if we have more? */
+    proto = dir->map[tdir->pmq.sig_id_array[0]];
+
+end:
+    PmqReset(&tdir->pmq);
+
+    if (dir->mpm_ctx.Cleanup != NULL) {
+        dir->mpm_ctx.Cleanup(&tdir->mpm_ctx);
+    }
+
+#ifdef DEBUG
+    printf("AppLayerDetectGetProto: returning %" PRIu16 " (%s): ", proto, flags & STREAM_TOCLIENT ? "TOCLIENT" : "TOSERVER");
+    switch (proto) {
+        case ALPROTO_HTTP:
+            printf("HTTP\n");
+            break;
+        case ALPROTO_FTP:
+            printf("FTP\n");
+            break;
+        case ALPROTO_SSL:
+            printf("SSL\n");
+            break;
+        case ALPROTO_SSH:
+            printf("SSH\n");
+            break;
+        case ALPROTO_IMAP:
+            printf("IMAP\n");
+            break;
+        case ALPROTO_SMTP:
+            printf("SMTP\n");
+            break;
+        case ALPROTO_JABBER:
+            printf("JABBER\n");
+            break;
+        case ALPROTO_MSN:
+            printf("MSN\n");
+            break;
+        case ALPROTO_UNKNOWN:
+        default:
+            printf("UNKNOWN\n");
+            PrintRawDataFp(stdout,buf,buflen);
+            break;
+    }
+#endif
+    return proto;
 }
 
 void *AppLayerDetectProtoThread(void *td)
@@ -86,11 +323,13 @@ void *AppLayerDetectProtoThread(void *td)
                        or make it part of the stream setup */
                     StreamL7DataPtrInit(ssn,StreamL7GetStorageSize());
                 }
-                al_data_ptr = ssn->l7data[al_proto_id];
+                if (ssn->l7data != NULL) {
+                    al_data_ptr = ssn->l7data[al_proto_id];
+                }
             }
             mutex_unlock(&smsg->flow->m);
 
-            if (al_data_ptr != NULL) {
+            if (ssn != NULL && ssn->l7data != NULL) {
                 if (smsg->flags & STREAM_START) {
                     //printf("L7AppDetectThread: stream initializer (len %" PRIu32 " (%" PRIu32 "))\n", smsg->data.data_len, MSG_DATA_SIZE);
 
@@ -101,7 +340,7 @@ void *AppLayerDetectProtoThread(void *td)
                     if (al_data_ptr == NULL) {
                         al_proto = (AppLayerDetectProtoData *)PoolGet(al_detect_proto_pool);
                         if (al_proto != NULL) {
-                            al_proto->proto = AppLayerDetectGetProto(smsg->data.data, smsg->data.data_len);
+                            al_proto->proto = AppLayerDetectGetProto(&alp_proto_ctx, &alp_proto_tctx, smsg->data.data, smsg->data.data_len, smsg->flags);
                             store = 1;
 
                             AppLayerParse(smsg->flow, al_proto->proto, smsg->flags, smsg->data.data, smsg->data.data_len);
@@ -118,11 +357,11 @@ void *AppLayerDetectProtoThread(void *td)
                      * a start msg should have gotten us one */
                     if (al_data_ptr != NULL) {
                         al_proto = (AppLayerDetectProtoData *)al_data_ptr;
-//                        printf("AppLayerDetectThread: already established that the proto is %" PRIu32 "\n", al_proto->proto);
+                        //printf("AppLayerDetectThread: already established that the proto is %" PRIu32 "\n", al_proto->proto);
 
                         AppLayerParse(smsg->flow, al_proto->proto, smsg->flags, smsg->data.data, smsg->data.data_len);
                     } else {
-//                        printf("AppLayerDetectThread: smsg not start, but no l7 data? Weird\n");
+                        //printf("AppLayerDetectThread: smsg not start, but no l7 data? Weird\n");
                     }
                 }
             }
@@ -166,3 +405,278 @@ void AppLayerDetectProtoThreadSpawn()
     return;
 }
 
+#ifdef UNITTESTS
+
+int AlpDetectTest01(void) {
+    char *buf = strdup("HTTP");
+    int r = 1;
+    AlpProtoDetectCtx ctx;
+
+    AlpProtoInit(&ctx);
+
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 1) {
+        r = 0;
+    }
+
+    buf = strdup("GET");
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, buf, 4, 0, STREAM_TOSERVER);
+    if (ctx.toserver.id != 1) {
+        r = 0;
+    }
+    free(buf);
+
+    AlpProtoDestroy(&ctx);
+    return r;
+}
+
+int AlpDetectTest02(void) {
+    char *buf = strdup("HTTP");
+    int r = 1;
+    AlpProtoDetectCtx ctx;
+
+    AlpProtoInit(&ctx);
+
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 1) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_HTTP) {
+        r = 0;
+    }
+
+    buf = strdup("220 ");
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_FTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 2) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_FTP) {
+        r = 0;
+    }
+
+    AlpProtoDestroy(&ctx);
+    return r;
+}
+
+int AlpDetectTest03(void) {
+    uint8_t l7data[] = "HTTP/1.1 200 OK\r\nServer: Apache/1.0\r\n\r\n";
+    char *buf = strdup("HTTP");
+    int r = 1;
+    AlpProtoDetectCtx ctx;
+    AlpProtoDetectThreadCtx tctx;
+
+    AlpProtoInit(&ctx);
+
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 1) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_HTTP) {
+        r = 0;
+    }
+
+    buf = strdup("220 ");
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_FTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 2) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_FTP) {
+        r = 0;
+    }
+
+    AlpProtoFinalizeGlobal(&ctx);
+    AlpProtoFinalizeThread(&ctx, &tctx);
+
+    uint32_t cnt = ctx.toclient.mpm_ctx.Scan(&ctx.toclient.mpm_ctx, &tctx.toclient.mpm_ctx, NULL, l7data, sizeof(l7data));
+    if (cnt != 1) {
+        printf("cnt %u != 1: ", cnt);
+        r = 0;
+    }
+
+    AlpProtoDestroy(&ctx);
+    return r;
+}
+
+int AlpDetectTest04(void) {
+    uint8_t l7data[] = "HTTP/1.1 200 OK\r\nServer: Apache/1.0\r\n\r\n";
+    char *buf = strdup("200 ");
+    int r = 1;
+    AlpProtoDetectCtx ctx;
+    AlpProtoDetectThreadCtx tctx;
+
+    AlpProtoInit(&ctx);
+
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 1) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_HTTP) {
+        r = 0;
+    }
+
+    AlpProtoFinalizeGlobal(&ctx);
+    AlpProtoFinalizeThread(&ctx, &tctx);
+
+    uint32_t cnt = ctx.toclient.mpm_ctx.Scan(&ctx.toclient.mpm_ctx, &tctx.toclient.mpm_ctx, NULL, l7data, sizeof(l7data));
+    if (cnt != 0) {
+        printf("cnt %u != 0: ", cnt);
+        r = 0;
+    }
+
+    AlpProtoDestroy(&ctx);
+    return r;
+}
+
+int AlpDetectTest05(void) {
+    uint8_t l7data[] = "HTTP/1.1 200 OK\r\nServer: Apache/1.0\r\n\r\n<HTML><BODY>Blahblah</BODY></HTML>";
+    char *buf = strdup("HTTP");
+    int r = 1;
+
+    AlpProtoDetectCtx ctx;
+    AlpProtoDetectThreadCtx tctx;
+
+    AlpProtoInit(&ctx);
+
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 1) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_HTTP) {
+        r = 0;
+    }
+
+    buf = strdup("220 ");
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_FTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 2) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_FTP) {
+        r = 0;
+    }
+
+    AlpProtoFinalizeGlobal(&ctx);
+    AlpProtoFinalizeThread(&ctx, &tctx);
+
+    uint8_t proto = AppLayerDetectGetProto(&ctx, &tctx, l7data,sizeof(l7data), STREAM_TOCLIENT);
+    if (proto != ALPROTO_HTTP) {
+        printf("proto %" PRIu8 " != %" PRIu8 ": ", proto, ALPROTO_HTTP);
+        r = 0;
+    }
+
+    AlpProtoDestroy(&ctx);
+    return r;
+}
+
+int AlpDetectTest06(void) {
+    uint8_t l7data[] = "220 Welcome to the OISF FTP server\r\n";
+    char *buf = strdup("HTTP");
+    int r = 1;
+    AlpProtoDetectCtx ctx;
+    AlpProtoDetectThreadCtx tctx;
+
+    AlpProtoInit(&ctx);
+
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 1) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_HTTP) {
+        r = 0;
+    }
+
+    buf = strdup("220 ");
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_FTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 2) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_FTP) {
+        r = 0;
+    }
+
+    AlpProtoFinalizeGlobal(&ctx);
+    AlpProtoFinalizeThread(&ctx, &tctx);
+
+    uint8_t proto = AppLayerDetectGetProto(&ctx, &tctx, l7data,sizeof(l7data), STREAM_TOCLIENT);
+    if (proto != ALPROTO_FTP) {
+        printf("proto %" PRIu8 " != %" PRIu8 ": ", proto, ALPROTO_FTP);
+        r = 0;
+    }
+
+    AlpProtoDestroy(&ctx);
+    return r;
+}
+
+int AlpDetectTest07(void) {
+    uint8_t l7data[] = "220 Welcome to the OISF HTTP/FTP server\r\n";
+    char *buf = strdup("HTTP");
+    int r = 1;
+    AlpProtoDetectCtx ctx;
+    AlpProtoDetectThreadCtx tctx;
+
+    AlpProtoInit(&ctx);
+
+    AlpProtoAdd(&ctx, IPPROTO_TCP, ALPROTO_HTTP, buf, 4, 0, STREAM_TOCLIENT);
+    free(buf);
+
+    if (ctx.toclient.id != 1) {
+        r = 0;
+    }
+
+    if (ctx.toclient.map[ctx.toclient.id - 1] != ALPROTO_HTTP) {
+        r = 0;
+    }
+
+    AlpProtoFinalizeGlobal(&ctx);
+    AlpProtoFinalizeThread(&ctx, &tctx);
+
+    uint8_t proto = AppLayerDetectGetProto(&ctx, &tctx, l7data,sizeof(l7data), STREAM_TOCLIENT);
+    if (proto != ALPROTO_UNKNOWN) {
+        printf("proto %" PRIu8 " != %" PRIu8 ": ", proto, ALPROTO_UNKNOWN);
+        r = 0;
+    }
+
+    AlpProtoDestroy(&ctx);
+    return r;
+}
+
+void AlpDetectRegisterTests(void) {
+    UtRegisterTest("AlpDetectTest01", AlpDetectTest01, 1);
+    UtRegisterTest("AlpDetectTest02", AlpDetectTest02, 1);
+    UtRegisterTest("AlpDetectTest03", AlpDetectTest03, 1);
+    UtRegisterTest("AlpDetectTest04", AlpDetectTest04, 1);
+    UtRegisterTest("AlpDetectTest05", AlpDetectTest05, 1);
+    UtRegisterTest("AlpDetectTest06", AlpDetectTest06, 1);
+    UtRegisterTest("AlpDetectTest07", AlpDetectTest07, 1);
+}
+
+#endif /* UNITTESTS */
+

src/app-layer-detect-proto.h

@@ -6,6 +6,7 @@ void *AppLayerDetectProtoThread(void *td);
 void AppLayerDetectProtoThreadInit(void);
 
 void AppLayerDetectProtoThreadSpawn(void);
+void AlpDetectRegisterTests(void);
 
 #endif /* __APP_LAYER_DETECT_PROTO_H__ */
 

src/app-layer-parser.c

@@ -484,7 +484,7 @@ int AppLayerParse(Flow *f, uint8_t proto, uint8_t flags, uint8_t *input, uint32_
     }
 
     if (parser_idx == 0) {
-        printf("AppLayerParse: no parser for protocol %" PRIu32 "\n", proto);
+        //printf("AppLayerParse: no parser for protocol %" PRIu32 "\n", proto);
         return 0;
     }
 

src/app-layer-protos.h

@@ -6,6 +6,11 @@ enum {
     ALPROTO_HTTP,
     ALPROTO_FTP,
     ALPROTO_SMTP,
+    ALPROTO_SSL,
+    ALPROTO_SSH,
+    ALPROTO_IMAP,
+    ALPROTO_MSN,
+    ALPROTO_JABBER,
 
     /* keep last */
     ALPROTO_MAX,

src/detect-content.c

@@ -48,7 +48,6 @@
 int DetectContentMatch (ThreadVars *, PatternMatcherThread *, Packet *, Signature *, SigMatch *);
 int DetectContentSetup (DetectEngineCtx *, Signature *, SigMatch *, char *);
 void DetectContentRegisterTests(void);
-void DetectContentFree(void *);
 
 uint8_t nocasetable[256];
 #define _nc(c) nocasetable[(c)]
@@ -282,14 +281,17 @@ int DetectContentMatch (ThreadVars *t, PatternMatcherThread *pmt, Packet *p, Sig
 DetectContentData *DetectContentParse (char *contentstr)
 {
     DetectContentData *cd = NULL;
-    char *str = contentstr;
-    char dubbed = 0;
+    char *str;
     uint16_t len;
 
+    if (strlen(contentstr) == 0)
+        return NULL;
+
     if (contentstr[0] == '\"' && contentstr[strlen(contentstr)-1] == '\"') {
         str = strdup(contentstr+1);
         str[strlen(contentstr)-2] = '\0';
-        dubbed = 1;
+    } else {
+        str = strdup(contentstr);
     }
 
     len = strlen(str);
@@ -396,12 +398,16 @@ DetectContentData *DetectContentParse (char *contentstr)
     cd->distance = 0;
     cd->flags = 0;
 
-    if (dubbed != 0) free(str);
+    free(str);
     return cd;
 
 error:
-    if (dubbed != 0) free(str);
-    if (cd != NULL) free(cd);
+    free(str);
+    if (cd != NULL) {
+        if (cd->content != NULL)
+            free(cd->content);
+        free(cd);
+    }
     return NULL;
 }
 
@@ -598,6 +604,40 @@ int DetectContentParseTest06 (void) {
     return result;
 }
 
+/**
+ * \test DetectCotentParseTest07 test an empty content
+ */
+int DetectContentParseTest07 (void) {
+    int result = 1;
+    DetectContentData *cd = NULL;
+    char *teststring = "\"\"";
+
+    cd = DetectContentParse(teststring);
+    if (cd != NULL) {
+        printf("expected NULL got %p: ", cd);
+        result = 0;
+        DetectContentFree(cd);
+    }
+    return result;
+}
+
+/**
+ * \test DetectCotentParseTest08 test an empty content
+ */
+int DetectContentParseTest08 (void) {
+    int result = 1;
+    DetectContentData *cd = NULL;
+    char *teststring = "";
+
+    cd = DetectContentParse(teststring);
+    if (cd != NULL) {
+        printf("expected NULL got %p: ", cd);
+        result = 0;
+        DetectContentFree(cd);
+    }
+    return result;
+}
+
 
 /**
  * \brief this function registers unit tests for DetectFlow
@@ -609,5 +649,7 @@ void DetectContentRegisterTests(void) {
     UtRegisterTest("DetectContentParseTest04", DetectContentParseTest04, 1);
     UtRegisterTest("DetectContentParseTest05", DetectContentParseTest05, 1);
     UtRegisterTest("DetectContentParseTest06", DetectContentParseTest06, 1);
+    UtRegisterTest("DetectContentParseTest07", DetectContentParseTest07, 1);
+    UtRegisterTest("DetectContentParseTest08", DetectContentParseTest08, 1);
 }
 

src/detect-content.h

@@ -25,6 +25,8 @@ typedef struct DetectContentData_ {
 /* prototypes */
 void DetectContentRegister (void);
 uint32_t DetectContentMaxId(DetectEngineCtx *);
+DetectContentData *DetectContentParse (char *contentstr);
+void DetectContentFree(DetectContentData *);
 
 #endif /* __DETECT_CONTENT_H__ */
 

src/detect-engine-mpm.c

@@ -46,11 +46,7 @@ uint32_t PacketPatternMatch(ThreadVars *t, PatternMatcherThread *pmt, Packet *p)
 
 /* cleans up the mpm instance after a match */
 void PacketPatternCleanup(ThreadVars *t, PatternMatcherThread *pmt) {
-    int i;
-    for (i = 0; i < pmt->pmq.sig_id_array_cnt; i++) {
-        pmt->pmq.sig_bitarray[(pmt->pmq.sig_id_array[i] / 8)] &= ~(1<<(pmt->pmq.sig_id_array[i] % 8));
-    }
-    pmt->pmq.sig_id_array_cnt = 0;
+    PmqReset(&pmt->pmq);
 
     if (pmt->sgh == NULL)
         return;
@@ -818,23 +814,8 @@ int PatternMatcherThreadInit(ThreadVars *t, void *initdata, void **data) {
      */
     mpm_ctx[0].InitThreadCtx(&mpm_ctx[0], &pmt->mtc, DetectContentMaxId(de_ctx));
     mpm_ctx[0].InitThreadCtx(&mpm_ctx[0], &pmt->mtcu, DetectUricontentMaxId(de_ctx));
-    uint32_t max_sig_id = DetectEngineGetMaxSigId(de_ctx);
 
-    /* sig callback testing stuff below */
-    pmt->pmq.sig_id_array = malloc(max_sig_id * sizeof(uint32_t));
-    if (pmt->pmq.sig_id_array == NULL) {
-        printf("ERROR: could not setup memory for pattern matcher: %s\n", strerror(errno));
-        exit(1);
-    }
-    memset(pmt->pmq.sig_id_array, 0, max_sig_id * sizeof(uint32_t));
-    pmt->pmq.sig_id_array_cnt = 0;
-    /* lookup bitarray */
-    pmt->pmq.sig_bitarray = malloc(max_sig_id / 8 + 1);
-    if (pmt->pmq.sig_bitarray == NULL) {
-        printf("ERROR: could not setup memory for pattern matcher: %s\n", strerror(errno));
-        exit(1);
-    }
-    memset(pmt->pmq.sig_bitarray, 0, max_sig_id / 8 + 1);
+    PmqSetup(&pmt->pmq, DetectEngineGetMaxSigId(de_ctx));
 
     /* IP-ONLY */
     DetectEngineIPOnlyThreadInit(de_ctx,&pmt->io_ctx);

src/eidps.c

@@ -959,6 +959,7 @@ int main(int argc, char **argv)
         HTTPParserRegisterTests();
         DecodePPPoERegisterTests();
         DecodeICMPV4RegisterTests();
+        AlpDetectRegisterTests();
         UtRunTests();
         UtCleanup();
         exit(0);

src/stream-tcp-reassemble.c

@@ -797,6 +797,8 @@ static void StreamTcpSetupInitMsg(Packet *p, StreamMsg *smsg) {
 }
 
 static void StreamTcpSetupMsg(Packet *p, StreamMsg *smsg) {
+    smsg->flags = 0;
+
     if (p->flowflags & FLOW_PKT_TOSERVER) {
         COPY_ADDRESS(&p->flow->src,&smsg->data.src_ip);
         COPY_ADDRESS(&p->flow->dst,&smsg->data.dst_ip);

src/util-mpm.c

@@ -13,6 +13,79 @@
 #include "util-mpm-b2g.h"
 #include "util-mpm-b3g.h"
 
+/** \brief Setup a pmq
+  * \param pmq Pattern matcher queue to be initialized
+  * \param maxid Max id to be matched on
+  * \retval -1 error
+  * \retval 0 ok
+  */
+int PmqSetup(PatternMatcherQueue *pmq, uint32_t maxid) {
+    if (pmq == NULL)
+        return -1;
+
+    memset(pmq, 0, sizeof(PatternMatcherQueue));
+
+    pmq->sig_id_array = malloc(maxid * sizeof(uint32_t));
+    if (pmq->sig_id_array == NULL) {
+        printf("ERROR: could not setup memory for pattern matcher: %s\n", strerror(errno));
+        return -1;
+    }
+    memset(pmq->sig_id_array, 0, maxid * sizeof(uint32_t));
+    pmq->sig_id_array_cnt = 0;
+
+    /* lookup bitarray */
+    pmq->sig_bitarray = malloc(maxid / 8 + 1);
+    if (pmq->sig_bitarray == NULL) {
+        printf("ERROR: could not setup memory for pattern matcher: %s\n", strerror(errno));
+        return -1;
+    }
+    memset(pmq->sig_bitarray, 0, maxid / 8 + 1);
+
+    return 0;
+}
+
+/** \brief Reset a Pmq for reusage. Meant to be called after a single search.
+ *  \param pmq Pattern matcher to be reset.
+ */
+void PmqReset(PatternMatcherQueue *pmq) {
+    int i;
+    for (i = 0; i < pmq->sig_id_array_cnt; i++) {
+        pmq->sig_bitarray[(pmq->sig_id_array[i] / 8)] &= ~(1<<(pmq->sig_id_array[i] % 8));
+    }
+    pmq->sig_id_array_cnt = 0;
+}
+
+/** \brief Cleanup a Pmq
+  * \param pmq Pattern matcher queue to be cleaned up.
+  */
+void PmqCleanup(PatternMatcherQueue *pmq) {
+    if (pmq == NULL)
+        return;
+
+    if (pmq->sig_id_array != NULL) {
+        free(pmq->sig_id_array);
+        pmq->sig_id_array = NULL;
+    }
+
+    if (pmq->sig_bitarray != NULL) {
+        free(pmq->sig_bitarray);
+        pmq->sig_bitarray = NULL;
+    }
+
+    pmq->sig_id_array_cnt = 0;
+}
+
+/** \brief Cleanup and free a Pmq
+  * \param pmq Pattern matcher queue to be free'd.
+  */
+void PmqFree(PatternMatcherQueue *pmq) {
+    if (pmq == NULL)
+        return;
+
+    PmqCleanup(pmq);
+    free(pmq);
+}
+
 /* cleanup list with all matches
  *
  * used at search runtime (or actually once per search) */

src/util-mpm.h

@@ -53,22 +53,18 @@ typedef struct MpmThreadCtx_ {
     MpmMatch *qlist;
     /* spare list */
     MpmMatch *sparelist;
-
-    uint32_t matches;
-
 } MpmThreadCtx;
 
 #define PMQ_MODE_SCAN   0
 #define PMQ_MODE_SEARCH 1
 
-/* helper structure for the detection engine. The Pattern Matcher thread
- * has this and passes a pointer to it to the pattern matcher. The actual
- * pattern matcher will fill the structure. */
+/** \brief helper structure for the pattern matcher engine. The Pattern Matcher
+ *         thread has this and passes a pointer to it to the pattern matcher.
+ *         The actual pattern matcher will fill the structure. */
 typedef struct PatternMatcherQueue_ {
-    /* sig callback stuff XXX consider a separate struct for this*/
     uint32_t *sig_id_array; /* array with internal sig id's that had a
-                                pattern match. These will be inspected
-                                futher by the detection engine. */
+                               pattern match. These will be inspected
+                               futher by the detection engine. */
     uint32_t sig_id_array_cnt;
     uint8_t *sig_bitarray;
     char mode; /* 0: scan, 1: search */
@@ -134,6 +130,12 @@ typedef struct MpmTableElmt_ {
     uint8_t flags;
 } MpmTableElmt;
 
+
+int PmqSetup(PatternMatcherQueue *, uint32_t);
+void PmqReset(PatternMatcherQueue *);
+void PmqCleanup(PatternMatcherQueue *);
+void PmqFree(PatternMatcherQueue *);
+
 void MpmMatchCleanup(MpmThreadCtx *);
 MpmMatch *MpmMatchAlloc(MpmThreadCtx *);
 int MpmMatchAppend(MpmThreadCtx *, PatternMatcherQueue *, MpmEndMatch *, MpmMatchBucket *, uint16_t, uint16_t);