Fix app layer detect to actually work.

src/Makefile.am

@@ -132,7 +132,7 @@ app-layer-http.c app-layer-http.h \
 app-layer-tls.c app-layer-tls.h \
 app-layer-smb.c app-layer-smb.h \
 app-layer-dcerpc.c app-layer-dcerpc.h \
-app-layer-tls-detect-version.c app-layer-tls-detect-version.h \
+detect-tls-version.c detect-tls-version.h \
 app-layer-protos.h \
 conf.c conf.h \
 conf-yaml-loader.c conf-yaml-loader.h \

src/app-layer-tls-detect-version.h

@@ -1,12 +0,0 @@
-#ifndef __APP_LAYER_TLS_DETECT_VERSION_H__
-#define __APP_LAYER_TLS_DETECT_VERSION_H__
-
-typedef struct AppLayerTlsDetectVersionData_ {
-    uint8_t ver; /** tls version to match */
-} AppLayerTlsDetectVersionData;
-
-/* prototypes */
-void AppLayerTlsDetectVersionRegister (void);
-
-#endif /* __APP_LAYER_TLS_DETECT_VERSION_H__ */
-

src/app-layer.c

@@ -1,30 +1,47 @@
 #include "eidps-common.h"
 #include "app-layer.h"
 #include "stream-tcp-private.h"
+#include "util-debug.h"
 
-/** \brief Get the active app layer state from the packet */
+/** \brief Get the active app layer state from the packet
+ *  \param p packet pointer
+ *  \retval alstate void pointer to the state
+ *  \retval NULL in case we have no state */
 void *AppLayerGetProtoStateFromPacket(Packet *p) {
-    if (p == NULL || p->flow == NULL)
-        return NULL;
+    SCEnter();
+
+    if (p == NULL || p->flow == NULL) {
+        SCReturnPtr(NULL, "void");
+    }
 
     TcpSession *ssn = (TcpSession *)p->flow->protoctx;
-    if (ssn == NULL || ssn->aldata == NULL)
-        return NULL;
+    if (ssn == NULL || ssn->aldata == NULL) {
+        SCReturnPtr(NULL, "void");
+    }
+
+    SCLogDebug("ssn->alproto %u", ssn->alproto);
 
-    void *alstate = ssn->aldata[ssn->alproto];
-    return alstate;
+    void *alstate = ssn->aldata[AlpGetStateIdx(ssn->alproto)];
+    SCReturnPtr(alstate, "void");
 }
 
-/** \brief Get the active app layer state from the flow */
+/** \brief Get the active app layer state from the flow
+ *  \param f flow pointer
+ *  \retval alstate void pointer to the state
+ *  \retval NULL in case we have no state */
 void *AppLayerGetProtoStateFromFlow(Flow *f) {
+    SCEnter();
+
     if (f == NULL)
-        return NULL;
+        SCReturnPtr(NULL, "void");
 
     TcpSession *ssn = (TcpSession *)f->protoctx;
     if (ssn == NULL || ssn->aldata == NULL)
-        return NULL;
+        SCReturnPtr(NULL, "void");
+
+    SCLogDebug("ssn->alproto %u", ssn->alproto);
 
-    void *alstate = ssn->aldata[ssn->alproto];
-    return alstate;
+    void *alstate = ssn->aldata[AlpGetStateIdx(ssn->alproto)];
+    SCReturnPtr(alstate, "void");
 }
 

src/detect-engine-port.c

@@ -1,6 +1,6 @@
 /** Copyright (c) 2009 Open Information Security Foundation.
  *
- * \Author Victor Julien Copyright (c) 2008
+ * \author Victor Julien Copyright (c) 2008
  * Ports part of the detection engine.
  *
  * TODO VJ
@@ -897,6 +897,7 @@ static int DetectPortParseInsert(DetectPort **head, DetectPort *new) {
 static int DetectPortParseInsertString(DetectPort **head, char *s) {
     DetectPort *ad = NULL, *ad_any = NULL;
     int r = 0;
+    char port_any = FALSE;
 
     SCLogDebug("head %p, *head %p, s %s", head, *head, s);
 
@@ -907,6 +908,10 @@ static int DetectPortParseInsertString(DetectPort **head, char *s) {
         goto error;
     }
 
+    if (ad->flags & PORT_FLAG_ANY) {
+        port_any = TRUE;
+    }
+
     /** handle the not case, we apply the negation then insert the part(s) */
     if (ad->flags & PORT_FLAG_NOT) {
         DetectPort *ad2 = NULL;
@@ -931,7 +936,9 @@ static int DetectPortParseInsertString(DetectPort **head, char *s) {
         goto error;
 
     /** if any, insert 0.0.0.0/0 and ::/0 as well */
-    if (r == 1 && ad->flags & PORT_FLAG_ANY) {
+    if (r == 1 && port_any == TRUE) {
+        SCLogDebug("inserting 0:65535 as port is \"any\"");
+
         ad_any = PortParse("0:65535");
         if (ad_any == NULL)
             goto error;

src/detect-engine-proto.c

@@ -123,6 +123,21 @@ error:
     return -1;
 }
 
+/** \brief see if a DetectProto contains a certain proto
+ *  \param dp detect proto to inspect
+ *  \param proto protocol (such as IPPROTO_TCP) to look for
+ *  \retval 0 protocol not in the set
+ *  \retval 1 protocol is in the set */
+int DetectProtoContainsProto(DetectProto *dp, int proto) {
+    if (dp->flags & DETECT_PROTO_ANY)
+        return 1;
+
+    if (dp->proto[proto / 8] & (1<<(proto % 8)))
+        return 1;
+
+    return 0;
+}
+
 /* TESTS */
 
 #ifdef UNITTESTS
@@ -180,11 +195,11 @@ static int ProtoTestParse01 (void)
     memset(&dp,0,sizeof(DetectProto));
 
     int r = DetectProtoParse(&dp, "6");
-    if (r == 0) {
+    if (r != 0) {
         return 1;
     }
 
-    SCLogDebug("ProtoTestParse01: Error in parsing the \"6\" string");
+    SCLogDebug("DetectProtoParse should have rejected the \"6\" string");
     return 0;
 }
 /**

src/detect-engine-proto.h

@@ -12,6 +12,7 @@ typedef struct DetectProto_ {
 /* prototypes */
 void DetectProtoRegister (void);
 int DetectProtoParse(DetectProto *dp, char *str);
+int DetectProtoContainsProto(DetectProto *, int);
 
 #endif /* __DETECT_PROTO_H__ */
 

src/detect-parse.c

@@ -317,6 +317,9 @@ int SigParseProto(Signature *s, const char *protostr) {
         if (s->alproto != ALPROTO_UNKNOWN) {
             /* indicate that the signature is app-layer */
             s->flags |= SIG_FLAG_APPLAYER;
+
+            /* app layer is always TCP for now */
+            s->proto.proto[IPPROTO_TCP / 8] |= 1 << (IPPROTO_TCP % 8);
             return 0;
         }
 
@@ -437,9 +440,8 @@ int SigParseBasics(Signature *s, char *sigstr, char ***result) {
 
     /* For "ip" we parse the ports as well, even though they will be just "any".
      *  We do this for later sgh building for the tcp and udp protocols. */
-    if (strcasecmp(arr[CONFIG_PROTO],"tcp") == 0 ||
-        strcasecmp(arr[CONFIG_PROTO],"udp") == 0 ||
-        strcasecmp(arr[CONFIG_PROTO],"ip") == 0) {
+    if (DetectProtoContainsProto(&s->proto, IPPROTO_TCP) ||
+        DetectProtoContainsProto(&s->proto, IPPROTO_UDP)) {
         if (SigParsePort(s, arr[CONFIG_SP], 0) < 0)
             goto error;
         if (SigParsePort(s, arr[CONFIG_DP], 1) < 0)

src/detect-tls-version.c

@@ -24,7 +24,7 @@
 #include "app-layer.h"
 
 #include "app-layer-tls.h"
-#include "app-layer-tls-detect-version.h"
+#include "detect-tls-version.h"
 
 
 /**
@@ -35,38 +35,38 @@
 static pcre *parse_regex;
 static pcre_extra *parse_regex_study;
 
-int AppLayerTlsDetectVersionMatch (ThreadVars *, DetectEngineThreadCtx *, Flow *, uint8_t, void *, Signature *, SigMatch *);
-int AppLayerTlsDetectVersionSetup (DetectEngineCtx *, Signature *, SigMatch *, char *);
-void AppLayerTlsDetectVersionRegisterTests(void);
-void AppLayerTlsDetectVersionFree(void *);
+int DetectTlsVersionMatch (ThreadVars *, DetectEngineThreadCtx *, Flow *, uint8_t, void *, Signature *, SigMatch *);
+int DetectTlsVersionSetup (DetectEngineCtx *, Signature *, SigMatch *, char *);
+void DetectTlsVersionRegisterTests(void);
+void DetectTlsVersionFree(void *);
 
 /**
  * \brief Registration function for keyword: tls.version
  */
-void AppLayerTlsDetectVersionRegister (void) {
+void DetectTlsVersionRegister (void) {
     sigmatch_table[DETECT_AL_TLS_VERSION].name = "tls.version";
     sigmatch_table[DETECT_AL_TLS_VERSION].Match = NULL;
-    sigmatch_table[DETECT_AL_TLS_VERSION].AppLayerMatch = AppLayerTlsDetectVersionMatch;
-    sigmatch_table[DETECT_AL_TLS_VERSION].Setup = AppLayerTlsDetectVersionSetup;
-    sigmatch_table[DETECT_AL_TLS_VERSION].Free  = AppLayerTlsDetectVersionFree;
-    sigmatch_table[DETECT_AL_TLS_VERSION].RegisterTests = AppLayerTlsDetectVersionRegisterTests;
+    sigmatch_table[DETECT_AL_TLS_VERSION].AppLayerMatch = DetectTlsVersionMatch;
+    sigmatch_table[DETECT_AL_TLS_VERSION].Setup = DetectTlsVersionSetup;
+    sigmatch_table[DETECT_AL_TLS_VERSION].Free  = DetectTlsVersionFree;
+    sigmatch_table[DETECT_AL_TLS_VERSION].RegisterTests = DetectTlsVersionRegisterTests;
 
     const char *eb;
     int eo;
     int opts = 0;
 
-	SCLogDebug("registering tls.version rule option\n");
+	SCLogDebug("registering tls.version rule option");
 
     parse_regex = pcre_compile(PARSE_REGEX, opts, &eb, &eo, NULL);
     if (parse_regex == NULL) {
-        SCLogDebug("Compile of \"%s\" failed at offset %" PRId32 ": %s\n",
+        SCLogDebug("Compile of \"%s\" failed at offset %" PRId32 ": %s",
                     PARSE_REGEX, eo, eb);
         goto error;
     }
 
     parse_regex_study = pcre_study(parse_regex, 0, &eb);
     if (eb != NULL) {
-        SCLogDebug("pcre study failed: %s\n", eb);
+        SCLogDebug("pcre study failed: %s", eb);
         goto error;
     }
     return;
@@ -81,29 +81,38 @@ error:
  * \param t pointer to thread vars
  * \param det_ctx pointer to the pattern matcher thread
  * \param p pointer to the current packet
- * \param m pointer to the sigmatch that we will cast into AppLayerTlsDetectVersionData
+ * \param m pointer to the sigmatch that we will cast into DetectTlsVersionData
  *
  * \retval 0 no match
  * \retval 1 match
  */
-int AppLayerTlsDetectVersionMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, uint8_t flags, void *state, Signature *s, SigMatch *m)
+int DetectTlsVersionMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, uint8_t flags, void *state, Signature *s, SigMatch *m)
 {
-    AppLayerTlsDetectVersionData *tls_data = (AppLayerTlsDetectVersionData *)m->ctx;
+    SCEnter();
+
+    DetectTlsVersionData *tls_data = (DetectTlsVersionData *)m->ctx;
     TlsState *tls_state = (TlsState *)state;
-    if (tls_state == NULL)
-        return 0;
+    if (tls_state == NULL) {
+        SCLogDebug("no tls state, no match");
+        SCReturnInt(0);
+    }
 
     int ret = 0;
     mutex_lock(&f->m);
+    SCLogDebug("looking for tls_data->ver 0x%02X (flags 0x%02X)", tls_data->ver, flags);
+
     if (flags & STREAM_TOCLIENT) {
-        if (tls_data->ver == tls_state->client_version)
+        SCLogDebug("server (toclient) version is 0x%02X", tls_state->server_version);
+        if (tls_data->ver == tls_state->server_version)
             ret = 1;
     } else if (flags & STREAM_TOSERVER) {
-        if (tls_data->ver == tls_state->server_version)
+        SCLogDebug("client (toserver) version is 0x%02X", tls_state->client_version);
+        if (tls_data->ver == tls_state->client_version)
             ret = 1;
     }
     mutex_unlock(&f->m);
-    return ret;
+
+    SCReturnInt(ret);
 }
 
 /**
@@ -111,13 +120,13 @@ int AppLayerTlsDetectVersionMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx
  *
  * \param idstr Pointer to the user provided id option
  *
- * \retval id_d pointer to AppLayerTlsDetectVersionData on success
+ * \retval id_d pointer to DetectTlsVersionData on success
  * \retval NULL on failure
  */
-AppLayerTlsDetectVersionData *AppLayerTlsDetectVersionParse (char *str)
+DetectTlsVersionData *DetectTlsVersionParse (char *str)
 {
     uint8_t temp;
-    AppLayerTlsDetectVersionData *tls = NULL;
+    DetectTlsVersionData *tls = NULL;
 	#define MAX_SUBSTRINGS 30
     int ret = 0, res = 0;
     int ov[MAX_SUBSTRINGS];
@@ -136,14 +145,14 @@ AppLayerTlsDetectVersionData *AppLayerTlsDetectVersionParse (char *str)
         char *tmp_str;
         res = pcre_get_substring((char *)str, ov, MAX_SUBSTRINGS, 1, &str_ptr);
         if (res < 0) {
-            SCLogDebug("AppLayerTlsDetectVersionParse: pcre_get_substring failed\n");
+            SCLogDebug("DetectTlsVersionParse: pcre_get_substring failed");
             goto error;
         }
 
         /* We have a correct id option */
-        tls = malloc(sizeof(AppLayerTlsDetectVersionData));
+        tls = malloc(sizeof(DetectTlsVersionData));
         if (tls == NULL) {
-            SCLogDebug("AppLayerTlsDetectVersionParse malloc failed\n");
+            SCLogDebug("DetectTlsVersionParse malloc failed");
             goto error;
         }
 
@@ -170,14 +179,14 @@ AppLayerTlsDetectVersionData *AppLayerTlsDetectVersionParse (char *str)
 
         free(orig);
 
-        SCLogDebug("will look for tls %"PRIu8"\n", tls->ver);
+        SCLogDebug("will look for tls %"PRIu8"", tls->ver);
     }
 
     return tls;
 
 error:
     if (tls != NULL)
-        AppLayerTlsDetectVersionFree(tls);
+        DetectTlsVersionFree(tls);
     return NULL;
 
 }
@@ -194,12 +203,12 @@ error:
  * \retval 0 on Success
  * \retval -1 on Failure
  */
-int AppLayerTlsDetectVersionSetup (DetectEngineCtx *de_ctx, Signature *s, SigMatch *m, char *str)
+int DetectTlsVersionSetup (DetectEngineCtx *de_ctx, Signature *s, SigMatch *m, char *str)
 {
-    AppLayerTlsDetectVersionData *tls = NULL;
+    DetectTlsVersionData *tls = NULL;
     SigMatch *sm = NULL;
 
-    tls = AppLayerTlsDetectVersionParse(str);
+    tls = DetectTlsVersionParse(str);
     if (tls == NULL) goto error;
 
     /* Okay so far so good, lets get this into a SigMatch
@@ -216,33 +225,33 @@ int AppLayerTlsDetectVersionSetup (DetectEngineCtx *de_ctx, Signature *s, SigMat
     return 0;
 
 error:
-    if (tls != NULL) AppLayerTlsDetectVersionFree(tls);
+    if (tls != NULL) DetectTlsVersionFree(tls);
     if (sm != NULL) free(sm);
     return -1;
 
 }
 
 /**
- * \brief this function will free memory associated with AppLayerTlsDetectVersionData
+ * \brief this function will free memory associated with DetectTlsVersionData
  *
- * \param id_d pointer to AppLayerTlsDetectVersionData
+ * \param id_d pointer to DetectTlsVersionData
  */
-void AppLayerTlsDetectVersionFree(void *ptr) {
-    AppLayerTlsDetectVersionData *id_d = (AppLayerTlsDetectVersionData *)ptr;
+void DetectTlsVersionFree(void *ptr) {
+    DetectTlsVersionData *id_d = (DetectTlsVersionData *)ptr;
     free(id_d);
 }
 
 #ifdef UNITTESTS /* UNITTESTS */
 
 /**
- * \test AppLayerTlsDetectVersionTestParse01 is a test to make sure that we parse the "id"
+ * \test DetectTlsVersionTestParse01 is a test to make sure that we parse the "id"
  *       option correctly when given valid id option
  */
-int AppLayerTlsDetectVersionTestParse01 (void) {
-    AppLayerTlsDetectVersionData *tls = NULL;
-    tls = AppLayerTlsDetectVersionParse("1.0");
+int DetectTlsVersionTestParse01 (void) {
+    DetectTlsVersionData *tls = NULL;
+    tls = DetectTlsVersionParse("1.0");
     if (tls != NULL && tls->ver == TLS_VERSION_10) {
-        AppLayerTlsDetectVersionFree(tls);
+        DetectTlsVersionFree(tls);
         return 1;
     }
 
@@ -250,15 +259,15 @@ int AppLayerTlsDetectVersionTestParse01 (void) {
 }
 
 /**
- * \test AppLayerTlsDetectVersionTestParse02 is a test to make sure that we parse the "id"
+ * \test DetectTlsVersionTestParse02 is a test to make sure that we parse the "id"
  *       option correctly when given an invalid id option
  *       it should return id_d = NULL
  */
-int AppLayerTlsDetectVersionTestParse02 (void) {
-    AppLayerTlsDetectVersionData *tls = NULL;
-    tls = AppLayerTlsDetectVersionParse("2.5");
+int DetectTlsVersionTestParse02 (void) {
+    DetectTlsVersionData *tls = NULL;
+    tls = DetectTlsVersionParse("2.5");
     if (tls == NULL) {
-        AppLayerTlsDetectVersionFree(tls);
+        DetectTlsVersionFree(tls);
         return 1;
     }
 
@@ -268,8 +277,8 @@ int AppLayerTlsDetectVersionTestParse02 (void) {
 #include "stream-tcp-reassemble.h"
 
 /** \test Send a get request in three chunks + more data. */
-static int AppLayerTlsDetectVersionTestDetect01(void) {
-    int result = 1;
+static int DetectTlsVersionTestDetect01(void) {
+    int result = 0;
     Flow f;
     uint8_t tlsbuf1[] = { 0x16 };
     uint32_t tlslen1 = sizeof(tlsbuf1);
@@ -299,6 +308,8 @@ static int AppLayerTlsDetectVersionTestDetect01(void) {
     StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
     f.protoctx = (void *)&ssn;
     p.flow = &f;
+    p.flowflags |= FLOW_PKT_TOSERVER;
+    ssn.alproto = ALPROTO_TLS;
 
     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
     if (de_ctx == NULL) {
@@ -362,6 +373,9 @@ static int AppLayerTlsDetectVersionTestDetect01(void) {
         goto end;
     }
 
+    SCLogDebug("tls_state is at %p, tls_state->server_version 0x%02X tls_state->client_version 0x%02X",
+        tls_state, tls_state->server_version, tls_state->client_version);
+
     /* do detect */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, &p);
 
@@ -380,7 +394,7 @@ end:
     return result;
 }
 
-static int AppLayerTlsDetectVersionTestDetect02(void) {
+static int DetectTlsVersionTestDetect02(void) {
     int result = 1;
     Flow f;
     uint8_t tlsbuf1[] = { 0x16 };
@@ -411,6 +425,7 @@ static int AppLayerTlsDetectVersionTestDetect02(void) {
     StreamL7DataPtrInit(&ssn,StreamL7GetStorageSize());
     f.protoctx = (void *)&ssn;
     p.flow = &f;
+    p.flowflags |= FLOW_PKT_TOSERVER;
 
     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
     if (de_ctx == NULL) {
@@ -478,6 +493,7 @@ static int AppLayerTlsDetectVersionTestDetect02(void) {
     SigMatchSignatures(&th_v, de_ctx, det_ctx, &p);
 
     if (PacketAlertCheck(&p, 1)) {
+        printf("signature 1 didn't match while it should have: ");
         goto end;
     }
 
@@ -495,13 +511,14 @@ end:
 #endif /* UNITTESTS */
 
 /**
- * \brief this function registers unit tests for AppLayerTlsDetectVersion
+ * \brief this function registers unit tests for DetectTlsVersion
  */
-void AppLayerTlsDetectVersionRegisterTests(void) {
+void DetectTlsVersionRegisterTests(void) {
 #ifdef UNITTESTS /* UNITTESTS */
-    UtRegisterTest("AppLayerTlsDetectVersionTestParse01", AppLayerTlsDetectVersionTestParse01, 1);
-    UtRegisterTest("AppLayerTlsDetectVersionTestParse02", AppLayerTlsDetectVersionTestParse02, 1);
-    UtRegisterTest("AppLayerTlsDetectVersionTestDetect01", AppLayerTlsDetectVersionTestDetect01, 1);
-    UtRegisterTest("AppLayerTlsDetectVersionTestDetect02", AppLayerTlsDetectVersionTestDetect02, 1);
+    UtRegisterTest("DetectTlsVersionTestParse01", DetectTlsVersionTestParse01, 1);
+    UtRegisterTest("DetectTlsVersionTestParse02", DetectTlsVersionTestParse02, 1);
+    UtRegisterTest("DetectTlsVersionTestDetect01", DetectTlsVersionTestDetect01, 1);
+    UtRegisterTest("DetectTlsVersionTestDetect02", DetectTlsVersionTestDetect02, 1);
 #endif /* UNITTESTS */
 }
+

src/detect-tls-version.h

@@ -0,0 +1,12 @@
+#ifndef __DETECT_TLS_VERSION_H__
+#define __DETECT_TLS_VERSION_H__
+
+typedef struct DetectTlsVersionData_ {
+    uint8_t ver; /** tls version to match */
+} DetectTlsVersionData;
+
+/* prototypes */
+void DetectTlsVersionRegister (void);
+
+#endif /* __DETECT_TLS_VERSION_H__ */
+

src/detect.c

@@ -64,7 +64,7 @@
 #include "util-rule-vars.h"
 
 #include "app-layer.h"
-#include "app-layer-tls-detect-version.h"
+#include "detect-tls-version.h"
 
 #include "action-globals.h"
 #include "tm-modules.h"
@@ -305,6 +305,8 @@ int PacketAlertAppend(Packet *p, uint32_t gid, uint32_t sid, uint8_t rev, uint8_
 }
 
 inline SigGroupHead *SigMatchSignaturesGetSgh(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p) {
+    SCEnter();
+
     int ds,f;
     SigGroupHead *sgh = NULL;
 
@@ -320,6 +322,8 @@ inline SigGroupHead *SigMatchSignaturesGetSgh(ThreadVars *th_v, DetectEngineCtx
     else
         f = 1;
 
+    SCLogDebug("ds %d, f %d", ds, f);
+
     /* find the right mpm instance */
     DetectAddress *ag = DetectAddressLookupInHead(de_ctx->dsize_gh[ds].flow_gh[f].src_gh[p->proto],&p->src);
     if (ag != NULL) {
@@ -327,46 +331,69 @@ inline SigGroupHead *SigMatchSignaturesGetSgh(ThreadVars *th_v, DetectEngineCtx
         ag = DetectAddressLookupInHead(ag->dst_gh,&p->dst);
         if (ag != NULL) {
             if (ag->port == NULL) {
+                SCLogDebug("we don't have ports");
                 sgh = ag->sh;
-
-                //printf("SigMatchSignatures: mc %p, mcu %p\n", det_ctx->mc, det_ctx->mcu);
-                //printf("sigs %" PRIu32 "\n", ag->sh->sig_cnt);
             } else {
-                //printf("SigMatchSignatures: we have ports\n");
+                SCLogDebug("we have ports");
 
                 DetectPort *sport = DetectPortLookupGroup(ag->port,p->sp);
                 if (sport != NULL) {
                     DetectPort *dport = DetectPortLookupGroup(sport->dst_ph,p->dp);
                     if (dport != NULL) {
                         sgh = dport->sh;
+                    } else {
+                        SCLogDebug("no dst port found for the packet");
                     }
+                } else {
+                    SCLogDebug("no src port found for the packet");
                 }
             }
+        } else {
+            SCLogDebug("no dst address found for the packet");
         }
+    } else {
+        SCLogDebug("no src address found for the packet");
     }
 
-    return sgh;
+    SCReturnPtr(sgh, "SigGroupHead");
 }
 
+/** \brief application layer detection
+ *
+ *  \param sgh signature group head for this proto/addrs/ports
+ */
 static int SigMatchSignaturesAppLayer(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, SigGroupHead *sgh, Packet *p)
 {
+    SCEnter();
+
     int match = 0, fmatch = 0;
     Signature *s = NULL;
     SigMatch *sm = NULL;
-    uint32_t idx,sig;
-
-    void *alstate = AppLayerGetProtoStateFromPacket(p);
-    if (alstate == NULL)
-        return 0;
-
+    uint32_t idx, sig;
     uint8_t flags = 0;
 
     /* if we didn't get a sig group head, we
      * have nothing to do.... */
     if (sgh == NULL) {
-        return 0;
+        SCLogDebug("no sgh to detect");
+        SCReturnInt(0);
+    }
+
+    /* grab the protocol state we will detect on */
+    void *alstate = AppLayerGetProtoStateFromPacket(p);
+    if (alstate == NULL) {
+        SCLogDebug("no application layer state to detect");
+        SCReturnInt(0);
+    }
+
+    if (p->flowflags & FLOW_PKT_TOSERVER) {
+        flags |= STREAM_TOSERVER;
+    } else if (p->flowflags & FLOW_PKT_TOCLIENT) {
+        flags |= STREAM_TOCLIENT;
     }
 
+    SCLogDebug("p->flowflags 0x%02X flags 0x%02X", p->flowflags, flags);
+
     /* inspect the sigs against the packet */
     for (idx = 0; idx < sgh->sig_cnt; idx++) {
         sig = sgh->match_array[idx];
@@ -409,39 +436,50 @@ static int SigMatchSignaturesAppLayer(ThreadVars *th_v, DetectEngineCtx *de_ctx,
                 continue;
         }
 
-        /* reset pkt ptr and offset */
-        det_ctx->pkt_ptr = NULL;
-        det_ctx->pkt_off = 0;
+        /* if we don't have sigmatches at this point we're a match */
+        if (s->match == NULL) {
+            fmatch = 1;
+            if (!(s->flags & SIG_FLAG_NOALERT)) {
+                PacketAlertAppend(p, 1, s->id, s->rev, s->prio, s->msg);
 
-        sm = s->match;
-        while (sm) {
-            if (sigmatch_table[sm->type].AppLayerMatch == NULL)
-                continue;
+                /* set verdict on packet */
+                p->action = s->action;
+            }
+        } else {
+            /* reset pkt ptr and offset */
+            det_ctx->pkt_ptr = NULL;
+            det_ctx->pkt_off = 0;
+
+            sm = s->match;
+            while (sm) {
+                if (sigmatch_table[sm->type].AppLayerMatch == NULL)
+                    continue;
 
-            match = sigmatch_table[sm->type].AppLayerMatch(th_v, det_ctx, p->flow, flags, alstate, s, sm);
-            if (match) {
-                /* okay, try the next match */
-                sm = sm->next;
+                match = sigmatch_table[sm->type].AppLayerMatch(th_v, det_ctx, p->flow, flags, alstate, s, sm);
+                if (match) {
+                    /* okay, try the next match */
+                    sm = sm->next;
 
-                /* only if the last matched as well, we have a hit */
-                if (sm == NULL) {
-                    fmatch = 1;
-                    //printf("DE : sig %" PRIu32 " matched\n", s->id);
-                    if (!(s->flags & SIG_FLAG_NOALERT)) {
-                        PacketAlertAppend(p, 1, s->id, s->rev, s->prio, s->msg);
+                    /* only if the last matched as well, we have a hit */
+                    if (sm == NULL) {
+                        fmatch = 1;
+                        //printf("DE : sig %" PRIu32 " matched\n", s->id);
+                        if (!(s->flags & SIG_FLAG_NOALERT)) {
+                            PacketAlertAppend(p, 1, s->id, s->rev, s->prio, s->msg);
 
-                        /* set verdict on packet */
-                        p->action = s->action;
+                            /* set verdict on packet */
+                            p->action = s->action;
+                        }
                     }
+                } else {
+                    /* done with this sig */
+                    sm = NULL;
                 }
-            } else {
-                /* done with this sig */
-                sm = NULL;
             }
         }
     }
 
-    return fmatch;
+    SCReturnInt(fmatch);
 }
 
 int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p)
@@ -825,6 +863,7 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) {
     DetectAddress *gr = NULL;
     uint32_t cnt = 0, cnt_iponly = 0;
     uint32_t cnt_payload = 0;
+    uint32_t cnt_applayer = 0;
 
     //DetectAddressPrintMemory();
     //DetectSigGroupPrintMemory();
@@ -849,21 +888,26 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) {
     for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) {
 
         de_ctx->sig_array[tmp_s->num] = tmp_s;
-        //printf(" + Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", tmp_s->id, tmp_s->num, tmp_s, de_ctx->sig_array[tmp_s->num]);
+        SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", tmp_s->id, tmp_s->num, tmp_s, de_ctx->sig_array[tmp_s->num]);
 
         /* see if the sig is ip only */
         if (SignatureIsIPOnly(de_ctx, tmp_s) == 1) {
             tmp_s->flags |= SIG_FLAG_IPONLY;
             cnt_iponly++;
-            //printf("(IP only)\n");
-            /*see if any sig is inspecting the packet payload*/
+
+            SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", tmp_s->id);
+
+        /* see if any sig is inspecting the packet payload */
         } else if (SignatureIsInspectingPayload(de_ctx, tmp_s) == 1) {
             tmp_s->flags |= SIG_FLAG_PAYLOAD;
             cnt_payload++;
-            //printf("\n");
-            //if (tmp_s->proto.flags & DETECT_PROTO_ANY) {
-            //printf("Signature %" PRIu32 " applies to all protocols.\n",tmp_s->id);
-            //}
+
+            SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", tmp_s->id);
+        }
+
+        if (tmp_s->flags & SIG_FLAG_APPLAYER) {
+            SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", tmp_s->id);
+            cnt_applayer++;
         }
 
 #ifdef DEBUG
@@ -920,8 +964,8 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) {
     //DetectPortPrintMemory();
 
     if (!(de_ctx->flags & DE_QUIET)) {
-        SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only rules and %" PRIu32 " are inspecting packet payload",
-            de_ctx->sig_cnt, cnt_iponly, cnt_payload);
+        SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only rules, %" PRIu32 " are inspecting packet payload, %"PRIu32" inspect application layer",
+            de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer);
         SCLogInfo("building signature grouping structure, stage 1: "
                "adding signatures to signature source addresses... done");
     }
@@ -2748,7 +2792,7 @@ void SigTableSetup(void) {
     DetectTtlRegister();
     DetectFastPatternRegister();
 
-    AppLayerTlsDetectVersionRegister();
+    DetectTlsVersionRegister();
 
     uint8_t i = 0;
     for (i = 0; i < DETECT_TBLSIZE; i++) {

src/util-debug.h

@@ -478,8 +478,8 @@ extern int sc_log_module_cleaned;
  */
 #define SCReturnPtr(x, type)  do {                                           \
                                   if (sc_log_global_log_level >= SC_LOG_DEBUG) { \
-                                      SCLogDebug("Returning pointer of "     \
-                                              "type %s ... <<", type);       \
+                                      SCLogDebug("Returning pointer %p of "  \
+                                              "type %s ... <<", x, type);    \
                                       SCLogCheckFDFilterExit(__FUNCTION__);  \
                                   }                                          \
                                   return x;                                  \