support relative pcre for client body. All pcre processing for client body moved to hcbd engine

src/detect-distance.c

@@ -396,22 +396,29 @@ static int DetectDistanceSetup (DetectEngineCtx *de_ctx, Signature *s,
 
             /* reassigning pm */
             pm = SigMatchGetLastSMFromLists(s, 2,
-                                            DETECT_AL_HTTP_CLIENT_BODY, pm->prev);
+                                            DETECT_AL_HTTP_CLIENT_BODY, pm->prev,
+                                            DETECT_PCRE, pm->prev);
             if (pm == NULL) {
                 SCLogError(SC_ERR_DISTANCE_MISSING_CONTENT, "distance for http_client_body "
                            "needs preceeding http_client_body content");
                 goto error;
             }
-            /* reassigning cd */
+
-            cd = (DetectContentData *)pm->ctx;
+            if (pm->type == DETECT_PCRE) {
-            if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
+                DetectPcreData *tmp_pd = (DetectPcreData *)pm->ctx;
-                SCLogError(SC_ERR_INVALID_SIGNATURE, "Previous keyword "
+                tmp_pd->flags |=  DETECT_PCRE_RELATIVE_NEXT;
-                           "has a fast_pattern:only; set.  You can't "
+            } else {
-                           "have relative keywords around a fast_pattern "
+                /* reassigning cd */
-                           "only content");
+                cd = (DetectContentData *)pm->ctx;
-                goto error;
+                if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
+                    SCLogError(SC_ERR_INVALID_SIGNATURE, "Previous keyword "
+                               "has a fast_pattern:only; set.  You can't "
+                               "have relative keywords around a fast_pattern "
+                               "only content");
+                    goto error;
+                }
+                cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
             }
-            cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
 
             break;
 

src/detect-http-client-body.c

@@ -240,16 +240,22 @@ int DetectHttpClientBodySetup(DetectEngineCtx *de_ctx, Signature *s, char *arg)
         } /* if (pm != NULL) */
 
         /* reassigning pm */
-        pm = SigMatchGetLastSMFromLists(s, 2,
+        pm = SigMatchGetLastSMFromLists(s, 4,
-                                        DETECT_AL_HTTP_CLIENT_BODY, s->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]);
+                                        DETECT_AL_HTTP_CLIENT_BODY, s->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH],
+                                        DETECT_PCRE, s->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]);
         if (pm == NULL) {
             SCLogError(SC_ERR_INVALID_SIGNATURE, "http_client_body seen with a "
                        "distance or within without a previous http_client_body "
                        "content.  Invalidating signature.");
             goto error;
         }
-        DetectContentData *tmp_cd = (DetectContentData *)pm->ctx;
+        if (pm->type == DETECT_PCRE) {
-        tmp_cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
+            DetectPcreData *tmp_pd = (DetectPcreData *)pm->ctx;
+            tmp_pd->flags |= DETECT_PCRE_RELATIVE_NEXT;
+        } else {
+            DetectContentData *tmp_cd = (DetectContentData *)pm->ctx;
+            tmp_cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
+        }
     }
     cd->id = DetectPatternGetId(de_ctx->mpm_pattern_id_store, cd, DETECT_AL_HTTP_CLIENT_BODY);
     sm->type = DETECT_AL_HTTP_CLIENT_BODY;
@@ -2529,6 +2535,159 @@ int DetectHttpClientBodyTest33(void)
     return result;
 }
 
+int DetectHttpClientBodyTest34(void)
+{
+    DetectEngineCtx *de_ctx = NULL;
+    int result = 0;
+
+    if ( (de_ctx = DetectEngineCtxInit()) == NULL)
+        goto end;
+
+    de_ctx->flags |= DE_QUIET;
+    de_ctx->sig_list = SigInit(de_ctx, "alert icmp any any -> any any "
+                               "(pcre:/one/P; "
+                               "content:two; within:5; http_client_body; sid:1;)");
+    if (de_ctx->sig_list == NULL) {
+        printf("de_ctx->sig_list == NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists[DETECT_SM_LIST_PMATCH] != NULL) {
+        printf("de_ctx->sig_list->sm_lists[DETECT_SM_LIST_PMATCH] != NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists[DETECT_SM_LIST_HCBDMATCH] == NULL) {
+        printf("de_ctx->sig_list->sm_lists[DETECT_SM_LIST_HCBDMATCH] == NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH] == NULL ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->type != DETECT_AL_HTTP_CLIENT_BODY ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev == NULL ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev->type != DETECT_PCRE) {
+
+        goto end;
+    }
+
+    DetectPcreData *pd1 = de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev->ctx;
+    DetectContentData *hcbd2 = de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->ctx;
+    if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT | DETECT_PCRE_HTTP_BODY_AL) ||
+        hcbd2->flags != DETECT_CONTENT_WITHIN ||
+        memcmp(hcbd2->content, "two", hcbd2->content_len) != 0) {
+        goto end;
+    }
+
+    result = 1;
+
+ end:
+    SigCleanSignatures(de_ctx);
+    DetectEngineCtxFree(de_ctx);
+    return result;
+}
+
+int DetectHttpClientBodyTest35(void)
+{
+    DetectEngineCtx *de_ctx = NULL;
+    int result = 0;
+
+    if ( (de_ctx = DetectEngineCtxInit()) == NULL)
+        goto end;
+
+    de_ctx->flags |= DE_QUIET;
+    de_ctx->sig_list = SigInit(de_ctx, "alert icmp any any -> any any "
+                               "(content:two; http_client_body; "
+                               "pcre:/one/PR; sid:1;)");
+    if (de_ctx->sig_list == NULL) {
+        printf("de_ctx->sig_list == NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists[DETECT_SM_LIST_PMATCH] != NULL) {
+        printf("de_ctx->sig_list->sm_lists[DETECT_SM_LIST_PMATCH] != NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists[DETECT_SM_LIST_HCBDMATCH] == NULL) {
+        printf("de_ctx->sig_list->sm_lists[DETECT_SM_LIST_HCBDMATCH] == NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH] == NULL ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->type != DETECT_PCRE ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev == NULL ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev->type != DETECT_AL_HTTP_CLIENT_BODY) {
+
+        goto end;
+    }
+
+    DetectContentData *hcbd1 = de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev->ctx;
+    DetectPcreData *pd2 = de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->ctx;
+    if (pd2->flags != (DETECT_PCRE_RELATIVE | DETECT_PCRE_HTTP_BODY_AL) ||
+        hcbd1->flags != DETECT_CONTENT_RELATIVE_NEXT ||
+        memcmp(hcbd1->content, "two", hcbd1->content_len) != 0) {
+        goto end;
+    }
+
+    result = 1;
+
+ end:
+    SigCleanSignatures(de_ctx);
+    DetectEngineCtxFree(de_ctx);
+    return result;
+}
+
+int DetectHttpClientBodyTest36(void)
+{
+    DetectEngineCtx *de_ctx = NULL;
+    int result = 0;
+
+    if ( (de_ctx = DetectEngineCtxInit()) == NULL)
+        goto end;
+
+    de_ctx->flags |= DE_QUIET;
+    de_ctx->sig_list = SigInit(de_ctx, "alert icmp any any -> any any "
+                               "(pcre:/one/P; "
+                               "content:two; distance:5; http_client_body; sid:1;)");
+    if (de_ctx->sig_list == NULL) {
+        printf("de_ctx->sig_list == NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists[DETECT_SM_LIST_PMATCH] != NULL) {
+        printf("de_ctx->sig_list->sm_lists[DETECT_SM_LIST_PMATCH] != NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists[DETECT_SM_LIST_HCBDMATCH] == NULL) {
+        printf("de_ctx->sig_list->sm_lists[DETECT_SM_LIST_HCBDMATCH] == NULL\n");
+        goto end;
+    }
+
+    if (de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH] == NULL ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->type != DETECT_AL_HTTP_CLIENT_BODY ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev == NULL ||
+        de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev->type != DETECT_PCRE) {
+
+        goto end;
+    }
+
+    DetectPcreData *pd1 = de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->prev->ctx;
+    DetectContentData *hcbd2 = de_ctx->sig_list->sm_lists_tail[DETECT_SM_LIST_HCBDMATCH]->ctx;
+    if (pd1->flags != (DETECT_PCRE_RELATIVE_NEXT | DETECT_PCRE_HTTP_BODY_AL) ||
+        hcbd2->flags != DETECT_CONTENT_DISTANCE ||
+        memcmp(hcbd2->content, "two", hcbd2->content_len) != 0) {
+        goto end;
+    }
+
+    result = 1;
+
+ end:
+    SigCleanSignatures(de_ctx);
+    DetectEngineCtxFree(de_ctx);
+    return result;
+}
+
 #endif /* UNITTESTS */
 
 void DetectHttpClientBodyRegisterTests(void)
@@ -2568,6 +2727,9 @@ void DetectHttpClientBodyRegisterTests(void)
     UtRegisterTest("DetectHttpClientBodyTest31", DetectHttpClientBodyTest31, 1);
     UtRegisterTest("DetectHttpClientBodyTest32", DetectHttpClientBodyTest32, 1);
     UtRegisterTest("DetectHttpClientBodyTest33", DetectHttpClientBodyTest33, 1);
+    UtRegisterTest("DetectHttpClientBodyTest34", DetectHttpClientBodyTest34, 1);
+    UtRegisterTest("DetectHttpClientBodyTest35", DetectHttpClientBodyTest35, 1);
+    UtRegisterTest("DetectHttpClientBodyTest36", DetectHttpClientBodyTest36, 1);
 #endif /* UNITTESTS */
 
     return;

src/detect-parse.c

@@ -169,6 +169,32 @@ void SigMatchAppendAppLayer(Signature *s, SigMatch *new) {
     s->sm_cnt++;
 }
 
+/**
+ * \brief Append a SigMatch to the list type.
+ *
+ * \param s    Signature.
+ * \param new  The sig match to append.
+ * \param list The list to append to.
+ */
+void SigMatchAppendSMToList(Signature *s, SigMatch *new, int list)
+{
+    if (s->sm_lists[list] == NULL) {
+        s->sm_lists[list] = new;
+        s->sm_lists_tail[list] = new;
+        new->next = NULL;
+        new->prev = NULL;
+    } else {
+        SigMatch *cur = s->sm_lists_tail[list];
+        cur->next = new;
+        new->prev = cur;
+        new->next = NULL;
+        s->sm_lists_tail[list] = new;
+    }
+
+    new->idx = s->sm_cnt;
+    s->sm_cnt++;
+}
+
 /**
  * \brief append a SigMatch of type uricontent to the Signature structure
  * \param s pointer to the Signature

src/detect-parse.h

@@ -63,6 +63,7 @@ void SigMatchAppendPacket(Signature *, SigMatch *);
 void SigMatchAppendUricontent(Signature *, SigMatch *);
 void SigMatchAppendAppLayer(Signature *, SigMatch *);
 void SigMatchAppendTag(Signature *, SigMatch *);
+void SigMatchAppendSMToList(Signature *, SigMatch *, int);
 
 int DetectParseDupSigHashInit(DetectEngineCtx *);
 void DetectParseDupSigHashFree(DetectEngineCtx *);

src/detect-pcre.c

@@ -743,8 +743,8 @@ int DetectPcrePayloadMatch(DetectEngineThreadCtx *det_ctx, Signature *s,
     DetectPcreData *pe = (DetectPcreData *)sm->ctx;
 
     /* If we want to inspect the http body, we will use HTP L7 parser */
-    if (pe->flags & DETECT_PCRE_HTTP_BODY_AL)
+    //if (pe->flags & DETECT_PCRE_HTTP_BODY_AL)
-        SCReturnInt(0);
+    //    SCReturnInt(0);
 
     if (s->flags & SIG_FLAG_RECURSIVE) {
         ptr = payload + det_ctx->payload_offset;
@@ -1320,13 +1320,11 @@ static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, char *regexst
 
         SigMatchAppendAppLayer(s, sm);
     } else if (pd->flags & DETECT_PCRE_HTTP_BODY_AL) {
-        sm->type = DETECT_PCRE_HTTPBODY;
-
         SCLogDebug("Body inspection modifier set");
         s->flags |= SIG_FLAG_APPLAYER;
         AppLayerHtpEnableRequestBodyCallback();
 
-        SigMatchAppendAppLayer(s, sm);
+        SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_HCBDMATCH);
     } else if (pd->flags & DETECT_PCRE_URI) {
         s->flags |= SIG_FLAG_APPLAYER;
 
@@ -1340,9 +1338,7 @@ static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, char *regexst
 
         SigMatchAppendUricontent(s, sm);
     } else {
-        if (s->alproto == ALPROTO_DCERPC &&
+        if (s->alproto == ALPROTO_DCERPC && pd->flags & DETECT_PCRE_RELATIVE) {
-            pd->flags & DETECT_PCRE_RELATIVE)
-        {
             SigMatch *pm = NULL;
             SigMatch *dm = NULL;
 
@@ -1376,7 +1372,7 @@ static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, char *regexst
     prev_sm = SigMatchGetLastSMFromLists(s, 8,
                                          DETECT_CONTENT, sm->prev,
                                          DETECT_URICONTENT, sm->prev,
-                                         DETECT_BYTEJUMP, sm->prev,
+                                         DETECT_AL_HTTP_CLIENT_BODY, sm->prev,
                                          DETECT_PCRE, sm->prev);
     if (prev_sm == NULL) {
         if (s->alproto == ALPROTO_DCERPC) {
@@ -1391,11 +1387,12 @@ static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, char *regexst
     }
 
     DetectContentData *cd = NULL;
-    DetectContentData *ud = NULL;
     DetectPcreData *pe = NULL;
 
     switch (prev_sm->type) {
         case DETECT_CONTENT:
+        case DETECT_URICONTENT:
+        case DETECT_AL_HTTP_CLIENT_BODY:
             /* Set the relative next flag on the prev sigmatch */
             cd = (DetectContentData *)prev_sm->ctx;
             if (cd == NULL) {
@@ -1406,17 +1403,6 @@ static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, char *regexst
 
             break;
 
-        case DETECT_URICONTENT:
-            /* Set the relative next flag on the prev sigmatch */
-            ud = (DetectContentData *)prev_sm->ctx;
-            if (ud == NULL) {
-                SCLogError(SC_ERR_INVALID_SIGNATURE, "uricontent not setup properly");
-                SCReturnInt(-1);
-            }
-            ud->flags |= DETECT_CONTENT_RELATIVE_NEXT;
-
-            break;
-
         case DETECT_PCRE:
             pe = (DetectPcreData *) prev_sm->ctx;
             if (pe == NULL) {
@@ -1427,12 +1413,6 @@ static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, char *regexst
 
             break;
 
-        case DETECT_BYTEJUMP:
-            SCLogDebug("No setting relative_next for bytejump.  We "
-                       "have no use for it");
-
-            break;
-
         default:
             /* this will never hit */
             SCLogError(SC_ERR_INVALID_SIGNATURE, "prev sigmatch has unknown type: %"PRIu16,
@@ -2012,11 +1992,10 @@ static int DetectPcreModifPTest04(void) {
         "Transfer-Encoding: chunked\r\n"
         "Content-Type: text/html; charset=utf-8\r\n"
         "\r\n"
-        "88b7\r\n"
+        "15"
         "\r\n"
-        "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\r\n"
+        "<!DOCTYPE html PUBLIC\r\n"
-        "\r\n"
+        "0\r\n";
-        "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en-gb\" lang=\"en-gb\">\r\n\r\n";
 
     uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
     TcpSession ssn;
@@ -2135,13 +2114,11 @@ static int DetectPcreModifPTest05(void) {
         "Transfer-Encoding: chunked\r\n"
         "Content-Type: text/html; charset=utf-8\r\n"
         "\r\n"
-        "88b7\r\n"
+        "15"
         "\r\n"
         "<!DOC";
 
-    uint8_t httpbuf2[] = "TYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Transitional//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\">\r\n"
+    uint8_t httpbuf2[] = "<!DOCTYPE html PUBLIC\r\n0\r\n";
-        "\r\n"
-        "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en-gb\" lang=\"en-gb\">\r\n\r\n";
 
     uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
     uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
@@ -2218,7 +2195,7 @@ static int DetectPcreModifPTest05(void) {
         goto end;
     }
 
-    if (!(PacketAlertCheck(p1, 1))) {
+    if (PacketAlertCheck(p1, 1)) {
         printf("sid 1 didn't match on p1 but should have: ");
         goto end;
     }
@@ -2239,7 +2216,7 @@ static int DetectPcreModifPTest05(void) {
     /* do detect for p2 */
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p2);
 
-    if ((PacketAlertCheck(p2, 1))) {
+    if (!(PacketAlertCheck(p2, 1))) {
         printf("sid 1 did match on p2 but should have: ");
         goto end;
     }

src/detect-within.c

@@ -417,23 +417,30 @@ static int DetectWithinSetup (DetectEngineCtx *de_ctx, Signature *s, char *withi
             cd->flags |= DETECT_CONTENT_WITHIN;
 
             /* reassigning pm */
-            pm = SigMatchGetLastSMFromLists(s, 2,
+            pm = SigMatchGetLastSMFromLists(s, 4,
-                                            DETECT_AL_HTTP_CLIENT_BODY, pm->prev);
+                                            DETECT_AL_HTTP_CLIENT_BODY, pm->prev,
+                                            DETECT_PCRE, pm->prev);
             if (pm == NULL) {
                 SCLogError(SC_ERR_DISTANCE_MISSING_CONTENT, "distance for http_client_body "
                            "needs preceeding http_client_body content");
                 goto error;
             }
-            /* reassigning cd */
+
-            cd = (DetectContentData *)pm->ctx;
+            if (pm->type == DETECT_PCRE) {
-            if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
+                DetectPcreData *tmp_pd = (DetectPcreData *)pm->ctx;
-                SCLogError(SC_ERR_INVALID_SIGNATURE, "Previous keyword "
+                tmp_pd->flags |=  DETECT_PCRE_RELATIVE_NEXT;
-                           "has a fast_pattern:only; set.  You can't "
+            } else {
-                           "have relative keywords around a fast_pattern "
+                /* reassigning cd */
-                           "only content");
+                cd = (DetectContentData *)pm->ctx;
-                goto error;
+                if (cd->flags & DETECT_CONTENT_FAST_PATTERN_ONLY) {
+                    SCLogError(SC_ERR_INVALID_SIGNATURE, "Previous keyword "
+                               "has a fast_pattern:only; set.  You can't "
+                               "have relative keywords around a fast_pattern "
+                               "only content");
+                    goto error;
+                }
+                cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
             }
-            cd->flags |= DETECT_CONTENT_RELATIVE_NEXT;
 
             break;