make client body buffer limit configurable. Also some minor changes

src/detect-engine-hcbd.c

@@ -57,7 +57,7 @@
 #include "app-layer-protos.h"
 
 /**
- * \brief Run the actual payload match function for uricontent.
+ * \brief Run the actual payload match function for http request body.
  *
  *        For accounting the last match in relative matching the
  *        det_ctx->payload_offset int is used.
@@ -66,8 +66,8 @@
  * \param det_ctx     Detection engine thread context.
  * \param s           Signature to inspect.
  * \param sm          SigMatch to inspect.
- * \param payload     Ptr to the uricontent payload to inspect.
+ * \param payload     Ptr to the request body to inspect.
- * \param payload_len Length of the uricontent payload.
+ * \param payload_len Length of the request body.
  *
  * \retval 0 no match.
  * \retval 1 match.
@@ -252,15 +252,15 @@ match:
  * \param det_ctx   Detection engine thread ctx.
  * \param f         Pointer to the flow.
  * \param htp_state http state.
- * \param call_mpm  1 if we are also to call the mpm no the buffered bodies or
- *                  0 if we to just buffer the bodies.
  *
  * \retval cnt The match count from the mpm call.  If call_mpm is 0, the retval
  *             is ignored.
  */
-static uint32_t DetectEngineInspectHttpClientBodyMpmInspect(DetectEngineThreadCtx *det_ctx,
+static uint32_t DetectEngineInspectHttpClientBodyMpmInspect(DetectEngineCtx *de_ctx,
-                                                            Flow *f, HtpState *htp_state,
+                                                            DetectEngineThreadCtx *det_ctx,
-                                                            int call_mpm) {
+                                                            Signature *s, Flow *f,
+                                                            HtpState *htp_state)
+{
     uint32_t cnt = 0;
     size_t idx = 0;
     htp_tx_t *tx = NULL;
@@ -271,7 +271,8 @@ static uint32_t DetectEngineInspectHttpClientBodyMpmInspect(DetectEngineThreadCt
 
         /* if the buffer already exists, use it */
         if (det_ctx->hcbd_buffers[i] != NULL) {
-            if (call_mpm) {
+            /* we only call the mpm if the hcbd mpm has been set */
+            if (s->flags & SIG_FLAG_MPM_HCBDCONTENT) {
                 cnt += HttpClientBodyPatternSearch(det_ctx,
                                                    det_ctx->hcbd_buffers[i],
                                                    det_ctx->hcbd_buffers_len[i]);
@@ -279,7 +280,6 @@ static uint32_t DetectEngineInspectHttpClientBodyMpmInspect(DetectEngineThreadCt
             continue;
         }
 
-        tx = list_get(htp_state->connp->conn->transactions, idx);
         tx = list_get(htp_state->connp->conn->transactions, idx);
         if (tx == NULL)
             continue;
@@ -314,11 +314,11 @@ static uint32_t DetectEngineInspectHttpClientBodyMpmInspect(DetectEngineThreadCt
             }
 
             uint8_t *chunks_buffer = NULL;
-            uint32_t chunks_buffer_len = 0;
+            int32_t chunks_buffer_len = 0;
             while (cur != NULL) {
                 /* \todo Currently we limit the body length we inspect.  We
                  * should change to handling chunks statefully */
-                if (chunks_buffer_len > 20000)
+                if (chunks_buffer_len > de_ctx->hcbd_buffer_limit)
                     break;
                 chunks_buffer_len += cur->len;
                 if ( (chunks_buffer = SCRealloc(chunks_buffer, chunks_buffer_len)) == NULL) {
@@ -332,8 +332,8 @@ static uint32_t DetectEngineInspectHttpClientBodyMpmInspect(DetectEngineThreadCt
             det_ctx->hcbd_buffers[i] = chunks_buffer;
             det_ctx->hcbd_buffers_len[i] = chunks_buffer_len;
 
-            /* carry out the mpm */
+            /* carry out the mpm if we have hcbd mpm set */
-            if (call_mpm)
+            if (s->flags & SIG_FLAG_MPM_HCBDCONTENT)
                 cnt += HttpClientBodyPatternSearch(det_ctx, chunks_buffer, chunks_buffer_len);
         } /* else - if (htud->body.nchunks == 0) */
     } /* for (idx = AppLayerTransactionGetInspectId(f); .. */
@@ -395,31 +395,34 @@ int DetectEngineInspectHttpClientBody(DetectEngineCtx *de_ctx,
             goto end;
 
         /* assign space to hold buffers.  Each per transaction */
-        det_ctx->hcbd_buffers = malloc(det_ctx->hcbd_buffers_list_len * sizeof(uint8_t *));
+        det_ctx->hcbd_buffers = SCMalloc(det_ctx->hcbd_buffers_list_len * sizeof(uint8_t *));
         if (det_ctx->hcbd_buffers == NULL) {
             SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
-            exit(EXIT_FAILURE);
+            goto end;
         }
         memset(det_ctx->hcbd_buffers, 0, det_ctx->hcbd_buffers_list_len * sizeof(uint8_t *));
 
-        det_ctx->hcbd_buffers_len = malloc(det_ctx->hcbd_buffers_list_len * sizeof(uint32_t));
+        det_ctx->hcbd_buffers_len = SCMalloc(det_ctx->hcbd_buffers_list_len * sizeof(uint32_t));
         if (det_ctx->hcbd_buffers_len == NULL) {
             SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
-            exit(EXIT_FAILURE);
+            goto end;
         }
         memset(det_ctx->hcbd_buffers_len, 0, det_ctx->hcbd_buffers_list_len * sizeof(uint32_t));
     } /* if (det_ctx->hcbd_buffers_list_len == 0) */
 
     if (s->flags & SIG_FLAG_MPM_HCBDCONTENT) {
         if (det_ctx->de_mpm_scanned_hcbd == FALSE) {
-            uint32_t cnt = DetectEngineInspectHttpClientBodyMpmInspect(det_ctx, f, htp_state, 1);
+            uint32_t cnt = DetectEngineInspectHttpClientBodyMpmInspect(de_ctx,
+                                                                       det_ctx, s,
+                                                                       f, htp_state);
             if (cnt <= 0)
                 det_ctx->de_have_hcbd = FALSE;
 
             det_ctx->de_mpm_scanned_hcbd = TRUE;
         }
     } else {
-        DetectEngineInspectHttpClientBodyMpmInspect(det_ctx, f, htp_state, 0);
+        DetectEngineInspectHttpClientBodyMpmInspect(de_ctx, det_ctx, s, f,
+                                                    htp_state);
     }
 
     if (det_ctx->de_have_hcbd == FALSE &&
@@ -456,6 +459,28 @@ end:
     SCReturnInt(r);
 }
 
+/**
+ * \brief Clean the hcbd buffers.
+ *
+ * \param det_ctx Pointer to the detection engine thread ctx.
+ */
+void DetectEngineCleanHCBDBuffers(DetectEngineThreadCtx *det_ctx)
+{
+    if (det_ctx->hcbd_buffers_list_len != 0) {
+        int i;
+        for (i = 0; i < det_ctx->hcbd_buffers_list_len; i++) {
+            if (det_ctx->hcbd_buffers[i] != NULL)
+                SCFree(det_ctx->hcbd_buffers[i]);
+        }
+        SCFree(det_ctx->hcbd_buffers);
+        det_ctx->hcbd_buffers = NULL;
+        det_ctx->hcbd_buffers_list_len = 0;
+    }
+
+    return;
+}
+
+
 /***********************************Unittests**********************************/
 
 #ifdef UNITTESTS

src/detect-engine-hcbd.h

@@ -23,8 +23,11 @@
 #ifndef __DETECT_ENGINE_HCBD_H__
 #define __DETECT_ENGINE_HCBD_H__
 
+#define ENGINE_HCBD_BUFFER_LIMIT 20000
+
 int DetectEngineInspectHttpClientBody(DetectEngineCtx *, DetectEngineThreadCtx *,
                                       Signature *, Flow *, uint8_t, void *);
+void DetectEngineCleanHCBDBuffers(DetectEngineThreadCtx *);
 void HttpClientBodyRegisterTests(void);
 
 #endif /* __DETECT_ENGINE_HCBD_H__ */

src/detect-engine-mpm.c

@@ -2330,8 +2330,9 @@ uint32_t DetectContentGetId(MpmPatternIdStore *ht, DetectContentData *co) {
     MpmPatternIdTableElmt *r = NULL;
     uint32_t id = 0;
 
-    e = malloc(sizeof(MpmPatternIdTableElmt));
+    e = SCMalloc(sizeof(MpmPatternIdTableElmt));
     BUG_ON(e == NULL);
+    memset(e, 0, sizeof(MpmPatternIdTableElmt));
     e->pattern = SCMalloc(co->content_len);
     BUG_ON(e->pattern == NULL);
     memcpy(e->pattern, co->content, co->content_len);
@@ -2379,7 +2380,7 @@ uint32_t DetectUricontentGetId(MpmPatternIdStore *ht, DetectContentData *co) {
     MpmPatternIdTableElmt *r = NULL;
     uint32_t id = 0;
 
-    e = malloc(sizeof(MpmPatternIdTableElmt));
+    e = SCMalloc(sizeof(MpmPatternIdTableElmt));
     BUG_ON(e == NULL);
     e->pattern = SCMalloc(co->content_len);
     BUG_ON(e->pattern == NULL);
@@ -2438,7 +2439,7 @@ uint32_t DetectPatternGetId(MpmPatternIdStore *ht, void *ctx, uint8_t sm_type)
     MpmPatternIdTableElmt *r = NULL;
     PatIntId id = 0;
 
-    e = malloc(sizeof(MpmPatternIdTableElmt));
+    e = SCMalloc(sizeof(MpmPatternIdTableElmt));
     if (e == NULL) {
         SCLogError(SC_ERR_MEM_ALLOC, "Error allocating memory");
         exit(EXIT_FAILURE);

src/detect-engine.c

@@ -35,6 +35,7 @@
 #include "detect-engine-address.h"
 #include "detect-engine-port.h"
 #include "detect-engine-mpm.h"
+#include "detect-engine-hcbd.h"
 #include "detect-engine-iponly.h"
 #include "detect-engine-tag.h"
 
@@ -188,6 +189,7 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) {
     const char *max_uniq_toserver_dp_groups_str = NULL;
 
     char *sgh_mpm_context = NULL;
+    char *hcbd_buffer_limit = NULL;
 
     ConfNode *de_ctx_custom = ConfGetNode("detect-engine");
     ConfNode *opt = NULL;
@@ -198,6 +200,9 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) {
                 de_ctx_profile = opt->head.tqh_first->val;
             } else if (strcmp(opt->val, "sgh-mpm-context") == 0) {
                 sgh_mpm_context = opt->head.tqh_first->val;
+            } else if (strcmp(opt->val,
+                              "http-client-request-body-buffer-limit") == 0) {
+                hcbd_buffer_limit = opt->head.tqh_first->val;
             }
         }
     }
@@ -239,6 +244,14 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) {
         }
     }
 
+    if (hcbd_buffer_limit == NULL) {
+        de_ctx->hcbd_buffer_limit = ENGINE_HCBD_BUFFER_LIMIT;
+    } else {
+        de_ctx->hcbd_buffer_limit = atoi(hcbd_buffer_limit);
+        if (de_ctx->hcbd_buffer_limit <= 0)
+            de_ctx->hcbd_buffer_limit = ENGINE_HCBD_BUFFER_LIMIT;
+    }
+
     opt = NULL;
     switch (profile) {
         case ENGINE_PROFILE_LOW:

src/detect-http-raw-header.h

@@ -0,0 +1,30 @@
+/* Copyright (C) 2007-2010 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Pablo Rincon <pablo.rincon.crespo@gmail.com>
+ */
+
+#ifndef __DETECT_HTTP_HEADER_H__
+#define __DETECT_HTTP_HEADER_H__
+
+void DetectHttpHeaderRegister(void);
+void DetectHttpRawHeaderRegister(void);
+
+#endif /* __DETECT_HTTP_HEADER_H__ */

src/detect.c

@@ -114,6 +114,7 @@
 #include "detect-http-header.h"
 #include "detect-http-uri.h"
 #include "detect-http-stat-msg.h"
+#include "detect-engine-hcbd.h"
 
 #include "util-rule-vars.h"
 
@@ -1309,16 +1310,7 @@ end:
     PacketPatternCleanup(th_v, det_ctx);
         //}
 
-    if (det_ctx->hcbd_buffers_list_len != 0) {
+    DetectEngineCleanHCBDBuffers(det_ctx);
-        int i;
-        for (i = 0; i < det_ctx->hcbd_buffers_list_len; i++) {
-            if (det_ctx->hcbd_buffers[i] != NULL)
-                SCFree(det_ctx->hcbd_buffers[i]);
-        }
-        SCFree(det_ctx->hcbd_buffers);
-        det_ctx->hcbd_buffers = NULL;
-        det_ctx->hcbd_buffers_list_len = 0;
-    }
 
     /* store the found sgh (or NULL) in the flow to save us from looking it
      * up again for the next packet. Also return any stream chunk we processed

src/detect.h

@@ -609,6 +609,8 @@ typedef struct DetectEngineCtx_ {
     /* maximum recursion depth for content inspection */
     int inspection_recursion_limit;
 
+    int hcbd_buffer_limit;
+
     /* array containing all sgh's in use so we can loop
      * through it in Stage4. */
     struct SigGroupHead_ **sgh_array;

suricata.yaml

@@ -110,6 +110,9 @@ engine-analysis:
 # might end up taking too much time in the content inspection code.
 # If the argument specified is 0, the engine uses an internally defined
 # default limit.  On not specifying a value, we use no limits on the recursion.
+#
+# The option http-client-body-buffer-limit, is used to specify the buffer
+# limit for http request body we inspect.
 detect-engine:
   - profile: medium
   - custom-values:
@@ -123,6 +126,7 @@ detect-engine:
       toserver_dp_groups: 25
   - sgh-mpm-context: auto
   - inspection-recursion-limit: 3000
+  - http-client-request-body-buffer-limit: 20000
 
 # Suricata is multi-threaded. Here the threading can be influenced.
 threading: