Improve out of memory handling during initialization.

src/detect.c

@@ -1435,8 +1435,8 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) {
                "adding signatures to signature source addresses... done");
     }
     return 0;
+
 error:
-    printf("SigAddressPrepareStage1 error\n");
     return -1;
 }
 
@@ -3133,8 +3133,14 @@ int SigAddressPrepareStage5(DetectEngineCtx *de_ctx) {
  * \retval 0 Always
  */
 int SigGroupBuild (DetectEngineCtx *de_ctx) {
-    SigAddressPrepareStage1(de_ctx);
+    if (SigAddressPrepareStage1(de_ctx) != 0) {
-    SigAddressPrepareStage2(de_ctx);
+        SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
+        exit(EXIT_FAILURE);
+    }
+    if (SigAddressPrepareStage2(de_ctx) != 0) {
+        SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
+        exit(EXIT_FAILURE);
+    }
 
 #ifdef __SC_CUDA_SUPPORT__
     unsigned int cuda_total = 0;
@@ -3161,8 +3167,14 @@ int SigGroupBuild (DetectEngineCtx *de_ctx) {
 
 #endif
 
-    SigAddressPrepareStage3(de_ctx);
+    if (SigAddressPrepareStage3(de_ctx) != 0) {
-    SigAddressPrepareStage4(de_ctx);
+        SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
+        exit(EXIT_FAILURE);
+    }
+    if (SigAddressPrepareStage4(de_ctx) != 0) {
+        SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed");
+        exit(EXIT_FAILURE);
+    }
 
 #ifdef __SC_CUDA_SUPPORT__
     unsigned int cuda_free_after_alloc = 0;

src/util-error.h

@@ -200,6 +200,7 @@ typedef enum {
     SC_ERR_DAG_NOSUPPORT,           /**< no ERF/DAG support compiled in */
     SC_ERR_FATAL,
     SC_ERR_DCERPC,
+    SC_ERR_DETECT_PREPARE,          /**< preparing the detection engine failed */
 } SCError;
 
 const char *SCErrorToString(SCError);

src/util-mpm-b2g.c

@@ -739,8 +739,11 @@ void B2gInitCtx (MpmCtx *mpm_ctx, int module_handle) {
     BUG_ON(mpm_ctx->ctx != NULL);
 
     mpm_ctx->ctx = SCMalloc(sizeof(B2gCtx));
-    if (mpm_ctx->ctx == NULL)
+    if (mpm_ctx->ctx == NULL) {
-        return;
+        SCLogError(SC_ERR_MEM_ALLOC, "SCMalloc failed: %s, while trying "
+            "to allocate %"PRIdMAX" bytes", strerror(errno), (intmax_t)(sizeof(B2gCtx)));
+        exit(EXIT_FAILURE);
+    }
 
     memset(mpm_ctx->ctx, 0, sizeof(B2gCtx));
 
@@ -750,8 +753,11 @@ void B2gInitCtx (MpmCtx *mpm_ctx, int module_handle) {
     /* initialize the hash we use to speed up pattern insertions */
     B2gCtx *ctx = (B2gCtx *)mpm_ctx->ctx;
     ctx->init_hash = SCMalloc(sizeof(B2gPattern *) * INIT_HASH_SIZE);
-    if (ctx->init_hash == NULL)
+    if (ctx->init_hash == NULL) {
-        return;
+        SCLogError(SC_ERR_MEM_ALLOC, "SCMalloc failed: %s, while trying "
+            "to allocate %"PRIdMAX" bytes", strerror(errno), (intmax_t)(sizeof(B2gPattern *) * INIT_HASH_SIZE));
+        exit(EXIT_FAILURE);
+    }
 
     memset(ctx->init_hash, 0, sizeof(B2gPattern *) * INIT_HASH_SIZE);
 
@@ -846,8 +852,11 @@ void B2gThreadInitCtx(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, uint32_t ma
 
     if (sizeof(B2gThreadCtx) > 0) { /* size can be null when optimized */
         mpm_thread_ctx->ctx = SCMalloc(sizeof(B2gThreadCtx));
-        if (mpm_thread_ctx->ctx == NULL)
+        if (mpm_thread_ctx->ctx == NULL) {
-            return;
+            SCLogError(SC_ERR_MEM_ALLOC, "SCMalloc failed: %s, while trying "
+                    "to allocate %"PRIdMAX" bytes", strerror(errno), (intmax_t)(sizeof(B2gThreadCtx)));
+            exit(EXIT_FAILURE);
+        }
 
         memset(mpm_thread_ctx->ctx, 0, sizeof(B2gThreadCtx));