detect/flowbits: fix stack overflow in analyzer

Fix stack overflow in DetectFlowbitsAnalyze.

Use dynamically allocated array instead of stack and free
it after it is no longer needed.

src/detect-engine-build.c

@@ -1418,7 +1418,9 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
         SCLogConfig("building signature grouping structure, stage 1: "
                "preprocessing rules... complete");
     }
-    DetectFlowbitsAnalyze(de_ctx);
+
+    if (DetectFlowbitsAnalyze(de_ctx) != 0)
+        goto error;
 
     return 0;
 

src/detect-flowbits.c

@@ -400,16 +400,20 @@ static void DetectFlowbitsAnalyzeDump(const DetectEngineCtx *de_ctx,
         struct FBAnalyze *array, uint32_t elements);
 #endif
 
-void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
+int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx)
 {
     const uint32_t max_fb_id = de_ctx->max_fb_id;
     if (max_fb_id == 0)
-        return;
+        return 0;
 
 #define MAX_SIDS 8
     uint32_t array_size = max_fb_id + 1;
-    struct FBAnalyze array[array_size];
+    struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze));
-    memset(&array, 0, array_size * sizeof(struct FBAnalyze));
+
+    if (array == NULL) {
+        SCLogError(SC_ERR_MEM_ALLOC, "Unable to allocate flowbit analyze array");
+        return -1;
+    }
 
     SCLogDebug("fb analyzer array size: %"PRIu64,
             (uint64_t)(array_size * sizeof(struct FBAnalyze)));
@@ -633,6 +637,9 @@ end:
         SCFree(array[i].isnotset_sids);
         SCFree(array[i].toggle_sids);
     }
+    SCFree(array);
+
+    return 0;
 }
 
 #ifdef PROFILING

src/detect.h

@@ -1494,7 +1494,7 @@ void DetectSignatureApplyActions(Packet *p, const Signature *s, const uint8_t);
 void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size);
 void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx);
 
-void DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx);
+int DetectFlowbitsAnalyze(DetectEngineCtx *de_ctx);
 
 int DetectMetadataHashInit(DetectEngineCtx *de_ctx);
 void DetectMetadataHashFree(DetectEngineCtx *de_ctx);