mpm: allocate StateQueue on the heap

So that we can have multi-tenant on MacOS without stack
overflows because of the size of the structure...

Ticket: #6263.
(cherry picked from commit 92fce2fdc0)

src/util-mpm-ac.c

@@ -59,6 +59,7 @@
 #include "util-memcmp.h"
 #include "util-mpm-ac.h"
 #include "util-memcpy.h"
+#include "util-validate.h"
 
 void SCACInitCtx(MpmCtx *);
 void SCACInitThreadCtx(MpmCtx *, MpmThreadCtx *);
@@ -353,7 +354,7 @@ static inline void SCACDetermineLevel1Gap(MpmCtx *mpm_ctx)
     SCACCtx *ctx = (SCACCtx *)mpm_ctx->ctx;
     uint32_t u = 0;
 
-    int map[256];
+    uint8_t map[256];
     memset(map, 0, sizeof(map));
 
     for (u = 0; u < mpm_ctx->pattern_cnt; u++)
@@ -506,8 +507,10 @@ static inline void SCACCreateFailureTable(MpmCtx *mpm_ctx)
     int32_t state = 0;
     int32_t r_state = 0;
 
-    StateQueue q;
-    memset(&q, 0, sizeof(StateQueue));
+    StateQueue *q = SCCalloc(1, sizeof(StateQueue));
+    if (q == NULL) {
+        FatalError(SC_ERR_FATAL, "Error allocating memory");
+    }
 
     /* allot space for the failure table.  A failure entry in the table for
      * every state(SCACCtx->state_count) */
@@ -523,19 +526,19 @@ static inline void SCACCreateFailureTable(MpmCtx *mpm_ctx)
     for (ascii_code = 0; ascii_code < 256; ascii_code++) {
         int32_t temp_state = ctx->goto_table[0][ascii_code];
         if (temp_state != 0) {
-            SCACEnqueue(&q, temp_state);
+            SCACEnqueue(q, temp_state);
             ctx->failure_table[temp_state] = 0;
         }
     }
 
-    while (!SCACStateQueueIsEmpty(&q)) {
+    while (!SCACStateQueueIsEmpty(q)) {
         /* pick up every state from the queue and add failure transitions */
-        r_state = SCACDequeue(&q);
+        r_state = SCACDequeue(q);
         for (ascii_code = 0; ascii_code < 256; ascii_code++) {
             int32_t temp_state = ctx->goto_table[r_state][ascii_code];
             if (temp_state == SC_AC_FAIL)
                 continue;
-            SCACEnqueue(&q, temp_state);
+            SCACEnqueue(q, temp_state);
             state = ctx->failure_table[r_state];
 
             while(ctx->goto_table[state][ascii_code] == SC_AC_FAIL)
@@ -545,6 +548,7 @@ static inline void SCACCreateFailureTable(MpmCtx *mpm_ctx)
                                  mpm_ctx);
         }
     }
+    SCFree(q);
 
     return;
 }
@@ -569,30 +573,34 @@ static inline void SCACCreateDeltaTable(MpmCtx *mpm_ctx)
         mpm_ctx->memory_cnt++;
         mpm_ctx->memory_size += (ctx->state_count * sizeof(*ctx->state_table_u16));
 
-        StateQueue q;
-        memset(&q, 0, sizeof(StateQueue));
+        StateQueue *q = SCCalloc(1, sizeof(StateQueue));
+        if (q == NULL) {
+            FatalError(SC_ERR_FATAL, "Error allocating memory");
+        }
 
         for (ascii_code = 0; ascii_code < 256; ascii_code++) {
             SC_AC_STATE_TYPE_U16 temp_state = ctx->goto_table[0][ascii_code];
             ctx->state_table_u16[0][ascii_code] = temp_state;
             if (temp_state != 0)
-                SCACEnqueue(&q, temp_state);
+                SCACEnqueue(q, temp_state);
         }
 
-        while (!SCACStateQueueIsEmpty(&q)) {
-            r_state = SCACDequeue(&q);
+        while (!SCACStateQueueIsEmpty(q)) {
+            r_state = SCACDequeue(q);
 
             for (ascii_code = 0; ascii_code < 256; ascii_code++) {
                 int32_t temp_state = ctx->goto_table[r_state][ascii_code];
                 if (temp_state != SC_AC_FAIL) {
-                    SCACEnqueue(&q, temp_state);
-                    ctx->state_table_u16[r_state][ascii_code] = temp_state;
+                    SCACEnqueue(q, temp_state);
+                    DEBUG_VALIDATE_BUG_ON(temp_state > UINT16_MAX);
+                    ctx->state_table_u16[r_state][ascii_code] = (uint16_t)temp_state;
                 } else {
                     ctx->state_table_u16[r_state][ascii_code] =
                         ctx->state_table_u16[ctx->failure_table[r_state]][ascii_code];
                 }
             }
         }
+        SCFree(q);
     }
 
     if (!(ctx->state_count < 32767) || construct_both_16_and_32_state_tables) {
@@ -606,23 +614,25 @@ static inline void SCACCreateDeltaTable(MpmCtx *mpm_ctx)
         mpm_ctx->memory_cnt++;
         mpm_ctx->memory_size += (ctx->state_count * sizeof(*ctx->state_table_u32));
 
-        StateQueue q;
-        memset(&q, 0, sizeof(StateQueue));
+        StateQueue *q = SCCalloc(1, sizeof(StateQueue));
+        if (q == NULL) {
+            FatalError(SC_ERR_FATAL, "Error allocating memory");
+        }
 
         for (ascii_code = 0; ascii_code < 256; ascii_code++) {
             SC_AC_STATE_TYPE_U32 temp_state = ctx->goto_table[0][ascii_code];
             ctx->state_table_u32[0][ascii_code] = temp_state;
             if (temp_state != 0)
-                SCACEnqueue(&q, temp_state);
+                SCACEnqueue(q, temp_state);
         }
 
-        while (!SCACStateQueueIsEmpty(&q)) {
-            r_state = SCACDequeue(&q);
+        while (!SCACStateQueueIsEmpty(q)) {
+            r_state = SCACDequeue(q);
 
             for (ascii_code = 0; ascii_code < 256; ascii_code++) {
                 int32_t temp_state = ctx->goto_table[r_state][ascii_code];
                 if (temp_state != SC_AC_FAIL) {
-                    SCACEnqueue(&q, temp_state);
+                    SCACEnqueue(q, temp_state);
                     ctx->state_table_u32[r_state][ascii_code] = temp_state;
                 } else {
                     ctx->state_table_u32[r_state][ascii_code] =
@@ -630,6 +640,7 @@ static inline void SCACCreateDeltaTable(MpmCtx *mpm_ctx)
                 }
             }
         }
+        SCFree(q);
     }
 
     return;