detect: enforce max app-layer progress

Allow progress values in the range 0-47 so we have 48 bits to track
prefilter engines.

Mark bits 48-62 as reserved explicitly.

Add debug validation checks to make sure the reserved space isn't used.

src/app-layer-parser.h

@@ -49,11 +49,37 @@
 
 /* applies to DetectFlags uint64_t field */
 
+/** reserved for future use */
+#define APP_LAYER_TX_RESERVED1_FLAG  BIT_U64(48)
+#define APP_LAYER_TX_RESERVED2_FLAG  BIT_U64(49)
+#define APP_LAYER_TX_RESERVED3_FLAG  BIT_U64(50)
+#define APP_LAYER_TX_RESERVED4_FLAG  BIT_U64(51)
+#define APP_LAYER_TX_RESERVED5_FLAG  BIT_U64(52)
+#define APP_LAYER_TX_RESERVED6_FLAG  BIT_U64(53)
+#define APP_LAYER_TX_RESERVED7_FLAG  BIT_U64(54)
+#define APP_LAYER_TX_RESERVED8_FLAG  BIT_U64(55)
+#define APP_LAYER_TX_RESERVED9_FLAG  BIT_U64(56)
+#define APP_LAYER_TX_RESERVED10_FLAG BIT_U64(57)
+#define APP_LAYER_TX_RESERVED11_FLAG BIT_U64(58)
+#define APP_LAYER_TX_RESERVED12_FLAG BIT_U64(59)
+#define APP_LAYER_TX_RESERVED13_FLAG BIT_U64(60)
+#define APP_LAYER_TX_RESERVED14_FLAG BIT_U64(61)
+#define APP_LAYER_TX_RESERVED15_FLAG BIT_U64(62)
+
+#define APP_LAYER_TX_RESERVED_FLAGS                                                                \
+    (APP_LAYER_TX_RESERVED1_FLAG | APP_LAYER_TX_RESERVED2_FLAG | APP_LAYER_TX_RESERVED3_FLAG |     \
+            APP_LAYER_TX_RESERVED4_FLAG | APP_LAYER_TX_RESERVED5_FLAG |                            \
+            APP_LAYER_TX_RESERVED6_FLAG | APP_LAYER_TX_RESERVED7_FLAG |                            \
+            APP_LAYER_TX_RESERVED8_FLAG | APP_LAYER_TX_RESERVED9_FLAG |                            \
+            APP_LAYER_TX_RESERVED10_FLAG | APP_LAYER_TX_RESERVED11_FLAG |                          \
+            APP_LAYER_TX_RESERVED12_FLAG | APP_LAYER_TX_RESERVED13_FLAG |                          \
+            APP_LAYER_TX_RESERVED14_FLAG | APP_LAYER_TX_RESERVED15_FLAG)
+
 /** is tx fully inspected? */
 #define APP_LAYER_TX_INSPECTED_FLAG             BIT_U64(63)
 /** other 63 bits are for tracking which prefilter engine is already
  *  completely inspected */
-#define APP_LAYER_TX_PREFILTER_MASK             ~APP_LAYER_TX_INSPECTED_FLAG
+#define APP_LAYER_TX_PREFILTER_MASK ~(APP_LAYER_TX_INSPECTED_FLAG | APP_LAYER_TX_RESERVED_FLAGS)
 
 /** parser has successfully processed in the input, and has consumed
  *  all of it. */

src/detect-engine-mpm.c

@@ -97,6 +97,8 @@ void DetectAppLayerMpmRegister2(const char *name,
     SCLogDebug("registering %s/%d/%d/%p/%p/%u/%d", name, direction, priority,
             PrefilterRegister, GetData, alproto, tx_min_progress);
 
+    BUG_ON(tx_min_progress >= 48);
+
     if (PrefilterRegister == PrefilterGenericMpmRegister && GetData == NULL) {
         // must register GetData with PrefilterGenericMpmRegister
         abort();

src/detect-engine.c

@@ -179,6 +179,8 @@ void DetectAppLayerInspectEngineRegister2(const char *name,
         InspectEngineFuncPtr2 Callback2,
         InspectionBufferGetDataPtr GetData)
 {
+    BUG_ON(progress >= 48);
+
     DetectBufferTypeRegister(name);
     const int sm_list = DetectBufferTypeGetByName(name);
     if (sm_list == -1) {

src/detect.c

@@ -1238,6 +1238,7 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro
     DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(ipproto, alproto, tx_ptr);
     DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL;
     uint64_t prefilter_flags = detect_flags & APP_LAYER_TX_PREFILTER_MASK;
+    DEBUG_VALIDATE_BUG_ON(prefilter_flags & APP_LAYER_TX_RESERVED_FLAGS);
 
     DetectTransaction tx = {
                             .tx_ptr = tx_ptr,
@@ -1491,6 +1492,7 @@ static void DetectRunTx(ThreadVars *tv,
         }
         if (tx.prefilter_flags != tx.prefilter_flags_orig) {
             new_detect_flags |= tx.prefilter_flags;
+            DEBUG_VALIDATE_BUG_ON(new_detect_flags & APP_LAYER_TX_RESERVED_FLAGS);
             SCLogDebug("%p/%"PRIu64" updated prefilter flags %016"PRIx64" "
                     "(was: %016"PRIx64") for direction %s. Flag %016"PRIx64,
                     tx.tx_ptr, tx.tx_id, tx.prefilter_flags, tx.prefilter_flags_orig,
@@ -1501,6 +1503,7 @@ static void DetectRunTx(ThreadVars *tv,
                 (new_detect_flags | tx.detect_flags) != tx.detect_flags)
         {
             new_detect_flags |= tx.detect_flags;
+            DEBUG_VALIDATE_BUG_ON(new_detect_flags & APP_LAYER_TX_RESERVED_FLAGS);
             SCLogDebug("%p/%"PRIu64" Storing new flags %016"PRIx64" (was %016"PRIx64")",
                     tx.tx_ptr, tx.tx_id, new_detect_flags, tx.detect_flags);