detect: add debug validation checks to assert assumptions

src/detect-engine-prefilter.c

@@ -112,6 +112,12 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx,
         SCLogDebug("%" PRIu64 ": engine %p for %s progress %u  sub_state %u tx %u",
                 PcapPacketCntGet(p), engine, AppProtoToString(engine->alproto),
                 engine->ctx.app.tx_min_progress, engine->ctx.app.sub_state, tx->tx_type);
+
+        DEBUG_VALIDATE_BUG_ON(
+                AppLayerParserSupportsSubStates(engine->alproto) && engine->ctx.app.sub_state == 0);
+        DEBUG_VALIDATE_BUG_ON(!AppLayerParserSupportsSubStates(engine->alproto) &&
+                              engine->ctx.app.sub_state != 0);
+
         if (engine->alproto != ALPROTO_UNKNOWN && engine->ctx.app.sub_state != tx->tx_type) {
             SCLogDebug("%" PRIu64 ": engine %p sub_state %u mismatch with tx %u",
                     PcapPacketCntGet(p), engine, engine->ctx.app.sub_state, tx->tx_type);

src/detect.c

@@ -1349,6 +1349,11 @@ static int DetectRunTxInspectRule(ThreadVars *tv, DetectEngineCtx *de_ctx,
         if (!(inspect_flags & BIT_U32(engine->id)) &&
                 (direction == engine->dir || ((s->flags & SIG_FLAG_TXBOTHDIR) && direction == 1))) {
 
+            DEBUG_VALIDATE_BUG_ON(
+                    AppLayerParserSupportsSubStates(engine->alproto) && engine->sub_state == 0);
+            DEBUG_VALIDATE_BUG_ON(
+                    !AppLayerParserSupportsSubStates(engine->alproto) && engine->sub_state != 0);
+
             if (engine->alproto != ALPROTO_UNKNOWN && // app-layer-events is registered for each
                                                       // proto this way
                     tx->tx_type != engine->sub_state) {