detect/frames: limit mixing frames and other detection

Don't allow mixing of payload/stream/tx and frame keywords. Initial
support is only for 'pure' frame inspection.

src/detect-parse.c

@@ -1778,6 +1778,37 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
         SCReturnInt(0);
     }
 
+    bool has_pmatch = false;
+    bool has_frame = false;
+    bool has_app = false;
+    bool has_pkt = false;
+
+    for (int i = 0; i < nlists; i++) {
+        if (s->init_data->smlists[i] == NULL)
+            continue;
+        has_pmatch |= (i == DETECT_SM_LIST_PMATCH);
+
+        const DetectBufferType *b = DetectEngineBufferTypeGetById(de_ctx, i);
+        if (b == NULL)
+            continue;
+
+        has_frame |= b->frame;
+        has_app |= (b->frame == false && b->packet == false);
+        has_pkt |= b->packet;
+    }
+    if (has_pmatch && has_frame) {
+        SCLogError(SC_ERR_INVALID_SIGNATURE, "can't mix pure content and frame inspection");
+        SCReturnInt(0);
+    }
+    if (has_app && has_frame) {
+        SCLogError(SC_ERR_INVALID_SIGNATURE, "can't app-layer buffer and frame inspection");
+        SCReturnInt(0);
+    }
+    if (has_pkt && has_frame) {
+        SCLogError(SC_ERR_INVALID_SIGNATURE, "can't pkt buffer and frame inspection");
+        SCReturnInt(0);
+    }
+
     if (s->flags & SIG_FLAG_REQUIRE_PACKET) {
         for (int i = 0; i < nlists; i++) {
             if (s->init_data->smlists[i] == NULL)