detect: avoids memory leak on ja3 signature parsing

If a later keyword enforces a protocol incompatible with ja3

src/detect-parse.c

@@ -1912,6 +1912,12 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
     }
 #endif
 
+    if (s->init_data->init_flags & SIG_FLAG_INIT_JA3 && s->alproto != ALPROTO_UNKNOWN &&
+            s->alproto != ALPROTO_TLS && s->alproto != ALPROTO_QUIC) {
+        SCLogError(SC_ERR_INVALID_SIGNATURE, "Cannot have ja3 with protocol %s.",
+                AppProtoToString(s->alproto));
+        SCReturnInt(0);
+    }
     if ((s->flags & SIG_FLAG_FILESTORE) || s->file_flags != 0 ||
         (s->init_data->init_flags & SIG_FLAG_INIT_FILEDATA)) {
         if (s->alproto != ALPROTO_UNKNOWN &&

src/detect-tls-ja3-hash.c

@@ -135,6 +135,7 @@ static int DetectTlsJa3HashSetup(DetectEngineCtx *de_ctx, Signature *s, const ch
         }
         return -2;
     }
+    s->init_data->init_flags |= SIG_FLAG_INIT_JA3;
 
     return 0;
 }

src/detect-tls-ja3-string.c

@@ -124,6 +124,7 @@ static int DetectTlsJa3StringSetup(DetectEngineCtx *de_ctx, Signature *s, const
         }
         return -2;
     }
+    s->init_data->init_flags |= SIG_FLAG_INIT_JA3;
 
     return 0;
 }

src/detect-tls-ja3s-hash.c

@@ -133,6 +133,7 @@ static int DetectTlsJa3SHashSetup(DetectEngineCtx *de_ctx, Signature *s, const c
         }
         return -2;
     }
+    s->init_data->init_flags |= SIG_FLAG_INIT_JA3;
 
     return 0;
 }

src/detect-tls-ja3s-string.c

@@ -124,6 +124,7 @@ static int DetectTlsJa3SStringSetup(DetectEngineCtx *de_ctx, Signature *s, const
         }
         return -2;
     }
+    s->init_data->init_flags |= SIG_FLAG_INIT_JA3;
 
     return 0;
 }

src/detect.h

@@ -255,6 +255,7 @@ typedef struct DetectPort_ {
 #define SIG_FLAG_INIT_NEED_FLUSH            BIT_U32(7)
 #define SIG_FLAG_INIT_PRIO_EXPLICT          BIT_U32(8)  /**< priority is explicitly set by the priority keyword */
 #define SIG_FLAG_INIT_FILEDATA              BIT_U32(9)  /**< signature has filedata keyword */
+#define SIG_FLAG_INIT_JA3                   BIT_U32(10) /**< signature has ja3 keyword */
 
 /* signature mask flags */
 /** \note: additions should be added to the rule analyzer as well */