detect-filemagic: fix heap-use-after-free

This fixes the heap-use-after-free issue with sm being freed without
being removed from the signature (s) list. Move the protocol check for
rules with filemagic before the alloc and make the error log more
precise.

src/detect-filemagic.c

@@ -338,6 +338,11 @@ static int DetectFilemagicSetup (DetectEngineCtx *de_ctx, Signature *s, char *st
     DetectFilemagicData *filemagic = NULL;
     SigMatch *sm = NULL;
 
+    if (s->alproto != ALPROTO_HTTP && s->alproto != ALPROTO_SMTP) {
+        SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rules with filemagic need to have protocol set to http or smtp.");
+        goto error;
+    }
+
     filemagic = DetectFilemagicParse(str);
     if (filemagic == NULL)
         goto error;
@@ -359,11 +364,6 @@ static int DetectFilemagicSetup (DetectEngineCtx *de_ctx, Signature *s, char *st
 
     SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH);
 
-    if (s->alproto != ALPROTO_HTTP && s->alproto != ALPROTO_SMTP) {
-        SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
-        goto error;
-    }
-
     if (s->alproto == ALPROTO_HTTP) {
         AppLayerHtpNeedFileInspection();
     }