detect file: enforce protocol in single place

Instead of trying to enforce the app layer protocol in each file
function, enforce it in the generic validation function.

src/detect-fileext.c

@@ -209,11 +209,6 @@ static int DetectFileextSetup (DetectEngineCtx *de_ctx, Signature *s, char *str)
     sm->type = DETECT_FILEEXT;
     sm->ctx = (void *)fileext;
 
-    if (s->alproto != ALPROTO_HTTP && s->alproto != ALPROTO_SMTP) {
-        SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
-        goto error;
-    }
-
     SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH);
 
     if (s->alproto == ALPROTO_HTTP) {

src/detect-filemagic.c

@@ -338,11 +338,6 @@ static int DetectFilemagicSetup (DetectEngineCtx *de_ctx, Signature *s, char *st
     DetectFilemagicData *filemagic = NULL;
     SigMatch *sm = NULL;
 
-    if (s->alproto != ALPROTO_HTTP && s->alproto != ALPROTO_SMTP) {
-        SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rules with filemagic need to have protocol set to http or smtp.");
-        goto error;
-    }
-
     filemagic = DetectFilemagicParse(str);
     if (filemagic == NULL)
         goto error;

src/detect-filemd5.c

@@ -322,11 +322,6 @@ static int DetectFileMd5Setup (DetectEngineCtx *de_ctx, Signature *s, char *str)
     sm->type = DETECT_FILEMD5;
     sm->ctx = (void *)filemd5;
 
-    if (s->alproto != ALPROTO_HTTP && s->alproto != ALPROTO_SMTP) {
-        SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
-        goto error;
-    }
-
     SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH);
 
     if (s->alproto == ALPROTO_HTTP) {

src/detect-filename.c

@@ -49,6 +49,7 @@
 #include "stream-tcp.h"
 
 #include "detect-filename.h"
+#include "app-layer-parser.h"
 
 static int DetectFilenameMatch (ThreadVars *, DetectEngineThreadCtx *, Flow *,
         uint8_t, File *, Signature *, SigMatch *);
@@ -213,11 +214,6 @@ static int DetectFilenameSetup (DetectEngineCtx *de_ctx, Signature *s, char *str
     sm->type = DETECT_FILENAME;
     sm->ctx = (void *)filename;
 
-    if (s->alproto != ALPROTO_HTTP && s->alproto != ALPROTO_SMTP) {
-        SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
-        goto error;
-    }
-
     SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH);
 
     if (s->alproto == ALPROTO_HTTP) {

src/detect-filesize.c

@@ -287,11 +287,6 @@ static int DetectFilesizeSetup (DetectEngineCtx *de_ctx, Signature *s, char *str
     sm->type = DETECT_FILESIZE;
     sm->ctx = (SigMatchCtx *)fsd;
 
-    if (s->alproto != ALPROTO_HTTP && s->alproto != ALPROTO_SMTP) {
-        SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
-        goto error;
-    }
-
     SigMatchAppendSMToList(s, sm, DETECT_SM_LIST_FILEMATCH);
 
     if (s->alproto == ALPROTO_HTTP) {

src/detect-filestore.c

@@ -387,11 +387,6 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, char *st
         sm->ctx = (SigMatchCtx*)NULL;
     }
 
-    if (s->alproto != ALPROTO_HTTP && s->alproto != ALPROTO_SMTP) {
-        SCLogError(SC_ERR_CONFLICTING_RULE_KEYWORDS, "rule contains conflicting keywords.");
-        goto error;
-    }
-
     if (s->alproto == ALPROTO_HTTP) {
         AppLayerHtpNeedFileInspection();
     }

src/detect-parse.c

@@ -1421,6 +1421,15 @@ int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
     }
 #endif
 
+    if (((s->flags & SIG_FLAG_FILESTORE) || s->file_flags != 0) &&
+        s->alproto != ALPROTO_UNKNOWN &&
+        !AppLayerParserSupportsFiles(IPPROTO_TCP, s->alproto))
+    {
+        SCLogError(SC_ERR_NO_FILES_FOR_PROTOCOL, "protocol %s doesn't "
+                "support file matching", AppProtoToString(s->alproto));
+        SCReturnInt(0);
+    }
+
     SCReturnInt(1);
 }
 

src/util-error.c

@@ -325,6 +325,7 @@ const char * SCErrorToString(SCError err)
         CASE_CODE (SC_ERR_SMTP_LOG_GENERIC);
         CASE_CODE (SC_ERR_SSH_LOG_GENERIC);
         CASE_CODE (SC_ERR_NIC_OFFLOADING);
+        CASE_CODE (SC_ERR_NO_FILES_FOR_PROTOCOL);
     }
 
     return "UNKNOWN_ERROR";

src/util-error.h

@@ -315,6 +315,7 @@ typedef enum {
     SC_ERR_SMTP_LOG_GENERIC,
     SC_ERR_SSH_LOG_GENERIC,
     SC_ERR_NIC_OFFLOADING,
+    SC_ERR_NO_FILES_FOR_PROTOCOL,
 } SCError;
 
 const char *SCErrorToString(SCError);