detect: errors on 65k filestore signatures

Errors when a detection engine gets 65k filestore signatures to
avoid the hard limit to have 65k filestore per signature group
head

Ticket: #6393

src/detect-engine-siggroup.c

@@ -48,6 +48,7 @@
 
 #include "util-error.h"
 #include "util-debug.h"
+#include "util-validate.h"
 #include "util-cidr.h"
 #include "util-unittest.h"
 #include "util-unittest-helper.h"
@@ -552,6 +553,8 @@ void SigGroupHeadSetupFiles(const DetectEngineCtx *de_ctx, SigGroupHead *sgh)
         }
 #endif
         if (SignatureIsFilestoring(s)) {
+            // should be insured by caller that we do not overflow
+            DEBUG_VALIDATE_BUG_ON(sgh->filestore_cnt == UINT16_MAX);
             sgh->filestore_cnt++;
         }
     }

src/detect-filestore.c

@@ -333,6 +333,11 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const ch
     static bool warn_not_configured = false;
     static uint32_t de_version = 0;
 
+    if (de_ctx->filestore_cnt == UINT16_MAX) {
+        SCLogError("Cannot have more than 65535 filestore signatures");
+        return -1;
+    }
+
     /* Check on first-time loads (includes following a reload) */
     if (!warn_not_configured || (de_ctx->version != de_version)) {
         if (de_version != de_ctx->version) {
@@ -466,6 +471,7 @@ static int DetectFilestoreSetup (DetectEngineCtx *de_ctx, Signature *s, const ch
     }
 
     s->flags |= SIG_FLAG_FILESTORE;
+    de_ctx->filestore_cnt++;
 
     if (match)
         pcre2_match_data_free(match);

src/detect.h

@@ -1040,6 +1040,9 @@ typedef struct DetectEngineCtx_ {
 
     /* Track rule requirements for reporting after loading rules. */
     SCDetectRequiresStatus *requirements;
+
+    /* number of signatures using filestore, limited as u16 */
+    uint16_t filestore_cnt;
 } DetectEngineCtx;
 
 /* Engine groups profiles (low, medium, high, custom) */