detect/dataset: error if set couldn't be fully loaded

src/detect-dataset.c

@@ -377,6 +377,10 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
                 "failed to set up dataset '%s'.", name);
         return -1;
     }
+    if (set->hash && SC_ATOMIC_GET(set->hash->memcap_reached)) {
+        SCLogError(SC_ERR_THASH_INIT, "dataset too large for set memcap");
+        return -1;
+    }
 
     cd = SCCalloc(1, sizeof(DetectDatasetData));
     if (unlikely(cd == NULL))

src/util-thash.c

@@ -473,6 +473,10 @@ static THashData *THashDataGetNew(THashTableContext *ctx, void *data)
                 return NULL;
             }
 
+            if (!SC_ATOMIC_GET(ctx->memcap_reached)) {
+                SC_ATOMIC_SET(ctx->memcap_reached, true);
+            }
+
             /* freed data, but it's unlocked */
         } else {
             /* now see if we can alloc a new data */

src/util-thash.h

@@ -152,6 +152,8 @@ typedef struct THashTableContext_ {
 
     THashConfig config;
 
+    /* flag set if memcap was reached at least once. */
+    SC_ATOMIC_DECLARE(bool, memcap_reached);
 } THashTableContext;
 
 /** \brief check if a memory alloc would fit in the memcap