thresholds: use dedicated storage

Instead of a Host and IPPair table thresholding layer, use a dedicated
THash to store both. This allows hashing on host+sid+tracker or
ippair+sid+tracker, to create more unique hash keys.

This allows for fewer hash collisions.

The per rule tracking also uses this, so that the single big lock is no
longer a single point of contention.

Reimplement storage for flow thresholds to reuse as much logic as
possible from the host/ippair/rule thresholds.

Ticket: #426.

src/detect-detection-filter.c

@@ -378,7 +378,7 @@ static int DetectDetectionFilterTestSig1(void)
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -418,7 +418,7 @@ static int DetectDetectionFilterTestSig1(void)
     DetectEngineCtxFree(de_ctx);
 
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
 
     PASS;
 }
@@ -435,7 +435,7 @@ static int DetectDetectionFilterTestSig2(void)
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -480,7 +480,7 @@ static int DetectDetectionFilterTestSig2(void)
     DetectEngineCtxFree(de_ctx);
 
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
 
     PASS;
 }
@@ -493,7 +493,7 @@ static int DetectDetectionFilterTestSig3(void)
     ThreadVars th_v;
     DetectEngineThreadCtx *det_ctx;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -556,7 +556,7 @@ static int DetectDetectionFilterTestSig3(void)
     DetectEngineCtxFree(de_ctx);
 
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
 
     PASS;
 }

src/detect-engine-build.c

@@ -2219,8 +2219,6 @@ int SigGroupBuild(DetectEngineCtx *de_ctx)
     SCProfilingRuleInitCounters(de_ctx);
 #endif
 
-    ThresholdHashAllocate(de_ctx);
-
     if (!DetectEngineMultiTenantEnabled()) {
         VarNameStoreActivate();
     }

src/detect-engine-threshold.c

@@ -71,34 +71,150 @@
 #include "action-globals.h"
 #include "util-validate.h"
 
-static HostStorageId host_threshold_id = { .id = -1 };     /**< host storage id for thresholds */
-static IPPairStorageId ippair_threshold_id = { .id = -1 }; /**< ip pair storage id for thresholds */
+#include "util-thash.h"
+#include "util-hash-lookup3.h"
 
-HostStorageId ThresholdHostStorageId(void)
+struct Thresholds {
+    THashTableContext *thash;
+} ctx;
+
+static int ThresholdsInit(struct Thresholds *t);
+static void ThresholdsDestroy(struct Thresholds *t);
+
+void ThresholdInit(void)
 {
-    return host_threshold_id;
+    ThresholdsInit(&ctx);
 }
 
-void ThresholdInit(void)
+void ThresholdDestroy(void)
+{
+    ThresholdsDestroy(&ctx);
+}
+
+typedef struct ThresholdEntry_ {
+    uint32_t sid; /**< Signature id */
+    uint32_t gid; /**< Signature group id */
+    int track;    /**< Track type: by_src, by_dst, etc */
+
+    uint32_t tv_timeout;    /**< Timeout for new_action (for rate_filter)
+                                 its not "seconds", that define the time interval */
+    uint32_t seconds;       /**< Event seconds */
+    uint32_t current_count; /**< Var for count control */
+
+    SCTime_t tv1; /**< Var for time control */
+
+    Address addr;  /* used for src/dst/either tracking */
+    Address addr2; /* used for both tracking */
+} ThresholdEntry;
+
+static int ThresholdEntrySet(void *dst, void *src)
+{
+    const ThresholdEntry *esrc = src;
+    ThresholdEntry *edst = dst;
+    memset(edst, 0, sizeof(*edst));
+    *edst = *esrc;
+    return 0;
+}
+
+static void ThresholdEntryFree(void *ptr)
+{
+    // nothing to free, base data is part of hash
+}
+
+static inline uint32_t HashAddress(const Address *a)
+{
+    uint32_t key;
+
+    if (a->family == AF_INET) {
+        key = a->addr_data32[0];
+    } else if (a->family == AF_INET6) {
+        key = hashword(a->addr_data32, 4, 0);
+    } else
+        key = 0;
+
+    return key;
+}
+
+static inline int CompareAddress(const Address *a, const Address *b)
+{
+    if (a->family == b->family) {
+        switch (a->family) {
+            case AF_INET:
+                return (a->addr_data32[0] == b->addr_data32[0]);
+            case AF_INET6:
+                return CMP_ADDR(a, b);
+        }
+    }
+    return 0;
+}
+
+static uint32_t ThresholdEntryHash(void *ptr)
+{
+    const ThresholdEntry *e = ptr;
+    uint32_t hash = e->sid + e->gid + e->track;
+    switch (e->track) {
+        case TRACK_BOTH:
+            hash += HashAddress(&e->addr2);
+            /* fallthrough */
+        case TRACK_SRC:
+        case TRACK_DST:
+            hash += HashAddress(&e->addr);
+            break;
+    }
+    return hash;
+}
+
+static bool ThresholdEntryCompare(void *a, void *b)
 {
-    host_threshold_id = HostStorageRegister("threshold", sizeof(void *), NULL, ThresholdListFree);
-    if (host_threshold_id.id == -1) {
-        FatalError("Can't initiate host storage for thresholding");
+    const ThresholdEntry *e1 = a;
+    const ThresholdEntry *e2 = b;
+    SCLogDebug("sid1: %u sid2: %u", e1->sid, e2->sid);
+    if (!(e1->sid == e2->sid && e1->gid == e2->gid && e1->track == e2->track))
+        return false;
+    switch (e1->track) {
+        case TRACK_BOTH:
+            if (!(CompareAddress(&e1->addr2, &e2->addr2)))
+                return false;
+            /* fallthrough */
+        case TRACK_SRC:
+        case TRACK_DST:
+            if (!(CompareAddress(&e1->addr, &e2->addr)))
+                return false;
+            break;
     }
-    ippair_threshold_id = IPPairStorageRegister("threshold", sizeof(void *), NULL, ThresholdListFree);
-    if (ippair_threshold_id.id == -1) {
-        FatalError("Can't initiate IP pair storage for thresholding");
+    return true;
+}
+
+static bool ThresholdEntryExpire(void *data, const SCTime_t ts)
+{
+    const ThresholdEntry *e = data;
+    const SCTime_t entry = SCTIME_ADD_SECS(e->tv1, e->seconds);
+    if (SCTIME_CMP_GT(ts, entry)) {
+        return true;
     }
+    return false;
 }
 
-int ThresholdHostHasThreshold(Host *host)
+static int ThresholdsInit(struct Thresholds *t)
 {
-    return HostGetStorageById(host, host_threshold_id) ? 1 : 0;
+    uint64_t memcap = 16 * 1024 * 1024;
+    uint32_t hashsize = 16384;
+    t->thash = THashInit("thresholds", sizeof(ThresholdEntry), ThresholdEntrySet,
+            ThresholdEntryFree, ThresholdEntryHash, ThresholdEntryCompare, ThresholdEntryExpire, 0,
+            memcap, hashsize);
+    BUG_ON(t->thash == NULL);
+    return 0;
+}
+static void ThresholdsDestroy(struct Thresholds *t)
+{
+    if (t->thash) {
+        THashShutdown(t->thash);
+    }
 }
 
-int ThresholdIPPairHasThreshold(IPPair *pair)
+uint32_t ThresholdsExpire(const SCTime_t ts)
 {
-    return IPPairGetStorageById(pair, ippair_threshold_id) ? 1 : 0;
+    return THashExpire(ctx.thash, ts);
 }
 
 #include "util-hash.h"
@@ -234,7 +350,9 @@ static int SetupCache(const Packet *p, const int8_t track, const int8_t retval,
             n->expires_at = expires;
 
             if (HashTableAdd(threshold_cache_ht, n, 0) == 0) {
-                (void)THRESHOLD_CACHE_RB_INSERT(&threshold_cache_tree, n);
+                ThresholdCacheItem *r = THRESHOLD_CACHE_RB_INSERT(&threshold_cache_tree, n);
+                DEBUG_VALIDATE_BUG_ON(r != NULL); // duplicate; should be impossible
+                (void)r;                          // only used by DEBUG_VALIDATE_BUG_ON
                 return 1;
             }
             SCFree(n);
@@ -358,98 +476,18 @@ const DetectThresholdData *SigGetThresholdTypeIter(
     return NULL;
 }
 
-/**
- * \brief Remove timeout threshold hash elements
- *
- * \param head Current head element of storage
- * \param tv Current time
- *
- * \retval DetectThresholdEntry Return new head element or NULL if all expired
- *
- */
-
-static DetectThresholdEntry *ThresholdTimeoutCheck(DetectThresholdEntry *head, SCTime_t ts)
-{
-    DetectThresholdEntry *tmp = head;
-    DetectThresholdEntry *prev = NULL;
-    DetectThresholdEntry *new_head = head;
-
-    while (tmp != NULL) {
-        /* check if the 'check' timestamp is not before the creation ts.
-         * This can happen due to the async nature of the host timeout
-         * code that also calls this code from a management thread. */
-        SCTime_t entry = SCTIME_ADD_SECS(tmp->tv1, (time_t)tmp->seconds);
-        if (SCTIME_CMP_LTE(ts, entry)) {
-            prev = tmp;
-            tmp = tmp->next;
-            continue;
-        }
-
-        /* timed out */
-
-        DetectThresholdEntry *tde = tmp;
-        if (prev != NULL) {
-            prev->next = tmp->next;
-        }
-        else {
-            new_head = tmp->next;
-        }
-        tmp = tde->next;
-        SCFree(tde);
-    }
-
-    return new_head;
-}
-
-int ThresholdHostTimeoutCheck(Host *host, SCTime_t ts)
-{
-    DetectThresholdEntry* head = HostGetStorageById(host, host_threshold_id);
-    DetectThresholdEntry *new_head = ThresholdTimeoutCheck(head, ts);
-    if (new_head != head) {
-        HostSetStorageById(host, host_threshold_id, new_head);
-    }
-    return new_head == NULL;
-}
-
-int ThresholdIPPairTimeoutCheck(IPPair *pair, SCTime_t ts)
-{
-    DetectThresholdEntry* head = IPPairGetStorageById(pair, ippair_threshold_id);
-    DetectThresholdEntry *new_head = ThresholdTimeoutCheck(head, ts);
-    if (new_head != head) {
-        IPPairSetStorageById(pair, ippair_threshold_id, new_head);
-    }
-    return new_head == NULL;
-}
+typedef struct FlowThresholdEntryList_ {
+    struct FlowThresholdEntryList_ *next;
+    ThresholdEntry threshold;
+} FlowThresholdEntryList;
 
-static DetectThresholdEntry *DetectThresholdEntryAlloc(
-        const DetectThresholdData *td, uint32_t sid, uint32_t gid)
+static void FlowThresholdEntryListFree(FlowThresholdEntryList *list)
 {
-    SCEnter();
-
-    DetectThresholdEntry *ste = SCCalloc(1, sizeof(DetectThresholdEntry));
-    if (unlikely(ste == NULL)) {
-        SCReturnPtr(NULL, "DetectThresholdEntry");
+    for (FlowThresholdEntryList *i = list; i != NULL;) {
+        FlowThresholdEntryList *next = i->next;
+        SCFree(i);
+        i = next;
     }
-
-    ste->sid = sid;
-    ste->gid = gid;
-    ste->track = td->track;
-    ste->seconds = td->seconds;
-
-    SCReturnPtr(ste, "DetectThresholdEntry");
-}
-
-static DetectThresholdEntry *ThresholdHostLookupEntry(Host *h,
-        uint32_t sid, uint32_t gid)
-{
-    DetectThresholdEntry *e;
-
-    for (e = HostGetStorageById(h, host_threshold_id); e != NULL; e = e->next) {
-        if (e->sid == sid && e->gid == gid)
-            break;
-    }
-
-    return e;
 }
 
 /** struct for storing per flow thresholds. This will be stored in the Flow::flowvar list, so it
@@ -458,13 +496,13 @@ typedef struct FlowVarThreshold_ {
     uint8_t type;
     uint8_t pad[7];
     struct GenericVar_ *next;
-    DetectThresholdEntry *thresholds;
+    FlowThresholdEntryList *thresholds;
 } FlowVarThreshold;
 
 void FlowThresholdVarFree(void *ptr)
 {
     FlowVarThreshold *t = ptr;
-    ThresholdListFree(t->thresholds);
+    FlowThresholdEntryListFree(t->thresholds);
     SCFree(t);
 }
 
@@ -481,21 +519,21 @@ static FlowVarThreshold *FlowThresholdVarGet(Flow *f)
     return NULL;
 }
 
-static DetectThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid)
+static ThresholdEntry *ThresholdFlowLookupEntry(Flow *f, uint32_t sid, uint32_t gid)
 {
     FlowVarThreshold *t = FlowThresholdVarGet(f);
     if (t == NULL)
         return NULL;
 
-    for (DetectThresholdEntry *e = t->thresholds; e != NULL; e = e->next) {
-        if (e->sid == sid && e->gid == gid) {
-            return e;
+    for (FlowThresholdEntryList *e = t->thresholds; e != NULL; e = e->next) {
+        if (e->threshold.sid == sid && e->threshold.gid == gid) {
+            return &e->threshold;
         }
     }
     return NULL;
 }
 
-static int AddEntryToFlow(Flow *f, DetectThresholdEntry *e, SCTime_t packet_time)
+static int AddEntryToFlow(Flow *f, FlowThresholdEntryList *e, SCTime_t packet_time)
 {
     DEBUG_VALIDATE_BUG_ON(e == NULL);
 
@@ -509,27 +547,11 @@ static int AddEntryToFlow(Flow *f, DetectThresholdEntry *e, SCTime_t packet_time
         GenericVarAppend(&f->flowvar, (GenericVar *)t);
     }
 
-    e->current_count = 1;
-    e->tv1 = packet_time;
-    e->tv_timeout = 0;
     e->next = t->thresholds;
     t->thresholds = e;
     return 0;
 }
 
-static DetectThresholdEntry *ThresholdIPPairLookupEntry(IPPair *pair,
-        uint32_t sid, uint32_t gid)
-{
-    DetectThresholdEntry *e;
-
-    for (e = IPPairGetStorageById(pair, ippair_threshold_id); e != NULL; e = e->next) {
-        if (e->sid == sid && e->gid == gid)
-            break;
-    }
-
-    return e;
-}
-
 static int ThresholdHandlePacketSuppress(Packet *p,
         const DetectThresholdData *td, uint32_t sid, uint32_t gid)
 {
@@ -591,285 +613,208 @@ static inline void RateFilterSetAction(PacketAlert *pa, uint8_t new_action)
     }
 }
 
-/**
-* \brief Check if the entry reached threshold count limit
-*
-* \param lookup_tsh Current threshold entry
-* \param td Threshold settings
-* \param packet_time used to compare against previous detection and to set timeouts
-*
-* \retval int 1 if threshold reached for this entry
-*
-*/
-static int IsThresholdReached(
-        DetectThresholdEntry *lookup_tsh, const DetectThresholdData *td, SCTime_t packet_time)
+static int ThresholdSetup(const DetectThresholdData *td, ThresholdEntry *te,
+        const SCTime_t packet_time, const uint32_t sid, const uint32_t gid)
 {
-    int ret = 0;
-
-    /* Check if we have a timeout enabled, if so,
-    * we still matching (and enabling the new_action) */
-    if (lookup_tsh->tv_timeout != 0) {
-        if ((SCTIME_SECS(packet_time) - lookup_tsh->tv_timeout) > td->timeout) {
-            /* Ok, we are done, timeout reached */
-            lookup_tsh->tv_timeout = 0;
-        } else {
-            /* Already matching */
-            ret = 1;
-        } /* else - if ((packet_time - lookup_tsh->tv_timeout) > td->timeout) */
-
-    }
-    else {
-        /* Update the matching state with the timeout interval */
-        SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds);
-        if (SCTIME_CMP_LTE(packet_time, entry)) {
-            lookup_tsh->current_count++;
-            if (lookup_tsh->current_count > td->count) {
-                /* Then we must enable the new action by setting a
-                * timeout */
-                lookup_tsh->tv_timeout = SCTIME_SECS(packet_time);
-                ret = 1;
-            }
-        } else {
-            lookup_tsh->tv1 = packet_time;
-            lookup_tsh->current_count = 1;
-        }
-    } /* else - if (lookup_tsh->tv_timeout != 0) */
-
-    return ret;
-}
+    te->sid = sid;
+    te->gid = gid;
+    te->track = td->track;
+    te->seconds = td->seconds;
 
-static void AddEntryToHostStorage(Host *h, DetectThresholdEntry *e, SCTime_t packet_time)
-{
-    if (h && e) {
-        e->current_count = 1;
-        e->tv1 = packet_time;
-        e->tv_timeout = 0;
-        e->next = HostGetStorageById(h, host_threshold_id);
-        HostSetStorageById(h, host_threshold_id, e);
-    }
-}
+    te->current_count = 1;
+    te->tv1 = packet_time;
+    te->tv_timeout = 0;
 
-static void AddEntryToIPPairStorage(IPPair *pair, DetectThresholdEntry *e, SCTime_t packet_time)
-{
-    if (pair && e) {
-        e->current_count = 1;
-        e->tv1 = packet_time;
-        e->tv_timeout = 0;
-        e->next = IPPairGetStorageById(pair, ippair_threshold_id);
-        IPPairSetStorageById(pair, ippair_threshold_id, e);
+    switch (td->type) {
+        case TYPE_LIMIT:
+        case TYPE_RATE:
+            return 1;
+        case TYPE_THRESHOLD:
+        case TYPE_BOTH:
+            if (td->count == 1)
+                return 1;
+            return 0;
+        case TYPE_DETECTION:
+            return 0;
     }
+    return 0;
 }
 
-/**
- *  \retval 2 silent match (no alert but apply actions)
- *  \retval 1 normal match
- *  \retval 0 no match
- *
- *  If a new DetectThresholdEntry is generated to track the threshold
- *  for this rule, then it will be returned in new_tsh.
- */
-static int ThresholdHandlePacket(Packet *p, DetectThresholdEntry *lookup_tsh,
-        DetectThresholdEntry **new_tsh, const DetectThresholdData *td,
-        uint32_t sid, uint32_t gid, PacketAlert *pa)
+static int ThresholdCheckUpdate(const DetectThresholdData *td, ThresholdEntry *te,
+        const Packet *p, // ts only? - cache too
+        const uint32_t sid, PacketAlert *pa)
 {
     int ret = 0;
-
-    switch(td->type)   {
+    const SCTime_t packet_time = p->ts;
+    const SCTime_t entry = SCTIME_ADD_SECS(te->tv1, td->seconds);
+    switch (td->type) {
         case TYPE_LIMIT:
-        {
             SCLogDebug("limit");
 
-            if (lookup_tsh != NULL)  {
-                SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds);
-                if (SCTIME_CMP_LTE(p->ts, entry)) {
-                    lookup_tsh->current_count++;
-
-                    if (lookup_tsh->current_count <= td->count) {
-                        ret = 1;
-                    } else {
-                        ret = 2;
+            if (SCTIME_CMP_LTE(p->ts, entry)) {
+                te->current_count++;
 
-                        if (PacketIsIPv4(p)) {
-                            SetupCache(p, td->track, (int8_t)ret, sid, entry);
-                        }
-                    }
+                if (te->current_count <= td->count) {
+                    ret = 1;
                 } else {
-                    lookup_tsh->tv1 = p->ts;
-                    lookup_tsh->current_count = 1;
+                    ret = 2;
 
-                    ret = 1;
+                    if (PacketIsIPv4(p)) {
+                        SetupCache(p, td->track, (int8_t)ret, sid, entry);
+                    }
                 }
             } else {
-                *new_tsh = DetectThresholdEntryAlloc(td, sid, gid);
-
+                /* entry expired, reset */
+                te->tv1 = p->ts;
+                te->current_count = 1;
                 ret = 1;
             }
             break;
-        }
         case TYPE_THRESHOLD:
-        {
-            SCLogDebug("threshold");
-
-            if (lookup_tsh != NULL)  {
-                SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds);
-                if (SCTIME_CMP_LTE(p->ts, entry)) {
-                    lookup_tsh->current_count++;
+            if (SCTIME_CMP_LTE(p->ts, entry)) {
+                te->current_count++;
 
-                    if (lookup_tsh->current_count >= td->count) {
-                        ret = 1;
-                        lookup_tsh->current_count = 0;
-                    }
-                } else {
-                    lookup_tsh->tv1 = p->ts;
-                    lookup_tsh->current_count = 1;
-                }
-            } else {
-                if (td->count == 1)  {
+                if (te->current_count >= td->count) {
                     ret = 1;
-                } else {
-                    *new_tsh = DetectThresholdEntryAlloc(td, sid, gid);
+                    te->current_count = 0;
                 }
+            } else {
+                te->tv1 = p->ts;
+                te->current_count = 1;
             }
             break;
-        }
         case TYPE_BOTH:
-        {
-            SCLogDebug("both");
-
-            if (lookup_tsh != NULL) {
-                SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds);
-                if (SCTIME_CMP_LTE(p->ts, entry)) {
-                    /* within time limit */
-
-                    lookup_tsh->current_count++;
-                    if (lookup_tsh->current_count == td->count) {
-                        ret = 1;
-                    } else if (lookup_tsh->current_count > td->count) {
-                        /* silent match */
-                        ret = 2;
-
-                        if (PacketIsIPv4(p)) {
-                            SetupCache(p, td->track, (int8_t)ret, sid, entry);
-                        }
-                    }
-                } else {
-                    /* expired, so reset */
-                    lookup_tsh->tv1 = p->ts;
-                    lookup_tsh->current_count = 1;
+            if (SCTIME_CMP_LTE(p->ts, entry)) {
+                /* within time limit */
 
-                    /* if we have a limit of 1, this is a match */
-                    if (lookup_tsh->current_count == td->count) {
-                        ret = 1;
+                te->current_count++;
+                if (te->current_count == td->count) {
+                    ret = 1;
+                } else if (te->current_count > td->count) {
+                    /* silent match */
+                    ret = 2;
+
+                    if (PacketIsIPv4(p)) {
+                        SetupCache(p, td->track, (int8_t)ret, sid, entry);
                     }
                 }
             } else {
-                *new_tsh = DetectThresholdEntryAlloc(td, sid, gid);
+                /* expired, so reset */
+                te->tv1 = p->ts;
+                te->current_count = 1;
 
-                /* for the first match we return 1 to
-                 * indicate we should alert */
-                if (td->count == 1)  {
+                /* if we have a limit of 1, this is a match */
+                if (te->current_count == td->count) {
                     ret = 1;
                 }
             }
             break;
-        }
-        /* detection_filter */
         case TYPE_DETECTION:
-        {
             SCLogDebug("detection_filter");
 
-            if (lookup_tsh != NULL) {
-                SCTime_t entry = SCTIME_ADD_SECS(lookup_tsh->tv1, td->seconds);
-                if (SCTIME_CMP_LTE(p->ts, entry)) {
-                    /* within timeout */
-                    lookup_tsh->current_count++;
-                    if (lookup_tsh->current_count > td->count) {
-                        ret = 1;
-                    }
-                } else {
-                    /* expired, reset */
-                    lookup_tsh->tv1 = p->ts;
-                    lookup_tsh->current_count = 1;
+            if (SCTIME_CMP_LTE(p->ts, entry)) {
+                /* within timeout */
+                te->current_count++;
+                if (te->current_count > td->count) {
+                    ret = 1;
                 }
             } else {
-                *new_tsh = DetectThresholdEntryAlloc(td, sid, gid);
+                /* expired, reset */
+                te->tv1 = p->ts;
+                te->current_count = 1;
             }
             break;
-        }
-        /* rate_filter */
         case TYPE_RATE:
-        {
             SCLogDebug("rate_filter");
             ret = 1;
-            if (lookup_tsh && IsThresholdReached(lookup_tsh, td, p->ts)) {
-                RateFilterSetAction(pa, td->new_action);
-            } else if (!lookup_tsh) {
-                *new_tsh = DetectThresholdEntryAlloc(td, sid, gid);
+            /* Check if we have a timeout enabled, if so,
+             * we still matching (and enabling the new_action) */
+            if (te->tv_timeout != 0) {
+                if ((SCTIME_SECS(packet_time) - te->tv_timeout) > td->timeout) {
+                    /* Ok, we are done, timeout reached */
+                    te->tv_timeout = 0;
+                } else {
+                    /* Already matching */
+                    RateFilterSetAction(pa, td->new_action);
+                }
+            } else {
+                /* Update the matching state with the timeout interval */
+                if (SCTIME_CMP_LTE(packet_time, entry)) {
+                    te->current_count++;
+                    if (te->current_count > td->count) {
+                        /* Then we must enable the new action by setting a
+                         * timeout */
+                        te->tv_timeout = SCTIME_SECS(packet_time);
+                        RateFilterSetAction(pa, td->new_action);
+                    }
+                } else {
+                    te->tv1 = packet_time;
+                    te->current_count = 1;
+                }
             }
             break;
-        }
-        /* case TYPE_SUPPRESS: is not handled here */
-        default:
-            SCLogError("type %d is not supported", td->type);
     }
     return ret;
 }
 
-static int ThresholdHandlePacketIPPair(IPPair *pair, Packet *p, const DetectThresholdData *td,
-    uint32_t sid, uint32_t gid, PacketAlert *pa)
-{
-    int ret = 0;
-
-    DetectThresholdEntry *lookup_tsh = ThresholdIPPairLookupEntry(pair, sid, gid);
-    SCLogDebug("ippair lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid);
-
-    DetectThresholdEntry *new_tsh = NULL;
-    ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa);
-    if (new_tsh != NULL) {
-        AddEntryToIPPairStorage(pair, new_tsh, p->ts);
-    }
-
-    return ret;
-}
+#include "detect-engine-address-ipv6.h"
 
-/**
- *  \retval 2 silent match (no alert but apply actions)
- *  \retval 1 normal match
- *  \retval 0 no match
- */
-static int ThresholdHandlePacketHost(Host *h, Packet *p, const DetectThresholdData *td,
-        uint32_t sid, uint32_t gid, PacketAlert *pa)
+static int ThresholdGetFromHash(struct Thresholds *tctx, const Packet *p, const Signature *s,
+        const DetectThresholdData *td, PacketAlert *pa)
 {
-    int ret = 0;
-    DetectThresholdEntry *lookup_tsh = ThresholdHostLookupEntry(h, sid, gid);
-    SCLogDebug("lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid);
-
-    DetectThresholdEntry *new_tsh = NULL;
-    ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa);
-    if (new_tsh != NULL) {
-        AddEntryToHostStorage(h, new_tsh, p->ts);
+    /* fast track for count 1 threshold */
+    if (td->count == 1 && td->type == TYPE_THRESHOLD) {
+        return 1;
     }
-    return ret;
-}
 
-static int ThresholdHandlePacketRule(DetectEngineCtx *de_ctx, Packet *p,
-        const DetectThresholdData *td, const Signature *s, PacketAlert *pa)
-{
-    int ret = 0;
+    ThresholdEntry lookup;
+    memset(&lookup, 0, sizeof(lookup));
+    lookup.sid = s->id;
+    lookup.gid = s->gid;
+    lookup.track = td->track;
+    if (td->track == TRACK_SRC) {
+        COPY_ADDRESS(&p->src, &lookup.addr);
+    } else if (td->track == TRACK_DST) {
+        COPY_ADDRESS(&p->dst, &lookup.addr);
+    } else if (td->track == TRACK_BOTH) {
+        /* make sure lower ip address is first */
+        if (PacketIsIPv4(p)) {
+            if (SCNtohl(p->src.addr_data32[0]) < SCNtohl(p->dst.addr_data32[0])) {
+                COPY_ADDRESS(&p->src, &lookup.addr);
+                COPY_ADDRESS(&p->dst, &lookup.addr2);
+            } else {
+                COPY_ADDRESS(&p->dst, &lookup.addr);
+                COPY_ADDRESS(&p->src, &lookup.addr2);
+            }
+        } else {
+            if (AddressIPv6Lt(&p->src, &p->dst)) {
+                COPY_ADDRESS(&p->src, &lookup.addr);
+                COPY_ADDRESS(&p->dst, &lookup.addr2);
+            } else {
+                COPY_ADDRESS(&p->dst, &lookup.addr);
+                COPY_ADDRESS(&p->src, &lookup.addr2);
+            }
+        }
+    }
 
-    DetectThresholdEntry* lookup_tsh = (DetectThresholdEntry *)de_ctx->ths_ctx.th_entry[s->num];
-    SCLogDebug("by_rule lookup_tsh %p num %u", lookup_tsh, s->num);
+    struct THashDataGetResult res = THashGetFromHash(tctx->thash, &lookup);
+    if (res.data) {
+        SCLogDebug("found %p, is_new %s", res.data, BOOL2STR(res.is_new));
+        int r;
+        ThresholdEntry *te = res.data->data;
+        if (res.is_new) {
+            // new threshold, set up
+            r = ThresholdSetup(td, te, p->ts, s->id, s->gid);
+        } else {
+            // existing, check/update
+            r = ThresholdCheckUpdate(td, te, p, s->id, pa);
+        }
 
-    DetectThresholdEntry *new_tsh = NULL;
-    ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, s->id, s->gid, pa);
-    if (new_tsh != NULL) {
-        new_tsh->tv1 = p->ts;
-        new_tsh->current_count = 1;
-        new_tsh->tv_timeout = 0;
-        de_ctx->ths_ctx.th_entry[s->num] = new_tsh;
+        (void)THashDecrUsecnt(res.data);
+        THashDataUnlock(res.data);
+        return r;
     }
-
-    return ret;
+    return 0; // TODO error?
 }
 
 /**
@@ -881,15 +826,24 @@ static int ThresholdHandlePacketFlow(Flow *f, Packet *p, const DetectThresholdDa
         uint32_t sid, uint32_t gid, PacketAlert *pa)
 {
     int ret = 0;
-    DetectThresholdEntry *lookup_tsh = ThresholdFlowLookupEntry(f, sid, gid);
-    SCLogDebug("lookup_tsh %p sid %u gid %u", lookup_tsh, sid, gid);
-
-    DetectThresholdEntry *new_tsh = NULL;
-    ret = ThresholdHandlePacket(p, lookup_tsh, &new_tsh, td, sid, gid, pa);
-    if (new_tsh != NULL) {
-        if (AddEntryToFlow(f, new_tsh, p->ts) == -1) {
-            SCFree(new_tsh);
+    ThresholdEntry *found = ThresholdFlowLookupEntry(f, sid, gid);
+    SCLogDebug("found %p sid %u gid %u", found, sid, gid);
+
+    if (found == NULL) {
+        FlowThresholdEntryList *new = SCCalloc(1, sizeof(*new));
+        if (new == NULL)
+            return 0;
+
+        // new threshold, set up
+        ret = ThresholdSetup(td, &new->threshold, p->ts, sid, gid);
+
+        if (AddEntryToFlow(f, new, p->ts) == -1) {
+            SCFree(new);
+            return 0;
         }
+    } else {
+        // existing, check/update
+        ret = ThresholdCheckUpdate(td, found, p, sid, pa);
     }
     return ret;
 }
@@ -925,11 +879,8 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx
                 SCReturnInt(cache_ret);
             }
         }
-        Host *src = HostGetHostFromHash(&p->src);
-        if (src) {
-            ret = ThresholdHandlePacketHost(src,p,td,s->id,s->gid,pa);
-            HostRelease(src);
-        }
+
+        ret = ThresholdGetFromHash(&ctx, p, s, td, pa);
     } else if (td->track == TRACK_DST) {
         if (PacketIsIPv4(p) && (td->type == TYPE_LIMIT || td->type == TYPE_BOTH)) {
             int cache_ret = CheckCache(p, td->track, s->id);
@@ -937,21 +888,12 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx
                 SCReturnInt(cache_ret);
             }
         }
-        Host *dst = HostGetHostFromHash(&p->dst);
-        if (dst) {
-            ret = ThresholdHandlePacketHost(dst,p,td,s->id,s->gid,pa);
-            HostRelease(dst);
-        }
+
+        ret = ThresholdGetFromHash(&ctx, p, s, td, pa);
     } else if (td->track == TRACK_BOTH) {
-        IPPair *pair = IPPairGetIPPairFromHash(&p->src, &p->dst);
-        if (pair) {
-            ret = ThresholdHandlePacketIPPair(pair, p, td, s->id, s->gid, pa);
-            IPPairRelease(pair);
-        }
+        ret = ThresholdGetFromHash(&ctx, p, s, td, pa);
     } else if (td->track == TRACK_RULE) {
-        SCMutexLock(&de_ctx->ths_ctx.threshold_table_lock);
-        ret = ThresholdHandlePacketRule(de_ctx,p,td,s,pa);
-        SCMutexUnlock(&de_ctx->ths_ctx.threshold_table_lock);
+        ret = ThresholdGetFromHash(&ctx, p, s, td, pa);
     } else if (td->track == TRACK_FLOW) {
         if (p->flow) {
             ret = ThresholdHandlePacketFlow(p->flow, p, td, s->id, s->gid, pa);
@@ -961,125 +903,6 @@ int PacketAlertThreshold(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx
     SCReturnInt(ret);
 }
 
-/**
- * \brief Init threshold context hash tables
- *
- * \param de_ctx Detection Context
- *
- */
-void ThresholdHashInit(DetectEngineCtx *de_ctx)
-{
-    if (SCMutexInit(&de_ctx->ths_ctx.threshold_table_lock, NULL) != 0) {
-        FatalError("Threshold: Failed to initialize hash table mutex.");
-    }
-}
-
-/**
- * \brief Allocate threshold context hash tables
- *
- * \param de_ctx Detection Context
- */
-void ThresholdHashAllocate(DetectEngineCtx *de_ctx)
-{
-    const Signature *s = de_ctx->sig_list;
-    bool has_by_rule_tracking = false;
-
-    /* Find the signature with the highest signature number that is using
-       thresholding with by_rule tracking. */
-    uint32_t highest_signum = 0;
-    while (s != NULL) {
-        if (s->sm_arrays[DETECT_SM_LIST_SUPPRESS] != NULL) {
-            const SigMatchData *smd = NULL;
-            do {
-                const DetectThresholdData *td =
-                        SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_SUPPRESS);
-                if (td == NULL) {
-                    continue;
-                }
-                if (td->track != TRACK_RULE) {
-                    continue;
-                }
-                if (s->num >= highest_signum) {
-                    highest_signum = s->num;
-                    has_by_rule_tracking = true;
-                }
-            } while (smd != NULL);
-        }
-
-        if (s->sm_arrays[DETECT_SM_LIST_THRESHOLD] != NULL) {
-            const SigMatchData *smd = NULL;
-            do {
-                const DetectThresholdData *td =
-                        SigGetThresholdTypeIter(s, &smd, DETECT_SM_LIST_THRESHOLD);
-                if (td == NULL) {
-                    continue;
-                }
-                if (td->track != TRACK_RULE) {
-                    continue;
-                }
-                if (s->num >= highest_signum) {
-                    highest_signum = s->num;
-                    has_by_rule_tracking = true;
-                }
-            } while (smd != NULL);
-        }
-
-        s = s->next;
-    }
-
-    /* Skip allocating if by_rule tracking is not used */
-    if (has_by_rule_tracking == false) {
-        return;
-    }
-
-    de_ctx->ths_ctx.th_size = highest_signum + 1;
-    de_ctx->ths_ctx.th_entry = SCCalloc(de_ctx->ths_ctx.th_size, sizeof(DetectThresholdEntry *));
-    if (de_ctx->ths_ctx.th_entry == NULL) {
-        FatalError(
-                "failed to allocate memory for \"by_rule\" thresholding (tried to allocate %" PRIu32
-                " entries)",
-                de_ctx->ths_ctx.th_size);
-    }
-}
-
-/**
- * \brief Destroy threshold context hash tables
- *
- * \param de_ctx Detection Context
- *
- */
-void ThresholdContextDestroy(DetectEngineCtx *de_ctx)
-{
-    if (de_ctx->ths_ctx.th_entry != NULL) {
-        for (uint32_t i = 0; i < de_ctx->ths_ctx.th_size; i++) {
-            if (de_ctx->ths_ctx.th_entry[i] != NULL) {
-                SCFree(de_ctx->ths_ctx.th_entry[i]);
-            }
-        }
-        SCFree(de_ctx->ths_ctx.th_entry);
-    }
-    SCMutexDestroy(&de_ctx->ths_ctx.threshold_table_lock);
-}
-
-/**
- * \brief this function will free all the entries of a list
- *        DetectTagDataEntry
- *
- * \param td pointer to DetectTagDataEntryList
- */
-void ThresholdListFree(void *ptr)
-{
-    if (ptr != NULL) {
-        DetectThresholdEntry *entry = ptr;
-
-        while (entry != NULL) {
-            DetectThresholdEntry *next_entry = entry->next;
-            SCFree(entry);
-            entry = next_entry;
-        }
-    }
-}
-
 /**
  * @}
  */

src/detect-engine-threshold.h

@@ -26,16 +26,12 @@
 #define SURICATA_DETECT_ENGINE_THRESHOLD_H
 
 #include "detect.h"
-#include "host.h"
-#include "ippair.h"
-#include "host-storage.h"
+#include "detect-threshold.h"
 
 void ThresholdInit(void);
+void ThresholdDestroy(void);
 
-HostStorageId ThresholdHostStorageId(void);
-int ThresholdHostHasThreshold(Host *);
-
-int ThresholdIPPairHasThreshold(IPPair *pair);
+uint32_t ThresholdsExpire(const SCTime_t ts);
 
 const DetectThresholdData *SigGetThresholdTypeIter(
         const Signature *, const SigMatchData **, int list);
@@ -43,12 +39,6 @@ int PacketAlertThreshold(DetectEngineCtx *, DetectEngineThreadCtx *,
         const DetectThresholdData *, Packet *,
         const Signature *, PacketAlert *);
 
-void ThresholdHashInit(DetectEngineCtx *);
-void ThresholdHashAllocate(DetectEngineCtx *);
-void ThresholdContextDestroy(DetectEngineCtx *);
-
-int ThresholdHostTimeoutCheck(Host *, SCTime_t);
-int ThresholdIPPairTimeoutCheck(IPPair *, SCTime_t);
 void ThresholdListFree(void *ptr);
 void ThresholdCacheThreadFree(void);
 

src/detect-engine.c

@@ -2506,7 +2506,6 @@ static DetectEngineCtx *DetectEngineCtxInitReal(
 
     SigGroupHeadHashInit(de_ctx);
     MpmStoreInit(de_ctx);
-    ThresholdHashInit(de_ctx);
     DetectParseDupSigHashInit(de_ctx);
     DetectAddressMapInit(de_ctx);
     DetectMetadataHashInit(de_ctx);
@@ -2620,7 +2619,6 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     MpmStoreFree(de_ctx);
     DetectParseDupSigHashFree(de_ctx);
     SCSigSignatureOrderingModuleCleanup(de_ctx);
-    ThresholdContextDestroy(de_ctx);
     SigCleanSignatures(de_ctx);
     if (de_ctx->sig_array)
         SCFree(de_ctx->sig_array);

src/detect-threshold.c

@@ -509,7 +509,7 @@ static int DetectThresholdTestSig1(void)
     int result = 0;
     int alerts = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -587,8 +587,8 @@ static int DetectThresholdTestSig1(void)
 
     UTHFreePackets(&p, 1);
 
-    HostShutdown();
 end:
+    ThresholdDestroy();
     return result;
 }
 
@@ -610,7 +610,7 @@ static int DetectThresholdTestSig2(void)
     int result = 0;
     int alerts = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -658,15 +658,12 @@ static int DetectThresholdTestSig2(void)
         goto cleanup;
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -681,94 +678,49 @@ end:
 
 static int DetectThresholdTestSig3(void)
 {
-    Packet *p = NULL;
-    Signature *s = NULL;
     ThreadVars th_v;
-    DetectEngineThreadCtx *det_ctx;
-    int result = 0;
-    int alerts = 0;
-    DetectThresholdEntry *lookup_tsh = NULL;
-
-    HostInitConfig(HOST_QUIET);
-
     memset(&th_v, 0, sizeof(th_v));
 
-    p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
+    ThresholdInit();
+    Packet *p = UTHBuildPacketReal((uint8_t *)"A", 1, IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
 
     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
-    if (de_ctx == NULL) {
-        goto end;
-    }
-
+    FAIL_IF_NULL(de_ctx);
     de_ctx->flags |= DE_QUIET;
 
-    s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any 80 (msg:\"Threshold limit\"; threshold: type limit, track by_dst, count 5, seconds 60; sid:10;)");
-    if (s == NULL) {
-        goto end;
-    }
+    Signature *s = DetectEngineAppendSig(de_ctx,
+            "alert tcp any any -> any 80 (msg:\"Threshold limit\"; threshold: type limit, "
+            "track by_dst, count 5, seconds 60; sid:10;)");
+    FAIL_IF_NULL(s);
 
     SigGroupBuild(de_ctx);
+    DetectEngineThreadCtx *det_ctx;
     DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
 
     p->ts = TimeGet();
 
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1);
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1);
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
-
-    Host *host = HostLookupHostFromHash(&p->dst);
-    if (host == NULL) {
-        printf("host not found: ");
-        goto cleanup;
-    }
-
-    if (!(ThresholdHostHasThreshold(host))) {
-        HostRelease(host);
-        printf("host has no threshold: ");
-        goto cleanup;
-    }
-    HostRelease(host);
+    FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1);
 
     TimeSetIncrementTime(200);
     p->ts = TimeGet();
 
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1);
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
+    FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1);
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
-
-    host = HostLookupHostFromHash(&p->dst);
-    if (host == NULL) {
-        printf("host not found: ");
-        goto cleanup;
-    }
-    HostRelease(host);
-
-    lookup_tsh = HostGetStorageById(host, ThresholdHostStorageId());
-    if (lookup_tsh == NULL) {
-        HostRelease(host);
-        printf("lookup_tsh is NULL: ");
-        goto cleanup;
-    }
-
-    alerts = lookup_tsh->current_count;
-
-    if (alerts == 3)
-        result = 1;
-    else {
-        printf("alerts %u != 3: ", alerts);
-        goto cleanup;
-    }
-
-cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
+    FAIL_IF_NOT(PacketAlertCheck(p, 10) == 1);
 
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
-    return result;
+    ThresholdDestroy();
+    PASS;
 }
 
 /**
@@ -789,7 +741,7 @@ static int DetectThresholdTestSig4(void)
     int result = 0;
     int alerts = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -834,14 +786,11 @@ static int DetectThresholdTestSig4(void)
         goto cleanup;
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -863,7 +812,7 @@ static int DetectThresholdTestSig5(void)
     int result = 0;
     int alerts = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
     p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
@@ -921,15 +870,12 @@ static int DetectThresholdTestSig5(void)
     }
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -942,7 +888,7 @@ static int DetectThresholdTestSig6Ticks(void)
     int result = 0;
     int alerts = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
     p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
@@ -1004,15 +950,12 @@ static int DetectThresholdTestSig6Ticks(void)
         goto cleanup;
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -1029,7 +972,7 @@ static int DetectThresholdTestSig7(void)
     int alerts = 0;
     int drops = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -1095,14 +1038,11 @@ static int DetectThresholdTestSig7(void)
     }
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -1119,7 +1059,7 @@ static int DetectThresholdTestSig8(void)
     int alerts = 0;
     int drops = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -1185,14 +1125,11 @@ static int DetectThresholdTestSig8(void)
     }
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -1209,7 +1146,7 @@ static int DetectThresholdTestSig9(void)
     int alerts = 0;
     int drops = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -1275,14 +1212,11 @@ static int DetectThresholdTestSig9(void)
     }
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -1299,7 +1233,7 @@ static int DetectThresholdTestSig10(void)
     int alerts = 0;
     int drops = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -1365,14 +1299,11 @@ static int DetectThresholdTestSig10(void)
     }
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -1389,7 +1320,7 @@ static int DetectThresholdTestSig11(void)
     int alerts = 0;
     int drops = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -1455,14 +1386,11 @@ static int DetectThresholdTestSig11(void)
     }
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 end:
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -1479,7 +1407,7 @@ static int DetectThresholdTestSig12(void)
     int alerts = 0;
     int drops = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
 
@@ -1545,14 +1473,12 @@ static int DetectThresholdTestSig12(void)
     }
 
 cleanup:
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
-
     DetectEngineThreadCtxDeinit(&th_v, (void*)det_ctx);
     DetectEngineCtxFree(de_ctx);
 end:
     UTHFreePackets(&p, 1);
     HostShutdown();
+    ThresholdDestroy();
     return result;
 }
 
@@ -1573,7 +1499,7 @@ static int DetectThresholdTestSig13(void)
     DetectEngineThreadCtx *det_ctx;
     int alerts = 0;
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
     p = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
@@ -1616,12 +1542,10 @@ static int DetectThresholdTestSig13(void)
 
     FAIL_IF(alerts != 4);
 
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
     UTHFreePackets(&p, 1);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -1644,8 +1568,7 @@ static int DetectThresholdTestSig14(void)
     int alerts1 = 0;
     int alerts2 = 0;
 
-    HostInitConfig(HOST_QUIET);
-    IPPairInitConfig(IPPAIR_QUIET);
+    ThresholdInit();
 
     memset(&th_v, 0, sizeof(th_v));
     p1 = UTHBuildPacketReal((uint8_t *)"A",1,IPPROTO_TCP, "1.1.1.1", "2.2.2.2", 1024, 80);
@@ -1699,13 +1622,11 @@ static int DetectThresholdTestSig14(void)
     FAIL_IF(alerts1 != 3);
     FAIL_IF(alerts2 != 3);
 
-    SigGroupCleanup(de_ctx);
-    SigCleanSignatures(de_ctx);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
     UTHFreePackets(&p1, 1);
     UTHFreePackets(&p2, 1);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 

src/detect-threshold.h

@@ -62,21 +62,6 @@ typedef struct DetectThresholdData_ {
     DetectAddressHead addrs;
 } DetectThresholdData;
 
-typedef struct DetectThresholdEntry_ {
-    uint32_t sid;           /**< Signature id */
-    uint32_t gid;           /**< Signature group id */
-
-    uint32_t tv_timeout;    /**< Timeout for new_action (for rate_filter)
-                                 its not "seconds", that define the time interval */
-    uint32_t seconds;       /**< Event seconds */
-    uint32_t current_count; /**< Var for count control */
-    int track;          /**< Track type: by_src, by_src */
-
-    SCTime_t tv1; /**< Var for time control */
-    struct DetectThresholdEntry_ *next;
-} DetectThresholdEntry;
-
-
 /**
  * Registration function for threshold: keyword
  */

src/detect.h

@@ -785,17 +785,6 @@ typedef struct DetectEngineLookupFlow_ {
     struct SigGroupHead_ *sgh[256];
 } DetectEngineLookupFlow;
 
-#include "detect-threshold.h"
-
-/** \brief threshold ctx */
-typedef struct ThresholdCtx_    {
-    SCMutex threshold_table_lock;                   /**< Mutex for hash table */
-
-    /** to support rate_filter "by_rule" option */
-    DetectThresholdEntry **th_entry;
-    uint32_t th_size;
-} ThresholdCtx;
-
 typedef struct SigString_ {
     char *filename;
     char *sig_str;
@@ -886,7 +875,6 @@ typedef struct DetectEngineCtx_ {
     HashListTable *dup_sig_hash_table;
 
     DetectEngineIPOnlyCtx io_ctx;
-    ThresholdCtx ths_ctx;
 
     /* maximum recursion depth for content inspection */
     int inspection_recursion_limit;

src/flow-manager.c

@@ -47,6 +47,7 @@
 #include "util-debug.h"
 
 #include "threads.h"
+#include "detect-engine-threshold.h"
 
 #include "host-timeout.h"
 #include "defrag-hash.h"
@@ -931,6 +932,7 @@ static TmEcode FlowManager(ThreadVars *th_v, void *thread_data)
                 HostTimeoutHash(ts);
                 IPPairTimeoutHash(ts);
                 HttpRangeContainersTimeoutHash(ts);
+                ThresholdsExpire(ts);
                 other_last_sec = (uint32_t)SCTIME_SECS(ts);
             }
         }

src/host-timeout.c

@@ -25,7 +25,6 @@
 #include "host.h"
 
 #include "detect-engine-tag.h"
-#include "detect-engine-threshold.h"
 
 #include "host-bit.h"
 #include "host-timeout.h"
@@ -53,7 +52,6 @@ static int HostHostTimedOut(Host *h, SCTime_t ts)
 
     busy |= (h->iprep && SRepHostTimedOut(h) == 0);
     busy |= (TagHostHasTag(h) && TagTimeoutCheck(h, ts) == 0);
-    busy |= (ThresholdHostHasThreshold(h) && ThresholdHostTimeoutCheck(h, ts) == 0);
     busy |= (HostHasHostBits(h) && HostBitsTimedoutCheck(h, ts) == 0);
     SCLogDebug("host %p %s", h, busy ? "still active" : "timed out");
     return !busy;

src/ippair-timeout.c

@@ -25,7 +25,6 @@
 #include "ippair.h"
 #include "ippair-bit.h"
 #include "ippair-timeout.h"
-#include "detect-engine-threshold.h"
 
 /** \internal
  *  \brief See if we can really discard this ippair. Check use_cnt reference.
@@ -39,8 +38,6 @@
 static int IPPairTimedOut(IPPair *h, SCTime_t ts)
 {
     int vars = 0;
-    int thresholds = 0;
-
     /** never prune a ippair that is used by a packet
      *  we are currently processing in one of the threads */
     if (SC_ATOMIC_GET(h->use_cnt) > 0) {
@@ -50,12 +47,7 @@ static int IPPairTimedOut(IPPair *h, SCTime_t ts)
     if (IPPairHasBits(h) && IPPairBitsTimedoutCheck(h, ts) == 0) {
         vars = 1;
     }
-
-    if (ThresholdIPPairHasThreshold(h) && ThresholdIPPairTimeoutCheck(h, ts) == 0) {
-        thresholds = 1;
-    }
-
-    if (vars || thresholds) {
+    if (vars) {
         return 0;
     }
 

src/suricata.c

@@ -369,6 +369,7 @@ void GlobalsInitPreConfig(void)
 void GlobalsDestroy(void)
 {
     SCInstance *suri = &suricata;
+    ThresholdDestroy();
     HostShutdown();
     HTPFreeConfig();
     HTPAtExitPrintStats();

src/util-threshold-config.c

@@ -37,6 +37,7 @@
 #include "detect.h"
 #include "detect-engine.h"
 #include "detect-engine-address.h"
+#include "detect-engine-threshold.h"
 #include "detect-threshold.h"
 #include "detect-parse.h"
 #include "detect-engine-build.h"
@@ -1534,7 +1535,7 @@ static int SCThresholdConfTest09(void)
     ThreadVars th_v;
     memset(&th_v, 0, sizeof(th_v));
 
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP);
     FAIL_IF_NULL(p);
@@ -1603,7 +1604,7 @@ static int SCThresholdConfTest09(void)
     UTHFreePacket(p);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -1615,7 +1616,7 @@ static int SCThresholdConfTest09(void)
  */
 static int SCThresholdConfTest10(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     /* Create two different packets falling to the same rule, and
     *  because count:3, we should drop on match #4.
@@ -1685,15 +1686,15 @@ static int SCThresholdConfTest10(void)
     SigMatchSignatures(&th_v, de_ctx, det_ctx, p1);
     FAIL_IF(PacketTestAction(p1, ACTION_DROP));
     FAIL_IF(PacketAlertCheck(p1, 10) != 1);
-
+#if 0
     /* Ensure that a Threshold entry was installed at the sig */
     FAIL_IF_NULL(de_ctx->ths_ctx.th_entry[s->num]);
-
+#endif
     UTHFreePacket(p1);
     UTHFreePacket(p2);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -1705,7 +1706,7 @@ static int SCThresholdConfTest10(void)
  */
 static int SCThresholdConfTest11(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP);
     FAIL_IF_NULL(p);
@@ -1798,7 +1799,7 @@ static int SCThresholdConfTest11(void)
     UTHFreePacket(p);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -1810,7 +1811,7 @@ static int SCThresholdConfTest11(void)
  */
 static int SCThresholdConfTest12(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     Packet *p = UTHBuildPacket((uint8_t*)"lalala", 6, IPPROTO_TCP);
     FAIL_IF_NULL(p);
@@ -1903,7 +1904,7 @@ static int SCThresholdConfTest12(void)
     UTHFreePacket(p);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -1948,7 +1949,7 @@ static int SCThresholdConfTest13(void)
  */
 static int SCThresholdConfTest14(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     Packet *p1 = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10",
                                     "192.168.0.100", 1234, 24);
@@ -1997,7 +1998,7 @@ static int SCThresholdConfTest14(void)
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
 
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2009,7 +2010,7 @@ static int SCThresholdConfTest14(void)
  */
 static int SCThresholdConfTest15(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10",
                                     "192.168.0.100", 1234, 24);
@@ -2045,7 +2046,7 @@ static int SCThresholdConfTest15(void)
     UTHFreePacket(p);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2057,7 +2058,7 @@ static int SCThresholdConfTest15(void)
  */
 static int SCThresholdConfTest16(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.1.1",
                                     "192.168.0.100", 1234, 24);
@@ -2092,7 +2093,7 @@ static int SCThresholdConfTest16(void)
     UTHFreePacket(p);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2104,7 +2105,7 @@ static int SCThresholdConfTest16(void)
  */
 static int SCThresholdConfTest17(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
 
     Packet *p = UTHBuildPacketReal((uint8_t*)"lalala", 6, IPPROTO_TCP, "192.168.0.10",
                                     "192.168.0.100", 1234, 24);
@@ -2140,7 +2141,7 @@ static int SCThresholdConfTest17(void)
     UTHFreePacket(p);
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2171,7 +2172,7 @@ static FILE *SCThresholdConfGenerateInvalidDummyFD12(void)
  */
 static int SCThresholdConfTest18(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
     FAIL_IF_NULL(de_ctx);
     de_ctx->flags |= DE_QUIET;
@@ -2192,7 +2193,7 @@ static int SCThresholdConfTest18(void)
     FAIL_IF_NOT(de->type == TYPE_SUPPRESS && de->track == TRACK_DST);
 
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2223,7 +2224,7 @@ static FILE *SCThresholdConfGenerateInvalidDummyFD13(void)
  */
 static int SCThresholdConfTest19(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
     FAIL_IF_NULL(de_ctx);
     de_ctx->flags |= DE_QUIET;
@@ -2241,7 +2242,7 @@ static int SCThresholdConfTest19(void)
     FAIL_IF_NULL(de);
     FAIL_IF_NOT(de->type == TYPE_SUPPRESS && de->track == TRACK_DST);
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2273,7 +2274,7 @@ static FILE *SCThresholdConfGenerateValidDummyFD20(void)
  */
 static int SCThresholdConfTest20(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
     FAIL_IF_NULL(de_ctx);
     de_ctx->flags |= DE_QUIET;
@@ -2306,7 +2307,7 @@ static int SCThresholdConfTest20(void)
     FAIL_IF_NOT(smd->is_last);
 
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2319,7 +2320,7 @@ static int SCThresholdConfTest20(void)
  */
 static int SCThresholdConfTest21(void)
 {
-    HostInitConfig(HOST_QUIET);
+    ThresholdInit();
     DetectEngineCtx *de_ctx = DetectEngineCtxInit();
     FAIL_IF_NULL(de_ctx);
     de_ctx->flags |= DE_QUIET;
@@ -2351,7 +2352,7 @@ static int SCThresholdConfTest21(void)
     FAIL_IF_NOT(smd->is_last);
 
     DetectEngineCtxFree(de_ctx);
-    HostShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2384,7 +2385,7 @@ static int SCThresholdConfTest22(void)
     ThreadVars th_v;
     memset(&th_v, 0, sizeof(th_v));
 
-    IPPairInitConfig(IPPAIR_QUIET);
+    ThresholdInit();
 
     /* This packet will cause rate_filter */
     Packet *p1 = UTHBuildPacketSrcDst((uint8_t*)"lalala", 6, IPPROTO_TCP, "172.26.0.1", "172.26.0.10");
@@ -2487,7 +2488,7 @@ static int SCThresholdConfTest22(void)
 
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    IPPairShutdown();
+    ThresholdDestroy();
     PASS;
 }
 
@@ -2521,7 +2522,7 @@ static int SCThresholdConfTest23(void)
     ThreadVars th_v;
     memset(&th_v, 0, sizeof(th_v));
 
-    IPPairInitConfig(IPPAIR_QUIET);
+    ThresholdInit();
 
     /* Create two packets between same addresses in opposite direction */
     Packet *p1 = UTHBuildPacketSrcDst((uint8_t*)"lalala", 6, IPPROTO_TCP, "172.26.0.1", "172.26.0.10");
@@ -2568,7 +2569,7 @@ static int SCThresholdConfTest23(void)
 
     DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
     DetectEngineCtxFree(de_ctx);
-    IPPairShutdown();
+    ThresholdDestroy();
     PASS;
 }
 #endif /* UNITTESTS */