From 9d49364afa0413b08f19dde6c026aef563b61325 Mon Sep 17 00:00:00 2001 From: keiahny Date: Thu, 26 Mar 2026 16:35:49 +0100 Subject: [PATCH 1/3] host/ippair: add counters for active hosts and IP pair objects to stats log Track host_active/ippair_active counters in stats log for memcap and hash table sizing. Feature: #8438. --- src/app-layer.c | 2 ++ src/host.c | 15 +++++++++++---- src/host.h | 3 ++- src/ippair.c | 15 +++++++++++---- src/ippair.h | 3 ++- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/app-layer.c b/src/app-layer.c index 98fb98cade..6660a0ae9f 100644 --- a/src/app-layer.c +++ b/src/app-layer.c @@ -1173,8 +1173,10 @@ void AppLayerRegisterGlobalCounters(void) StatsRegisterGlobalCounter("http.byterange.memcap", HTPByteRangeMemcapGlobalCounter); StatsRegisterGlobalCounter("ippair.memuse", IPPairGetMemuse); StatsRegisterGlobalCounter("ippair.memcap", IPPairGetMemcap); + StatsRegisterGlobalCounter("ippair.active", IPPairGetActive); StatsRegisterGlobalCounter("host.memuse", HostGetMemuse); StatsRegisterGlobalCounter("host.memcap", HostGetMemcap); + StatsRegisterGlobalCounter("host.active", HostGetActive); } static bool IsAppLayerErrorExceptionPolicyStatsValid(enum ExceptionPolicy policy) diff --git a/src/host.c b/src/host.c index c055f7436a..3da5e5b021 100644 --- a/src/host.c +++ b/src/host.c @@ -53,7 +53,7 @@ static HostQueue host_spare_q; HostConfig host_config; SC_ATOMIC_DECLARE(uint64_t,host_memuse); -SC_ATOMIC_DECLARE(uint32_t,host_counter); +SC_ATOMIC_DECLARE(uint64_t, host_active); SC_ATOMIC_DECLARE(uint32_t,host_prune_idx); /** size of the host object. Maybe updated in HostInitConfig to include @@ -97,10 +97,16 @@ uint64_t HostGetMemuse(void) return memuse; } +uint64_t HostGetActive(void) +{ + uint64_t host_activecopy = SC_ATOMIC_GET(host_active); + return host_activecopy; +} + void HostMoveToSpare(Host *h) { HostEnqueue(&host_spare_q, h); - (void) SC_ATOMIC_SUB(host_counter, 1); + (void)SC_ATOMIC_SUB(host_active, 1); } Host *HostAlloc(void) @@ -175,7 +181,8 @@ void HostInitConfig(bool quiet) memset(&host_config, 0, sizeof(host_config)); //SC_ATOMIC_INIT(flow_flags); - SC_ATOMIC_INIT(host_counter); + SC_ATOMIC_INIT(host_active); + SC_ATOMIC_SET(host_active, 0); SC_ATOMIC_INIT(host_memuse); SC_ATOMIC_INIT(host_prune_idx); SC_ATOMIC_INIT(host_config.memcap); @@ -447,7 +454,7 @@ static Host *HostGetNew(Address *a) /* host is initialized (recycled) but *unlocked* */ } - (void) SC_ATOMIC_ADD(host_counter, 1); + (void)SC_ATOMIC_ADD(host_active, 1); SCMutexLock(&h->m); return h; } diff --git a/src/host.h b/src/host.h index 32d52271b8..8dc1077c86 100644 --- a/src/host.h +++ b/src/host.h @@ -130,7 +130,7 @@ typedef struct HostConfig_ { extern HostConfig host_config; SC_ATOMIC_EXTERN(uint64_t,host_memuse); -SC_ATOMIC_EXTERN(uint32_t,host_counter); +SC_ATOMIC_EXTERN(uint64_t, host_active); SC_ATOMIC_EXTERN(uint32_t,host_prune_idx); void HostInitConfig(bool quiet); @@ -155,5 +155,6 @@ void HostUnlock(Host *h); int HostSetMemcap(uint64_t); uint64_t HostGetMemcap(void); uint64_t HostGetMemuse(void); +uint64_t HostGetActive(void); #endif /* SURICATA_HOST_H */ diff --git a/src/ippair.c b/src/ippair.c index 32f523c386..51f1f42294 100644 --- a/src/ippair.c +++ b/src/ippair.c @@ -51,7 +51,7 @@ IPPairHashRow *ippair_hash; static IPPairQueue ippair_spare_q; IPPairConfig ippair_config; SC_ATOMIC_DECLARE(uint64_t,ippair_memuse); -SC_ATOMIC_DECLARE(uint32_t,ippair_counter); +SC_ATOMIC_DECLARE(uint64_t, ippair_active); SC_ATOMIC_DECLARE(uint32_t,ippair_prune_idx); /** size of the ippair object. Maybe updated in IPPairInitConfig to include @@ -95,10 +95,16 @@ uint64_t IPPairGetMemuse(void) return memusecopy; } +uint64_t IPPairGetActive(void) +{ + uint64_t ippair_activecopy = SC_ATOMIC_GET(ippair_active); + return ippair_activecopy; +} + void IPPairMoveToSpare(IPPair *h) { IPPairEnqueue(&ippair_spare_q, h); - (void) SC_ATOMIC_SUB(ippair_counter, 1); + (void)SC_ATOMIC_SUB(ippair_active, 1); } IPPair *IPPairAlloc(void) @@ -169,7 +175,8 @@ void IPPairInitConfig(bool quiet) memset(&ippair_config, 0, sizeof(ippair_config)); //SC_ATOMIC_INIT(flow_flags); - SC_ATOMIC_INIT(ippair_counter); + SC_ATOMIC_INIT(ippair_active); + SC_ATOMIC_SET(ippair_active, 0); SC_ATOMIC_INIT(ippair_memuse); SC_ATOMIC_INIT(ippair_prune_idx); SC_ATOMIC_INIT(ippair_config.memcap); @@ -490,7 +497,7 @@ static IPPair *IPPairGetNew(Address *a, Address *b) /* ippair is initialized (recycled) but *unlocked* */ } - (void) SC_ATOMIC_ADD(ippair_counter, 1); + (void)SC_ATOMIC_ADD(ippair_active, 1); SCMutexLock(&h->m); return h; } diff --git a/src/ippair.h b/src/ippair.h index 4f42ab92e4..f4c8b84c81 100644 --- a/src/ippair.h +++ b/src/ippair.h @@ -112,7 +112,7 @@ typedef struct IPPairConfig_ { extern IPPairConfig ippair_config; SC_ATOMIC_EXTERN(uint64_t,ippair_memuse); -SC_ATOMIC_EXTERN(uint32_t,ippair_counter); +SC_ATOMIC_EXTERN(uint64_t, ippair_active); SC_ATOMIC_EXTERN(uint32_t,ippair_prune_idx); void IPPairInitConfig(bool quiet); @@ -136,5 +136,6 @@ void IPPairUnlock(IPPair *); int IPPairSetMemcap(uint64_t size); uint64_t IPPairGetMemcap(void); uint64_t IPPairGetMemuse(void); +uint64_t IPPairGetActive(void); #endif /* SURICATA_IPPAIR_H */ From 2a3a8121bd2fa88e66da69e4b8009042de83bf2a Mon Sep 17 00:00:00 2001 From: keiahny Date: Sat, 28 Mar 2026 16:26:46 +0100 Subject: [PATCH 2/3] defrag: added counters for active trackers and max fragments per tracker to stats log Track defrag_tracker_active/defrag_max_fragments counters in stats log for memcap and hash table sizing. Feature: #8438. --- src/defrag-hash.c | 27 +++++++++++++++++++++++---- src/defrag-hash.h | 5 ++++- src/defrag.c | 7 ++++++- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/defrag-hash.c b/src/defrag-hash.c index a9fa43ab4a..c9460f180a 100644 --- a/src/defrag-hash.c +++ b/src/defrag-hash.c @@ -31,7 +31,8 @@ DefragTrackerHashRow *defragtracker_hash; DefragConfig defrag_config; SC_ATOMIC_DECLARE(uint64_t,defrag_memuse); -SC_ATOMIC_DECLARE(unsigned int,defragtracker_counter); +SC_ATOMIC_DECLARE(uint64_t, defrag_tracker_active); +SC_ATOMIC_DECLARE(uint64_t, defrag_max_fragments); SC_ATOMIC_DECLARE(unsigned int,defragtracker_prune_idx); static DefragTracker *DefragTrackerGetUsedDefragTracker( @@ -77,6 +78,18 @@ uint64_t DefragTrackerGetMemuse(void) return memusecopy; } +uint64_t DefragTrackerGetActive(void) +{ + uint64_t defragtracker_activecopy = SC_ATOMIC_GET(defrag_tracker_active); + return defragtracker_activecopy; +} + +uint64_t DefragTrackerGetMaxFragments(void) +{ + uint64_t defrag_max_fragmentscopy = SC_ATOMIC_GET(defrag_max_fragments); + return defrag_max_fragmentscopy; +} + enum ExceptionPolicy DefragGetMemcapExceptionPolicy(void) { return defrag_config.memcap_policy; @@ -85,7 +98,7 @@ enum ExceptionPolicy DefragGetMemcapExceptionPolicy(void) void DefragTrackerMoveToSpare(DefragTracker *h) { DefragTrackerEnqueue(&defragtracker_spare_q, h); - (void) SC_ATOMIC_SUB(defragtracker_counter, 1); + (void)SC_ATOMIC_SUB(defrag_tracker_active, 1); } static DefragTracker *DefragTrackerAlloc(void) @@ -169,7 +182,10 @@ void DefragInitConfig(bool quiet) SCLogDebug("initializing defrag engine..."); memset(&defrag_config, 0, sizeof(defrag_config)); - SC_ATOMIC_INIT(defragtracker_counter); + SC_ATOMIC_INIT(defrag_tracker_active); + SC_ATOMIC_SET(defrag_tracker_active, 0); + SC_ATOMIC_INIT(defrag_max_fragments); + SC_ATOMIC_SET(defrag_max_fragments, 0); SC_ATOMIC_INIT(defrag_memuse); SC_ATOMIC_INIT(defragtracker_prune_idx); SC_ATOMIC_INIT(defrag_config.memcap); @@ -283,6 +299,9 @@ void DefragInitConfig(bool quiet) SCLogConfig("defrag memory usage: %"PRIu64" bytes, maximum: %"PRIu64, SC_ATOMIC_GET(defrag_memuse), SC_ATOMIC_GET(defrag_config.memcap)); } + + StatsRegisterGlobalCounter("defrag.tracker_active", DefragTrackerGetActive); + StatsRegisterGlobalCounter("defrag.max_fragments", DefragTrackerGetMaxFragments); } /** \brief shutdown the flow engine @@ -511,7 +530,7 @@ static DefragTracker *DefragTrackerGetNew(ThreadVars *tv, DecodeThreadVars *dtv, /* tracker is initialized (recycled) but *unlocked* */ } - (void) SC_ATOMIC_ADD(defragtracker_counter, 1); + (void)SC_ATOMIC_ADD(defrag_tracker_active, 1); SCMutexLock(&dt->lock); return dt; } diff --git a/src/defrag-hash.h b/src/defrag-hash.h index 299967a492..491cb57fa8 100644 --- a/src/defrag-hash.h +++ b/src/defrag-hash.h @@ -85,7 +85,8 @@ typedef struct DefragConfig_ { extern DefragConfig defrag_config; SC_ATOMIC_EXTERN(uint64_t,defrag_memuse); -SC_ATOMIC_EXTERN(unsigned int,defragtracker_counter); +SC_ATOMIC_EXTERN(uint64_t, defrag_tracker_active); +SC_ATOMIC_EXTERN(uint64_t, defrag_max_fragments); SC_ATOMIC_EXTERN(unsigned int,defragtracker_prune_idx); void DefragInitConfig(bool quiet); @@ -100,6 +101,8 @@ void DefragTrackerMoveToSpare(DefragTracker *); int DefragTrackerSetMemcap(uint64_t); uint64_t DefragTrackerGetMemcap(void); uint64_t DefragTrackerGetMemuse(void); +uint64_t DefragTrackerGetActive(void); +uint64_t DefragTrackerGetMaxFragments(void); enum ExceptionPolicy DefragGetMemcapExceptionPolicy(void); #endif /* SURICATA_DEFRAG_HASH_H */ diff --git a/src/defrag.c b/src/defrag.c index ce84388021..338f70cc9b 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -120,7 +120,7 @@ void DefragTrackerFreeFrags(DefragTracker *tracker) { Frag *frag, *tmp; - + uint64_t frag_count = 0; /* Lock the frag pool as we'll be return items to it. */ SCMutexLock(&defrag_context->frag_pool_lock); @@ -128,6 +128,11 @@ DefragTrackerFreeFrags(DefragTracker *tracker) RB_REMOVE(IP_FRAGMENTS, &tracker->fragment_tree, frag); DefragFragReset(frag); PoolReturn(defrag_context->frag_pool, frag); + frag_count++; + } + + if (frag_count > (uint64_t)SC_ATOMIC_GET(defrag_max_fragments)) { + SC_ATOMIC_SET(defrag_max_fragments, frag_count); } SCMutexUnlock(&defrag_context->frag_pool_lock); From 996996b54474fe0d25a882a5c5047acf978c41ac Mon Sep 17 00:00:00 2001 From: keiahny Date: Tue, 18 Aug 2026 12:18:11 +0200 Subject: [PATCH 3/3] reassembly: added counters for active reassembly objects to stats log Track ra_active_segments counter in stats log for reassembly memcap and pool sizing. Feature: #8438. --- src/stream-tcp-reassemble.c | 20 ++++++++++++++++++++ src/stream-tcp-reassemble.h | 2 ++ 2 files changed, 22 insertions(+) diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c index 1a5b67f431..7f0e354363 100644 --- a/src/stream-tcp-reassemble.c +++ b/src/stream-tcp-reassemble.c @@ -86,6 +86,7 @@ static SCMutex segment_thread_pool_mutex = SCMUTEX_INITIALIZER; /* Memory use counter */ SC_ATOMIC_DECLARE(uint64_t, ra_memuse); +SC_ATOMIC_DECLARE(uint64_t, ra_active_segments); static int g_tcp_session_dump_enabled = 0; @@ -108,6 +109,12 @@ void StreamTcpReassembleInitMemuse(void) SC_ATOMIC_INIT(ra_memuse); } +void StreamTcpReassembleInitActive(void) +{ + SC_ATOMIC_INIT(ra_active_segments); + SC_ATOMIC_SET(ra_active_segments, 0); +} + /** * \brief Function to Increment the memory usage counter for the TCP reassembly * segments @@ -154,6 +161,12 @@ uint64_t StreamTcpReassembleMemuseGlobalCounter(void) return smemuse; } +uint64_t StreamTcpReassembleGetActive(void) +{ + uint64_t ra_active_segmentscopy = SC_ATOMIC_GET(ra_active_segments); + return ra_active_segmentscopy; +} + /** * \brief Function to Check the reassembly memory usage counter against the * allowed max memory usage for TCP segments. @@ -381,6 +394,8 @@ void StreamTcpSegmentReturntoPool(TcpSegment *seg) if (seg == NULL) return; + SC_ATOMIC_SUB(ra_active_segments, 1); + if (seg->pcap_hdr_storage && seg->pcap_hdr_storage->pktlen) { seg->pcap_hdr_storage->pktlen = 0; } @@ -531,6 +546,9 @@ int StreamTcpReassembleInit(bool quiet) #endif StatsRegisterGlobalCounter("tcp.reassembly_memuse", StreamTcpReassembleMemuseGlobalCounter); + + StatsRegisterGlobalCounter("tcp.reassembly_active_segments", StreamTcpReassembleGetActive); + return 0; } @@ -2121,6 +2139,7 @@ TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx) TcpSegment *seg = StreamTcpThreadCacheGetSegment(); if (seg) { StatsCounterIncr(&tv->stats, ra_ctx->counter_tcp_segment_from_cache); + SC_ATOMIC_ADD(ra_active_segments, 1); return seg; } @@ -2134,6 +2153,7 @@ TcpSegment *StreamTcpGetSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx) } else { memset(&seg->sbseg, 0, sizeof(seg->sbseg)); StatsCounterIncr(&tv->stats, ra_ctx->counter_tcp_segment_from_pool); + SC_ATOMIC_ADD(ra_active_segments, 1); } return seg; diff --git a/src/stream-tcp-reassemble.h b/src/stream-tcp-reassemble.h index 2af603bfde..468d832a5e 100644 --- a/src/stream-tcp-reassemble.h +++ b/src/stream-tcp-reassemble.h @@ -91,6 +91,7 @@ typedef struct TcpReassemblyThreadCtx_ { #define OS_POLICY_DEFAULT OS_POLICY_BSD void StreamTcpReassembleInitMemuse(void); +void StreamTcpReassembleInitActive(void); int StreamTcpReassembleHandleSegment( ThreadVars *, TcpReassemblyThreadCtx *, TcpSession *, TcpStream *, Packet *); int StreamTcpReassembleInit(bool); @@ -129,6 +130,7 @@ int StreamTcpReassembleSetMemcap(uint64_t size); uint64_t StreamTcpReassembleGetMemcap(void); int StreamTcpReassembleCheckMemcap(uint64_t size); uint64_t StreamTcpReassembleMemuseGlobalCounter(void); +uint64_t StreamTcpReassembleGetActive(void); void StreamTcpDisableAppLayer(Flow *f); int StreamTcpAppLayerIsDisabled(Flow *f);