mirror of https://github.com/OISF/suricata
Introduce host table, make tag use it
Add a host table similar to the flow table. A hash using fine grained locking. Flow manager for now takes care of book keeping / garbage collecting. Tag subsystem now uses this for host based tagging instead of the global tag hash table. Because the latter used a global lock and the new code uses very fine grained locking this patch should improve scalability.remotes/origin/HEAD
Victor Julien
14 years ago
parent
db24258acf
commit
a05df345de
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,138 @@
|
||||
/* Copyright (C) 2007-2012 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*
|
||||
* Host queue handler functions
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "threads.h"
|
||||
#include "debug.h"
|
||||
#include "host-queue.h"
|
||||
#include "util-error.h"
|
||||
#include "util-debug.h"
|
||||
#include "util-print.h"
|
||||
|
||||
HostQueue *HostQueueInit (HostQueue *q) {
|
||||
if (q != NULL) {
|
||||
memset(q, 0, sizeof(HostQueue));
|
||||
HQLOCK_INIT(q);
|
||||
}
|
||||
return q;
|
||||
}
|
||||
|
||||
HostQueue *HostQueueNew() {
|
||||
HostQueue *q = (HostQueue *)SCMalloc(sizeof(HostQueue));
|
||||
if (q == NULL) {
|
||||
SCLogError(SC_ERR_FATAL, "Fatal error encountered in HostQueueNew. Exiting...");
|
||||
exit(EXIT_SUCCESS);
|
||||
}
|
||||
q = HostQueueInit(q);
|
||||
return q;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Destroy a host queue
|
||||
*
|
||||
* \param q the host queue to destroy
|
||||
*/
|
||||
void HostQueueDestroy (HostQueue *q) {
|
||||
HQLOCK_DESTROY(q);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief add a host to a queue
|
||||
*
|
||||
* \param q queue
|
||||
* \param h host
|
||||
*/
|
||||
void HostEnqueue (HostQueue *q, Host *h) {
|
||||
#ifdef DEBUG
|
||||
BUG_ON(q == NULL || h == NULL);
|
||||
#endif
|
||||
|
||||
HQLOCK_LOCK(q);
|
||||
|
||||
/* more hosts in queue */
|
||||
if (q->top != NULL) {
|
||||
h->lnext = q->top;
|
||||
q->top->lprev = h;
|
||||
q->top = h;
|
||||
/* only host */
|
||||
} else {
|
||||
q->top = h;
|
||||
q->bot = h;
|
||||
}
|
||||
q->len++;
|
||||
#ifdef DBG_PERF
|
||||
if (q->len > q->dbg_maxlen)
|
||||
q->dbg_maxlen = q->len;
|
||||
#endif /* DBG_PERF */
|
||||
HQLOCK_UNLOCK(q);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief remove a host from the queue
|
||||
*
|
||||
* \param q queue
|
||||
*
|
||||
* \retval h host or NULL if empty list.
|
||||
*/
|
||||
Host *HostDequeue (HostQueue *q) {
|
||||
HQLOCK_LOCK(q);
|
||||
|
||||
Host *h = q->bot;
|
||||
if (h == NULL) {
|
||||
HQLOCK_UNLOCK(q);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* more packets in queue */
|
||||
if (q->bot->lprev != NULL) {
|
||||
q->bot = q->bot->lprev;
|
||||
q->bot->lnext = NULL;
|
||||
/* just the one we remove, so now empty */
|
||||
} else {
|
||||
q->top = NULL;
|
||||
q->bot = NULL;
|
||||
}
|
||||
|
||||
#ifdef DEBUG
|
||||
BUG_ON(q->len == 0);
|
||||
#endif
|
||||
if (q->len > 0)
|
||||
q->len--;
|
||||
|
||||
h->lnext = NULL;
|
||||
h->lprev = NULL;
|
||||
|
||||
HQLOCK_UNLOCK(q);
|
||||
return h;
|
||||
}
|
||||
|
||||
uint32_t HostQueueLen(HostQueue *q) {
|
||||
uint32_t len;
|
||||
HQLOCK_LOCK(q);
|
||||
len = q->len;
|
||||
HQLOCK_UNLOCK(q);
|
||||
return len;
|
||||
}
|
||||
|
||||
@ -0,0 +1,84 @@
|
||||
/* Copyright (C) 2007-2012 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#ifndef __HOST_QUEUE_H__
|
||||
#define __HOST_QUEUE_H__
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "host.h"
|
||||
|
||||
/** Spinlocks or Mutex for the host queues. */
|
||||
//#define HQLOCK_SPIN
|
||||
#define HQLOCK_MUTEX
|
||||
|
||||
#ifdef HQLOCK_SPIN
|
||||
#ifdef HQLOCK_MUTEX
|
||||
#error Cannot enable both HQLOCK_SPIN and HQLOCK_MUTEX
|
||||
#endif
|
||||
#endif
|
||||
|
||||
/* Define a queue for storing hosts */
|
||||
typedef struct HostQueue_
|
||||
{
|
||||
Host *top;
|
||||
Host *bot;
|
||||
uint32_t len;
|
||||
#ifdef DBG_PERF
|
||||
uint32_t dbg_maxlen;
|
||||
#endif /* DBG_PERF */
|
||||
#ifdef HQLOCK_MUTEX
|
||||
SCMutex m;
|
||||
#elif defined HQLOCK_SPIN
|
||||
SCSpinlock s;
|
||||
#else
|
||||
#error Enable HQLOCK_SPIN or HQLOCK_MUTEX
|
||||
#endif
|
||||
} HostQueue;
|
||||
|
||||
#ifdef HQLOCK_SPIN
|
||||
#define HQLOCK_INIT(q) SCSpinInit(&(q)->s, 0)
|
||||
#define HQLOCK_DESTROY(q) SCSpinDestroy(&(q)->s)
|
||||
#define HQLOCK_LOCK(q) SCSpinLock(&(q)->s)
|
||||
#define HQLOCK_TRYLOCK(q) SCSpinTrylock(&(q)->s)
|
||||
#define HQLOCK_UNLOCK(q) SCSpinUnlock(&(q)->s)
|
||||
#elif defined HQLOCK_MUTEX
|
||||
#define HQLOCK_INIT(q) SCMutexInit(&(q)->m, NULL)
|
||||
#define HQLOCK_DESTROY(q) SCMutexDestroy(&(q)->m)
|
||||
#define HQLOCK_LOCK(q) SCMutexLock(&(q)->m)
|
||||
#define HQLOCK_TRYLOCK(q) SCMutexTrylock(&(q)->m)
|
||||
#define HQLOCK_UNLOCK(q) SCMutexUnlock(&(q)->m)
|
||||
#else
|
||||
#error Enable HQLOCK_SPIN or HQLOCK_MUTEX
|
||||
#endif
|
||||
|
||||
/* prototypes */
|
||||
HostQueue *HostQueueNew();
|
||||
HostQueue *HostQueueInit(HostQueue *);
|
||||
void HostQueueDestroy (HostQueue *);
|
||||
|
||||
void HostEnqueue (HostQueue *, Host *);
|
||||
Host *HostDequeue (HostQueue *);
|
||||
uint32_t HostQueueLen(HostQueue *);
|
||||
|
||||
#endif /* __HOST_QUEUE_H__ */
|
||||
|
||||
@ -0,0 +1,159 @@
|
||||
/* Copyright (C) 2007-2012 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#include "suricata-common.h"
|
||||
#include "host.h"
|
||||
#include "detect-engine-tag.h"
|
||||
|
||||
uint32_t HostGetSpareCount(void) {
|
||||
return HostSpareQueueGetSize();
|
||||
}
|
||||
|
||||
uint32_t HostGetActiveCount(void) {
|
||||
return SC_ATOMIC_GET(host_counter);
|
||||
}
|
||||
|
||||
/** \internal
|
||||
* \brief See if we can really discard this host. Check use_cnt reference.
|
||||
*
|
||||
* \param h host
|
||||
* \param ts timestamp
|
||||
*
|
||||
* \retval 0 not timed out just yet
|
||||
* \retval 1 fully timed out, lets kill it
|
||||
*/
|
||||
static int HostHostTimedOut(Host *h, struct timeval *ts) {
|
||||
int tags = 0;
|
||||
int thresholds = 0;
|
||||
|
||||
/** never prune a host that is used by a packet
|
||||
* we are currently processing in one of the threads */
|
||||
if (h->use_cnt > 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (h->tag && TagTimeoutCheck(h, ts) == 0) {
|
||||
tags = 1;
|
||||
}
|
||||
if (h->threshold) {
|
||||
// run threshold cleanup
|
||||
}
|
||||
|
||||
if (tags || thresholds)
|
||||
return 0;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* \internal
|
||||
*
|
||||
* \brief check all hosts in a hash row for timing out
|
||||
*
|
||||
* \param hb host hash row *LOCKED*
|
||||
* \param h last host in the hash row
|
||||
* \param ts timestamp
|
||||
*
|
||||
* \retval cnt timed out hosts
|
||||
*/
|
||||
static uint32_t HostHashRowTimeout(HostHashRow *hb, Host *h, struct timeval *ts)
|
||||
{
|
||||
uint32_t cnt = 0;
|
||||
|
||||
do {
|
||||
if (SCMutexTrylock(&h->m) != 0) {
|
||||
h = h->hprev;
|
||||
continue;
|
||||
}
|
||||
|
||||
Host *next_host = h->hprev;
|
||||
|
||||
/* check if the host is fully timed out and
|
||||
* ready to be discarded. */
|
||||
if (HostHostTimedOut(h, ts) == 1) {
|
||||
/* remove from the hash */
|
||||
if (h->hprev != NULL)
|
||||
h->hprev->hnext = h->hnext;
|
||||
if (h->hnext != NULL)
|
||||
h->hnext->hprev = h->hprev;
|
||||
if (hb->head == h)
|
||||
hb->head = h->hnext;
|
||||
if (hb->tail == h)
|
||||
hb->tail = h->hprev;
|
||||
|
||||
h->hnext = NULL;
|
||||
h->hprev = NULL;
|
||||
|
||||
HostClearMemory (h);
|
||||
|
||||
/* no one is referring to this host, use_cnt 0, removed from hash
|
||||
* so we can unlock it and move it back to the spare queue. */
|
||||
SCMutexUnlock(&h->m);
|
||||
|
||||
/* move to spare list */
|
||||
HostMoveToSpare(h);
|
||||
|
||||
cnt++;
|
||||
} else {
|
||||
SCMutexUnlock(&h->m);
|
||||
}
|
||||
|
||||
h = next_host;
|
||||
} while (h != NULL);
|
||||
|
||||
return cnt;
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief time out hosts from the hash
|
||||
*
|
||||
* \param ts timestamp
|
||||
*
|
||||
* \retval cnt number of timed out host
|
||||
*/
|
||||
uint32_t HostTimeoutHash(struct timeval *ts) {
|
||||
uint32_t idx = 0;
|
||||
uint32_t cnt = 0;
|
||||
|
||||
for (idx = 0; idx < host_config.hash_size; idx++) {
|
||||
HostHashRow *hb = &host_hash[idx];
|
||||
if (hb == NULL)
|
||||
continue;
|
||||
if (HRLOCK_TRYLOCK(hb) != 0)
|
||||
continue;
|
||||
|
||||
/* host hash bucket is now locked */
|
||||
|
||||
if (hb->tail == NULL) {
|
||||
HRLOCK_UNLOCK(hb);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* we have a host, or more than one */
|
||||
cnt += HostHashRowTimeout(hb, hb->tail, ts);
|
||||
HRLOCK_UNLOCK(hb);
|
||||
}
|
||||
|
||||
return cnt;
|
||||
}
|
||||
|
||||
@ -0,0 +1,33 @@
|
||||
/* Copyright (C) 2007-2012 Open Information Security Foundation
|
||||
*
|
||||
* You can copy, redistribute or modify this Program under the terms of
|
||||
* the GNU General Public License version 2 as published by the Free
|
||||
* Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* version 2 along with this program; if not, write to the Free Software
|
||||
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||
* 02110-1301, USA.
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file
|
||||
*
|
||||
* \author Victor Julien <victor@inliniac.net>
|
||||
*/
|
||||
|
||||
#ifndef __HOST_TIMEOUT_H__
|
||||
#define __HOST_TIMEOUT_H__
|
||||
|
||||
uint32_t HostTimeoutHash(struct timeval *ts);
|
||||
|
||||
uint32_t HostGetSpareCount(void);
|
||||
uint32_t HostGetActiveCount(void);
|
||||
|
||||
#endif
|
||||
|
||||
Loading…
Reference in New Issue