mirror of https://github.com/OISF/suricata
Introduce host table, make tag use it
Add a host table similar to the flow table. A hash using fine grained locking. Flow manager for now takes care of book keeping / garbage collecting. Tag subsystem now uses this for host based tagging instead of the global tag hash table. Because the latter used a global lock and the new code uses very fine grained locking this patch should improve scalability.remotes/origin/master
Victor Julien
14 years ago
parent
db24258acf
commit
a05df345de
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,138 @@
|
|||||||
|
/* Copyright (C) 2007-2012 Open Information Security Foundation
|
||||||
|
*
|
||||||
|
* You can copy, redistribute or modify this Program under the terms of
|
||||||
|
* the GNU General Public License version 2 as published by the Free
|
||||||
|
* Software Foundation.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* version 2 along with this program; if not, write to the Free Software
|
||||||
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||||
|
* 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \file
|
||||||
|
*
|
||||||
|
* \author Victor Julien <victor@inliniac.net>
|
||||||
|
*
|
||||||
|
* Host queue handler functions
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "suricata-common.h"
|
||||||
|
#include "threads.h"
|
||||||
|
#include "debug.h"
|
||||||
|
#include "host-queue.h"
|
||||||
|
#include "util-error.h"
|
||||||
|
#include "util-debug.h"
|
||||||
|
#include "util-print.h"
|
||||||
|
|
||||||
|
HostQueue *HostQueueInit (HostQueue *q) {
|
||||||
|
if (q != NULL) {
|
||||||
|
memset(q, 0, sizeof(HostQueue));
|
||||||
|
HQLOCK_INIT(q);
|
||||||
|
}
|
||||||
|
return q;
|
||||||
|
}
|
||||||
|
|
||||||
|
HostQueue *HostQueueNew() {
|
||||||
|
HostQueue *q = (HostQueue *)SCMalloc(sizeof(HostQueue));
|
||||||
|
if (q == NULL) {
|
||||||
|
SCLogError(SC_ERR_FATAL, "Fatal error encountered in HostQueueNew. Exiting...");
|
||||||
|
exit(EXIT_SUCCESS);
|
||||||
|
}
|
||||||
|
q = HostQueueInit(q);
|
||||||
|
return q;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief Destroy a host queue
|
||||||
|
*
|
||||||
|
* \param q the host queue to destroy
|
||||||
|
*/
|
||||||
|
void HostQueueDestroy (HostQueue *q) {
|
||||||
|
HQLOCK_DESTROY(q);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief add a host to a queue
|
||||||
|
*
|
||||||
|
* \param q queue
|
||||||
|
* \param h host
|
||||||
|
*/
|
||||||
|
void HostEnqueue (HostQueue *q, Host *h) {
|
||||||
|
#ifdef DEBUG
|
||||||
|
BUG_ON(q == NULL || h == NULL);
|
||||||
|
#endif
|
||||||
|
|
||||||
|
HQLOCK_LOCK(q);
|
||||||
|
|
||||||
|
/* more hosts in queue */
|
||||||
|
if (q->top != NULL) {
|
||||||
|
h->lnext = q->top;
|
||||||
|
q->top->lprev = h;
|
||||||
|
q->top = h;
|
||||||
|
/* only host */
|
||||||
|
} else {
|
||||||
|
q->top = h;
|
||||||
|
q->bot = h;
|
||||||
|
}
|
||||||
|
q->len++;
|
||||||
|
#ifdef DBG_PERF
|
||||||
|
if (q->len > q->dbg_maxlen)
|
||||||
|
q->dbg_maxlen = q->len;
|
||||||
|
#endif /* DBG_PERF */
|
||||||
|
HQLOCK_UNLOCK(q);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief remove a host from the queue
|
||||||
|
*
|
||||||
|
* \param q queue
|
||||||
|
*
|
||||||
|
* \retval h host or NULL if empty list.
|
||||||
|
*/
|
||||||
|
Host *HostDequeue (HostQueue *q) {
|
||||||
|
HQLOCK_LOCK(q);
|
||||||
|
|
||||||
|
Host *h = q->bot;
|
||||||
|
if (h == NULL) {
|
||||||
|
HQLOCK_UNLOCK(q);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* more packets in queue */
|
||||||
|
if (q->bot->lprev != NULL) {
|
||||||
|
q->bot = q->bot->lprev;
|
||||||
|
q->bot->lnext = NULL;
|
||||||
|
/* just the one we remove, so now empty */
|
||||||
|
} else {
|
||||||
|
q->top = NULL;
|
||||||
|
q->bot = NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
#ifdef DEBUG
|
||||||
|
BUG_ON(q->len == 0);
|
||||||
|
#endif
|
||||||
|
if (q->len > 0)
|
||||||
|
q->len--;
|
||||||
|
|
||||||
|
h->lnext = NULL;
|
||||||
|
h->lprev = NULL;
|
||||||
|
|
||||||
|
HQLOCK_UNLOCK(q);
|
||||||
|
return h;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t HostQueueLen(HostQueue *q) {
|
||||||
|
uint32_t len;
|
||||||
|
HQLOCK_LOCK(q);
|
||||||
|
len = q->len;
|
||||||
|
HQLOCK_UNLOCK(q);
|
||||||
|
return len;
|
||||||
|
}
|
||||||
|
|
||||||
@ -0,0 +1,84 @@
|
|||||||
|
/* Copyright (C) 2007-2012 Open Information Security Foundation
|
||||||
|
*
|
||||||
|
* You can copy, redistribute or modify this Program under the terms of
|
||||||
|
* the GNU General Public License version 2 as published by the Free
|
||||||
|
* Software Foundation.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* version 2 along with this program; if not, write to the Free Software
|
||||||
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||||
|
* 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \file
|
||||||
|
*
|
||||||
|
* \author Victor Julien <victor@inliniac.net>
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef __HOST_QUEUE_H__
|
||||||
|
#define __HOST_QUEUE_H__
|
||||||
|
|
||||||
|
#include "suricata-common.h"
|
||||||
|
#include "host.h"
|
||||||
|
|
||||||
|
/** Spinlocks or Mutex for the host queues. */
|
||||||
|
//#define HQLOCK_SPIN
|
||||||
|
#define HQLOCK_MUTEX
|
||||||
|
|
||||||
|
#ifdef HQLOCK_SPIN
|
||||||
|
#ifdef HQLOCK_MUTEX
|
||||||
|
#error Cannot enable both HQLOCK_SPIN and HQLOCK_MUTEX
|
||||||
|
#endif
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Define a queue for storing hosts */
|
||||||
|
typedef struct HostQueue_
|
||||||
|
{
|
||||||
|
Host *top;
|
||||||
|
Host *bot;
|
||||||
|
uint32_t len;
|
||||||
|
#ifdef DBG_PERF
|
||||||
|
uint32_t dbg_maxlen;
|
||||||
|
#endif /* DBG_PERF */
|
||||||
|
#ifdef HQLOCK_MUTEX
|
||||||
|
SCMutex m;
|
||||||
|
#elif defined HQLOCK_SPIN
|
||||||
|
SCSpinlock s;
|
||||||
|
#else
|
||||||
|
#error Enable HQLOCK_SPIN or HQLOCK_MUTEX
|
||||||
|
#endif
|
||||||
|
} HostQueue;
|
||||||
|
|
||||||
|
#ifdef HQLOCK_SPIN
|
||||||
|
#define HQLOCK_INIT(q) SCSpinInit(&(q)->s, 0)
|
||||||
|
#define HQLOCK_DESTROY(q) SCSpinDestroy(&(q)->s)
|
||||||
|
#define HQLOCK_LOCK(q) SCSpinLock(&(q)->s)
|
||||||
|
#define HQLOCK_TRYLOCK(q) SCSpinTrylock(&(q)->s)
|
||||||
|
#define HQLOCK_UNLOCK(q) SCSpinUnlock(&(q)->s)
|
||||||
|
#elif defined HQLOCK_MUTEX
|
||||||
|
#define HQLOCK_INIT(q) SCMutexInit(&(q)->m, NULL)
|
||||||
|
#define HQLOCK_DESTROY(q) SCMutexDestroy(&(q)->m)
|
||||||
|
#define HQLOCK_LOCK(q) SCMutexLock(&(q)->m)
|
||||||
|
#define HQLOCK_TRYLOCK(q) SCMutexTrylock(&(q)->m)
|
||||||
|
#define HQLOCK_UNLOCK(q) SCMutexUnlock(&(q)->m)
|
||||||
|
#else
|
||||||
|
#error Enable HQLOCK_SPIN or HQLOCK_MUTEX
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* prototypes */
|
||||||
|
HostQueue *HostQueueNew();
|
||||||
|
HostQueue *HostQueueInit(HostQueue *);
|
||||||
|
void HostQueueDestroy (HostQueue *);
|
||||||
|
|
||||||
|
void HostEnqueue (HostQueue *, Host *);
|
||||||
|
Host *HostDequeue (HostQueue *);
|
||||||
|
uint32_t HostQueueLen(HostQueue *);
|
||||||
|
|
||||||
|
#endif /* __HOST_QUEUE_H__ */
|
||||||
|
|
||||||
@ -0,0 +1,159 @@
|
|||||||
|
/* Copyright (C) 2007-2012 Open Information Security Foundation
|
||||||
|
*
|
||||||
|
* You can copy, redistribute or modify this Program under the terms of
|
||||||
|
* the GNU General Public License version 2 as published by the Free
|
||||||
|
* Software Foundation.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* version 2 along with this program; if not, write to the Free Software
|
||||||
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||||
|
* 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \file
|
||||||
|
*
|
||||||
|
* \author Victor Julien <victor@inliniac.net>
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "suricata-common.h"
|
||||||
|
#include "host.h"
|
||||||
|
#include "detect-engine-tag.h"
|
||||||
|
|
||||||
|
uint32_t HostGetSpareCount(void) {
|
||||||
|
return HostSpareQueueGetSize();
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t HostGetActiveCount(void) {
|
||||||
|
return SC_ATOMIC_GET(host_counter);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** \internal
|
||||||
|
* \brief See if we can really discard this host. Check use_cnt reference.
|
||||||
|
*
|
||||||
|
* \param h host
|
||||||
|
* \param ts timestamp
|
||||||
|
*
|
||||||
|
* \retval 0 not timed out just yet
|
||||||
|
* \retval 1 fully timed out, lets kill it
|
||||||
|
*/
|
||||||
|
static int HostHostTimedOut(Host *h, struct timeval *ts) {
|
||||||
|
int tags = 0;
|
||||||
|
int thresholds = 0;
|
||||||
|
|
||||||
|
/** never prune a host that is used by a packet
|
||||||
|
* we are currently processing in one of the threads */
|
||||||
|
if (h->use_cnt > 0) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (h->tag && TagTimeoutCheck(h, ts) == 0) {
|
||||||
|
tags = 1;
|
||||||
|
}
|
||||||
|
if (h->threshold) {
|
||||||
|
// run threshold cleanup
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tags || thresholds)
|
||||||
|
return 0;
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \internal
|
||||||
|
*
|
||||||
|
* \brief check all hosts in a hash row for timing out
|
||||||
|
*
|
||||||
|
* \param hb host hash row *LOCKED*
|
||||||
|
* \param h last host in the hash row
|
||||||
|
* \param ts timestamp
|
||||||
|
*
|
||||||
|
* \retval cnt timed out hosts
|
||||||
|
*/
|
||||||
|
static uint32_t HostHashRowTimeout(HostHashRow *hb, Host *h, struct timeval *ts)
|
||||||
|
{
|
||||||
|
uint32_t cnt = 0;
|
||||||
|
|
||||||
|
do {
|
||||||
|
if (SCMutexTrylock(&h->m) != 0) {
|
||||||
|
h = h->hprev;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
Host *next_host = h->hprev;
|
||||||
|
|
||||||
|
/* check if the host is fully timed out and
|
||||||
|
* ready to be discarded. */
|
||||||
|
if (HostHostTimedOut(h, ts) == 1) {
|
||||||
|
/* remove from the hash */
|
||||||
|
if (h->hprev != NULL)
|
||||||
|
h->hprev->hnext = h->hnext;
|
||||||
|
if (h->hnext != NULL)
|
||||||
|
h->hnext->hprev = h->hprev;
|
||||||
|
if (hb->head == h)
|
||||||
|
hb->head = h->hnext;
|
||||||
|
if (hb->tail == h)
|
||||||
|
hb->tail = h->hprev;
|
||||||
|
|
||||||
|
h->hnext = NULL;
|
||||||
|
h->hprev = NULL;
|
||||||
|
|
||||||
|
HostClearMemory (h);
|
||||||
|
|
||||||
|
/* no one is referring to this host, use_cnt 0, removed from hash
|
||||||
|
* so we can unlock it and move it back to the spare queue. */
|
||||||
|
SCMutexUnlock(&h->m);
|
||||||
|
|
||||||
|
/* move to spare list */
|
||||||
|
HostMoveToSpare(h);
|
||||||
|
|
||||||
|
cnt++;
|
||||||
|
} else {
|
||||||
|
SCMutexUnlock(&h->m);
|
||||||
|
}
|
||||||
|
|
||||||
|
h = next_host;
|
||||||
|
} while (h != NULL);
|
||||||
|
|
||||||
|
return cnt;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \brief time out hosts from the hash
|
||||||
|
*
|
||||||
|
* \param ts timestamp
|
||||||
|
*
|
||||||
|
* \retval cnt number of timed out host
|
||||||
|
*/
|
||||||
|
uint32_t HostTimeoutHash(struct timeval *ts) {
|
||||||
|
uint32_t idx = 0;
|
||||||
|
uint32_t cnt = 0;
|
||||||
|
|
||||||
|
for (idx = 0; idx < host_config.hash_size; idx++) {
|
||||||
|
HostHashRow *hb = &host_hash[idx];
|
||||||
|
if (hb == NULL)
|
||||||
|
continue;
|
||||||
|
if (HRLOCK_TRYLOCK(hb) != 0)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
/* host hash bucket is now locked */
|
||||||
|
|
||||||
|
if (hb->tail == NULL) {
|
||||||
|
HRLOCK_UNLOCK(hb);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* we have a host, or more than one */
|
||||||
|
cnt += HostHashRowTimeout(hb, hb->tail, ts);
|
||||||
|
HRLOCK_UNLOCK(hb);
|
||||||
|
}
|
||||||
|
|
||||||
|
return cnt;
|
||||||
|
}
|
||||||
|
|
||||||
@ -0,0 +1,33 @@
|
|||||||
|
/* Copyright (C) 2007-2012 Open Information Security Foundation
|
||||||
|
*
|
||||||
|
* You can copy, redistribute or modify this Program under the terms of
|
||||||
|
* the GNU General Public License version 2 as published by the Free
|
||||||
|
* Software Foundation.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* version 2 along with this program; if not, write to the Free Software
|
||||||
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
||||||
|
* 02110-1301, USA.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* \file
|
||||||
|
*
|
||||||
|
* \author Victor Julien <victor@inliniac.net>
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef __HOST_TIMEOUT_H__
|
||||||
|
#define __HOST_TIMEOUT_H__
|
||||||
|
|
||||||
|
uint32_t HostTimeoutHash(struct timeval *ts);
|
||||||
|
|
||||||
|
uint32_t HostGetSpareCount(void);
|
||||||
|
uint32_t HostGetActiveCount(void);
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
||||||
Loading…
Reference in New Issue