You cannot select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
suricata/src/tm-threads.h

172 lines
5.7 KiB
C

Add per packet profiling. Per packet profiling uses tick based accounting. It has 2 outputs, a summary and a csv file that contains per packet stats. Stats per packet include: 1) total ticks spent 2) ticks spent per individual thread module 3) "threading overhead" which is simply calculated by subtracting (2) of (1). A number of changes were made to integrate the new code in a clean way: a number of generic enums are now placed in tm-threads-common.h so we can include them from any part of the engine. Code depends on --enable-profiling just like the rule profiling code. New yaml parameters: profiling: # packet profiling packets: # Profiling can be disabled here, but it will still have a # performance impact if compiled in. enabled: yes filename: packet_stats.log append: yes # per packet csv output csv: # Output can be disabled here, but it will still have a # performance impact if compiled in. enabled: no filename: packet_stats.csv Example output of summary stats: IP ver Proto cnt min max avg ------ ----- ------ ------ ---------- ------- IPv4 6 19436 11448 5404365 32993 IPv4 256 4 11511 49968 30575 Per Thread module stats: Thread Module IP ver Proto cnt min max avg ------------------------ ------ ----- ------ ------ ---------- ------- TMM_DECODEPCAPFILE IPv4 6 19434 1242 47889 1770 TMM_DETECT IPv4 6 19436 1107 137241 1504 TMM_ALERTFASTLOG IPv4 6 19436 90 1323 155 TMM_ALERTUNIFIED2ALERT IPv4 6 19436 108 1359 138 TMM_ALERTDEBUGLOG IPv4 6 19436 90 1134 154 TMM_LOGHTTPLOG IPv4 6 19436 414 5392089 7944 TMM_STREAMTCP IPv4 6 19434 828 1299159 19438 The proto 256 is a counter for handling of pseudo/tunnel packets. Example output of csv: pcap_cnt,ipver,ipproto,total,TMM_DECODENFQ,TMM_VERDICTNFQ,TMM_RECEIVENFQ,TMM_RECEIVEPCAP,TMM_RECEIVEPCAPFILE,TMM_DECODEPCAP,TMM_DECODEPCAPFILE,TMM_RECEIVEPFRING,TMM_DECODEPFRING,TMM_DETECT,TMM_ALERTFASTLOG,TMM_ALERTFASTLOG4,TMM_ALERTFASTLOG6,TMM_ALERTUNIFIEDLOG,TMM_ALERTUNIFIEDALERT,TMM_ALERTUNIFIED2ALERT,TMM_ALERTPRELUDE,TMM_ALERTDEBUGLOG,TMM_ALERTSYSLOG,TMM_LOGDROPLOG,TMM_ALERTSYSLOG4,TMM_ALERTSYSLOG6,TMM_RESPONDREJECT,TMM_LOGHTTPLOG,TMM_LOGHTTPLOG4,TMM_LOGHTTPLOG6,TMM_PCAPLOG,TMM_STREAMTCP,TMM_DECODEIPFW,TMM_VERDICTIPFW,TMM_RECEIVEIPFW,TMM_RECEIVEERFFILE,TMM_DECODEERFFILE,TMM_RECEIVEERFDAG,TMM_DECODEERFDAG,threading 1,4,6,172008,0,0,0,0,0,0,47889,0,0,48582,1323,0,0,0,0,1359,0,1134,0,0,0,0,0,8028,0,0,0,49356,0,0,0,0,0,0,0,14337 First line of the file contains labels. 2 example gnuplot scripts added to plot the data.
14 years ago
/* Copyright (C) 2007-2011 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
* \author Anoop Saldanha <anoopsaldanha@gmail.com>
*/
#ifndef __TM_THREADS_H__
#define __TM_THREADS_H__
#include "tmqh-packetpool.h"
Add per packet profiling. Per packet profiling uses tick based accounting. It has 2 outputs, a summary and a csv file that contains per packet stats. Stats per packet include: 1) total ticks spent 2) ticks spent per individual thread module 3) "threading overhead" which is simply calculated by subtracting (2) of (1). A number of changes were made to integrate the new code in a clean way: a number of generic enums are now placed in tm-threads-common.h so we can include them from any part of the engine. Code depends on --enable-profiling just like the rule profiling code. New yaml parameters: profiling: # packet profiling packets: # Profiling can be disabled here, but it will still have a # performance impact if compiled in. enabled: yes filename: packet_stats.log append: yes # per packet csv output csv: # Output can be disabled here, but it will still have a # performance impact if compiled in. enabled: no filename: packet_stats.csv Example output of summary stats: IP ver Proto cnt min max avg ------ ----- ------ ------ ---------- ------- IPv4 6 19436 11448 5404365 32993 IPv4 256 4 11511 49968 30575 Per Thread module stats: Thread Module IP ver Proto cnt min max avg ------------------------ ------ ----- ------ ------ ---------- ------- TMM_DECODEPCAPFILE IPv4 6 19434 1242 47889 1770 TMM_DETECT IPv4 6 19436 1107 137241 1504 TMM_ALERTFASTLOG IPv4 6 19436 90 1323 155 TMM_ALERTUNIFIED2ALERT IPv4 6 19436 108 1359 138 TMM_ALERTDEBUGLOG IPv4 6 19436 90 1134 154 TMM_LOGHTTPLOG IPv4 6 19436 414 5392089 7944 TMM_STREAMTCP IPv4 6 19434 828 1299159 19438 The proto 256 is a counter for handling of pseudo/tunnel packets. Example output of csv: pcap_cnt,ipver,ipproto,total,TMM_DECODENFQ,TMM_VERDICTNFQ,TMM_RECEIVENFQ,TMM_RECEIVEPCAP,TMM_RECEIVEPCAPFILE,TMM_DECODEPCAP,TMM_DECODEPCAPFILE,TMM_RECEIVEPFRING,TMM_DECODEPFRING,TMM_DETECT,TMM_ALERTFASTLOG,TMM_ALERTFASTLOG4,TMM_ALERTFASTLOG6,TMM_ALERTUNIFIEDLOG,TMM_ALERTUNIFIEDALERT,TMM_ALERTUNIFIED2ALERT,TMM_ALERTPRELUDE,TMM_ALERTDEBUGLOG,TMM_ALERTSYSLOG,TMM_LOGDROPLOG,TMM_ALERTSYSLOG4,TMM_ALERTSYSLOG6,TMM_RESPONDREJECT,TMM_LOGHTTPLOG,TMM_LOGHTTPLOG4,TMM_LOGHTTPLOG6,TMM_PCAPLOG,TMM_STREAMTCP,TMM_DECODEIPFW,TMM_VERDICTIPFW,TMM_RECEIVEIPFW,TMM_RECEIVEERFFILE,TMM_DECODEERFFILE,TMM_RECEIVEERFDAG,TMM_DECODEERFDAG,threading 1,4,6,172008,0,0,0,0,0,0,47889,0,0,48582,1323,0,0,0,0,1359,0,1134,0,0,0,0,0,8028,0,0,0,49356,0,0,0,0,0,0,0,14337 First line of the file contains labels. 2 example gnuplot scripts added to plot the data.
14 years ago
#include "tm-threads-common.h"
#include "tm-modules.h"
typedef struct TmSlot_ {
/* the TV holding this slot */
ThreadVars *tv;
/* function pointers */
TmEcode (*SlotFunc)(ThreadVars *, Packet *, void *, PacketQueue *,
PacketQueue *);
TmEcode (*PktAcqLoop)(ThreadVars *, void *, void *);
TmEcode (*SlotThreadInit)(ThreadVars *, void *, void **);
void (*SlotThreadExitPrintStats)(ThreadVars *, void *);
TmEcode (*SlotThreadDeinit)(ThreadVars *, void *);
/* data storage */
void *slot_initdata;
void *slot_data;
/* queue filled by the SlotFunc with packets that will
* be processed futher _before_ the current packet.
* The locks in the queue are NOT used */
PacketQueue slot_pre_pq;
/* queue filled by the SlotFunc with packets that will
* be processed futher _after_ the current packet. The
* locks in the queue are NOT used */
PacketQueue slot_post_pq;
/* store the thread module id */
int tm_id;
/* slot id, only used my TmVarSlot to know what the first slot is */
int id;
/* linked list, only used when you have multiple slots(used by TmVarSlot) */
struct TmSlot_ *slot_next;
} TmSlot;
extern ThreadVars *tv_root[TVT_MAX];
extern SCMutex tv_root_lock;
void TmSlotSetFuncAppend(ThreadVars *, TmModule *, void *);
TmSlot *TmSlotGetSlotForTM(int);
ThreadVars *TmThreadCreate(char *, char *, char *, char *, char *, char *,
void *(fn_p)(void *), int);
ThreadVars *TmThreadCreatePacketHandler(char *, char *, char *, char *, char *,
char *);
ThreadVars *TmThreadCreateMgmtThread(char *name, void *(fn_p)(void *), int);
TmEcode TmThreadSpawn(ThreadVars *);
void TmThreadSetFlags(ThreadVars *, uint8_t);
void TmThreadSetAOF(ThreadVars *, uint8_t);
void TmThreadKillThread(ThreadVars *);
void TmThreadKillThreads(void);
void TmThreadAppend(ThreadVars *, int);
void TmThreadRemove(ThreadVars *, int);
TmEcode TmThreadSetCPUAffinity(ThreadVars *, uint16_t);
TmEcode TmThreadSetThreadPriority(ThreadVars *, int);
TmEcode TmThreadSetCPU(ThreadVars *, uint8_t);
TmEcode TmThreadSetupOptions(ThreadVars *);
void TmThreadSetPrio(ThreadVars *);
int TmThreadGetNbThreads(uint8_t type);
void TmThreadInitMC(ThreadVars *);
void TmThreadTestThreadUnPaused(ThreadVars *);
void TmThreadContinue(ThreadVars *);
void TmThreadContinueThreads(void);
void TmThreadPause(ThreadVars *);
void TmThreadPauseThreads(void);
void TmThreadCheckThreadState(void);
TmEcode TmThreadWaitOnThreadInit(void);
ThreadVars *TmThreadsGetCallingThread(void);
int TmThreadsCheckFlag(ThreadVars *, uint8_t);
void TmThreadsSetFlag(ThreadVars *, uint8_t);
void TmThreadsUnsetFlag(ThreadVars *, uint8_t);
void TmThreadWaitForFlag(ThreadVars *, uint8_t);
TmEcode TmThreadsSlotVarRun (ThreadVars *tv, Packet *p, TmSlot *slot);
ThreadVars *TmThreadsGetTVContainingSlot(TmSlot *);
void TmThreadDisableReceiveThreads(void);
TmSlot *TmThreadGetFirstTmSlotForPartialPattern(const char *);
/**
* \brief Process the rest of the functions (if any) and queue.
*/
static inline TmEcode TmThreadsSlotProcessPkt(ThreadVars *tv, TmSlot *s, Packet *p)
{
TmEcode r = TM_ECODE_OK;
if (s == NULL) {
tv->tmqh_out(tv, p);
return r;
}
if (TmThreadsSlotVarRun(tv, p, s) == TM_ECODE_FAILED) {
TmqhOutputPacketpool(tv, p);
TmSlot *slot = s;
while (slot != NULL) {
TmqhReleasePacketsToPacketPool(&slot->slot_post_pq);
slot = slot->slot_next;
}
TmThreadsSetFlag(tv, THV_FAILED);
r = TM_ECODE_FAILED;
} else {
tv->tmqh_out(tv, p);
/* post process pq */
TmSlot *slot = s;
while (slot != NULL) {
if (slot->slot_post_pq.top != NULL) {
SCMutexLock(&slot->slot_post_pq.mutex_q);
while (slot->slot_post_pq.top != NULL) {
Packet *extra_p = PacketDequeue(&slot->slot_post_pq);
if (extra_p != NULL) {
if (slot->slot_next != NULL) {
r = TmThreadsSlotVarRun(tv, extra_p, slot->slot_next);
if (r == TM_ECODE_FAILED) {
TmqhReleasePacketsToPacketPool(&slot->slot_post_pq);
TmqhOutputPacketpool(tv, extra_p);
TmThreadsSetFlag(tv, THV_FAILED);
break;
}
}
tv->tmqh_out(tv, extra_p);
}
} /* while (slot->slot_post_pq.top != NULL) */
SCMutexUnlock(&slot->slot_post_pq.mutex_q);
} /* if (slot->slot_post_pq.top != NULL) */
slot = slot->slot_next;
} /* while (slot != NULL) */
}
return r;
}
#endif /* __TM_THREADS_H__ */