Add per packet profiling.
Per packet profiling uses tick based accounting. It has 2 outputs, a summary
and a csv file that contains per packet stats.
Stats per packet include:
1) total ticks spent
2) ticks spent per individual thread module
3) "threading overhead" which is simply calculated by subtracting (2) of (1).
A number of changes were made to integrate the new code in a clean way:
a number of generic enums are now placed in tm-threads-common.h so we can
include them from any part of the engine.
Code depends on --enable-profiling just like the rule profiling code.
New yaml parameters:
profiling:
# packet profiling
packets:
# Profiling can be disabled here, but it will still have a
# performance impact if compiled in.
enabled: yes
filename: packet_stats.log
append: yes
# per packet csv output
csv:
# Output can be disabled here, but it will still have a
# performance impact if compiled in.
enabled: no
filename: packet_stats.csv
Example output of summary stats:
IP ver Proto cnt min max avg
------ ----- ------ ------ ---------- -------
IPv4 6 19436 11448 5404365 32993
IPv4 256 4 11511 49968 30575
Per Thread module stats:
Thread Module IP ver Proto cnt min max avg
------------------------ ------ ----- ------ ------ ---------- -------
TMM_DECODEPCAPFILE IPv4 6 19434 1242 47889 1770
TMM_DETECT IPv4 6 19436 1107 137241 1504
TMM_ALERTFASTLOG IPv4 6 19436 90 1323 155
TMM_ALERTUNIFIED2ALERT IPv4 6 19436 108 1359 138
TMM_ALERTDEBUGLOG IPv4 6 19436 90 1134 154
TMM_LOGHTTPLOG IPv4 6 19436 414 5392089 7944
TMM_STREAMTCP IPv4 6 19434 828 1299159 19438
The proto 256 is a counter for handling of pseudo/tunnel packets.
Example output of csv:
pcap_cnt,ipver,ipproto,total,TMM_DECODENFQ,TMM_VERDICTNFQ,TMM_RECEIVENFQ,TMM_RECEIVEPCAP,TMM_RECEIVEPCAPFILE,TMM_DECODEPCAP,TMM_DECODEPCAPFILE,TMM_RECEIVEPFRING,TMM_DECODEPFRING,TMM_DETECT,TMM_ALERTFASTLOG,TMM_ALERTFASTLOG4,TMM_ALERTFASTLOG6,TMM_ALERTUNIFIEDLOG,TMM_ALERTUNIFIEDALERT,TMM_ALERTUNIFIED2ALERT,TMM_ALERTPRELUDE,TMM_ALERTDEBUGLOG,TMM_ALERTSYSLOG,TMM_LOGDROPLOG,TMM_ALERTSYSLOG4,TMM_ALERTSYSLOG6,TMM_RESPONDREJECT,TMM_LOGHTTPLOG,TMM_LOGHTTPLOG4,TMM_LOGHTTPLOG6,TMM_PCAPLOG,TMM_STREAMTCP,TMM_DECODEIPFW,TMM_VERDICTIPFW,TMM_RECEIVEIPFW,TMM_RECEIVEERFFILE,TMM_DECODEERFFILE,TMM_RECEIVEERFDAG,TMM_DECODEERFDAG,threading
1,4,6,172008,0,0,0,0,0,0,47889,0,0,48582,1323,0,0,0,0,1359,0,1134,0,0,0,0,0,8028,0,0,0,49356,0,0,0,0,0,0,0,14337
First line of the file contains labels.
2 example gnuplot scripts added to plot the data.
14 years ago
|
|
|
/* Copyright (C) 2007-2011 Open Information Security Foundation
|
|
|
|
|
*
|
|
|
|
|
* You can copy, redistribute or modify this Program under the terms of
|
|
|
|
|
* the GNU General Public License version 2 as published by the Free
|
|
|
|
|
* Software Foundation.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
|
* version 2 along with this program; if not, write to the Free Software
|
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
|
|
|
* 02110-1301, USA.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* \file
|
|
|
|
|
*
|
|
|
|
|
* \author Victor Julien <victor@inliniac.net>
|
|
|
|
|
* \author Anoop Saldanha <poonaatsoc@gmail.com>
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#ifndef __TM_THREADS_H__
|
|
|
|
|
#define __TM_THREADS_H__
|
|
|
|
|
|
Add per packet profiling.
Per packet profiling uses tick based accounting. It has 2 outputs, a summary
and a csv file that contains per packet stats.
Stats per packet include:
1) total ticks spent
2) ticks spent per individual thread module
3) "threading overhead" which is simply calculated by subtracting (2) of (1).
A number of changes were made to integrate the new code in a clean way:
a number of generic enums are now placed in tm-threads-common.h so we can
include them from any part of the engine.
Code depends on --enable-profiling just like the rule profiling code.
New yaml parameters:
profiling:
# packet profiling
packets:
# Profiling can be disabled here, but it will still have a
# performance impact if compiled in.
enabled: yes
filename: packet_stats.log
append: yes
# per packet csv output
csv:
# Output can be disabled here, but it will still have a
# performance impact if compiled in.
enabled: no
filename: packet_stats.csv
Example output of summary stats:
IP ver Proto cnt min max avg
------ ----- ------ ------ ---------- -------
IPv4 6 19436 11448 5404365 32993
IPv4 256 4 11511 49968 30575
Per Thread module stats:
Thread Module IP ver Proto cnt min max avg
------------------------ ------ ----- ------ ------ ---------- -------
TMM_DECODEPCAPFILE IPv4 6 19434 1242 47889 1770
TMM_DETECT IPv4 6 19436 1107 137241 1504
TMM_ALERTFASTLOG IPv4 6 19436 90 1323 155
TMM_ALERTUNIFIED2ALERT IPv4 6 19436 108 1359 138
TMM_ALERTDEBUGLOG IPv4 6 19436 90 1134 154
TMM_LOGHTTPLOG IPv4 6 19436 414 5392089 7944
TMM_STREAMTCP IPv4 6 19434 828 1299159 19438
The proto 256 is a counter for handling of pseudo/tunnel packets.
Example output of csv:
pcap_cnt,ipver,ipproto,total,TMM_DECODENFQ,TMM_VERDICTNFQ,TMM_RECEIVENFQ,TMM_RECEIVEPCAP,TMM_RECEIVEPCAPFILE,TMM_DECODEPCAP,TMM_DECODEPCAPFILE,TMM_RECEIVEPFRING,TMM_DECODEPFRING,TMM_DETECT,TMM_ALERTFASTLOG,TMM_ALERTFASTLOG4,TMM_ALERTFASTLOG6,TMM_ALERTUNIFIEDLOG,TMM_ALERTUNIFIEDALERT,TMM_ALERTUNIFIED2ALERT,TMM_ALERTPRELUDE,TMM_ALERTDEBUGLOG,TMM_ALERTSYSLOG,TMM_LOGDROPLOG,TMM_ALERTSYSLOG4,TMM_ALERTSYSLOG6,TMM_RESPONDREJECT,TMM_LOGHTTPLOG,TMM_LOGHTTPLOG4,TMM_LOGHTTPLOG6,TMM_PCAPLOG,TMM_STREAMTCP,TMM_DECODEIPFW,TMM_VERDICTIPFW,TMM_RECEIVEIPFW,TMM_RECEIVEERFFILE,TMM_DECODEERFFILE,TMM_RECEIVEERFDAG,TMM_DECODEERFDAG,threading
1,4,6,172008,0,0,0,0,0,0,47889,0,0,48582,1323,0,0,0,0,1359,0,1134,0,0,0,0,0,8028,0,0,0,49356,0,0,0,0,0,0,0,14337
First line of the file contains labels.
2 example gnuplot scripts added to plot the data.
14 years ago
|
|
|
#include "tm-threads-common.h"
|
|
|
|
|
#include "tm-modules.h"
|
|
|
|
|
|
|
|
|
|
typedef struct TmSlot_ {
|
|
|
|
|
/* function pointers */
|
|
|
|
|
TmEcode (*SlotFunc)(ThreadVars *, Packet *, void *, PacketQueue *,
|
|
|
|
|
PacketQueue *);
|
|
|
|
|
|
|
|
|
|
TmEcode (*PktAcqLoop)(ThreadVars *, void *, void *);
|
|
|
|
|
|
|
|
|
|
TmEcode (*SlotThreadInit)(ThreadVars *, void *, void **);
|
|
|
|
|
void (*SlotThreadExitPrintStats)(ThreadVars *, void *);
|
|
|
|
|
TmEcode (*SlotThreadDeinit)(ThreadVars *, void *);
|
|
|
|
|
|
|
|
|
|
/* data storage */
|
|
|
|
|
void *slot_initdata;
|
|
|
|
|
void *slot_data;
|
|
|
|
|
|
|
|
|
|
/* queue filled by the SlotFunc with packets that will
|
|
|
|
|
* be processed futher _before_ the current packet.
|
|
|
|
|
* The locks in the queue are NOT used */
|
|
|
|
|
PacketQueue slot_pre_pq;
|
|
|
|
|
|
|
|
|
|
/* queue filled by the SlotFunc with packets that will
|
|
|
|
|
* be processed futher _after_ the current packet. The
|
|
|
|
|
* locks in the queue are NOT used */
|
|
|
|
|
PacketQueue slot_post_pq;
|
|
|
|
|
|
|
|
|
|
/* slot id, only used my TmVarSlot to know what the first slot is */
|
|
|
|
|
int id;
|
|
|
|
|
|
|
|
|
|
/* linked list, only used when you have multiple slots(used by TmVarSlot) */
|
|
|
|
|
struct TmSlot_ *slot_next;
|
Add per packet profiling.
Per packet profiling uses tick based accounting. It has 2 outputs, a summary
and a csv file that contains per packet stats.
Stats per packet include:
1) total ticks spent
2) ticks spent per individual thread module
3) "threading overhead" which is simply calculated by subtracting (2) of (1).
A number of changes were made to integrate the new code in a clean way:
a number of generic enums are now placed in tm-threads-common.h so we can
include them from any part of the engine.
Code depends on --enable-profiling just like the rule profiling code.
New yaml parameters:
profiling:
# packet profiling
packets:
# Profiling can be disabled here, but it will still have a
# performance impact if compiled in.
enabled: yes
filename: packet_stats.log
append: yes
# per packet csv output
csv:
# Output can be disabled here, but it will still have a
# performance impact if compiled in.
enabled: no
filename: packet_stats.csv
Example output of summary stats:
IP ver Proto cnt min max avg
------ ----- ------ ------ ---------- -------
IPv4 6 19436 11448 5404365 32993
IPv4 256 4 11511 49968 30575
Per Thread module stats:
Thread Module IP ver Proto cnt min max avg
------------------------ ------ ----- ------ ------ ---------- -------
TMM_DECODEPCAPFILE IPv4 6 19434 1242 47889 1770
TMM_DETECT IPv4 6 19436 1107 137241 1504
TMM_ALERTFASTLOG IPv4 6 19436 90 1323 155
TMM_ALERTUNIFIED2ALERT IPv4 6 19436 108 1359 138
TMM_ALERTDEBUGLOG IPv4 6 19436 90 1134 154
TMM_LOGHTTPLOG IPv4 6 19436 414 5392089 7944
TMM_STREAMTCP IPv4 6 19434 828 1299159 19438
The proto 256 is a counter for handling of pseudo/tunnel packets.
Example output of csv:
pcap_cnt,ipver,ipproto,total,TMM_DECODENFQ,TMM_VERDICTNFQ,TMM_RECEIVENFQ,TMM_RECEIVEPCAP,TMM_RECEIVEPCAPFILE,TMM_DECODEPCAP,TMM_DECODEPCAPFILE,TMM_RECEIVEPFRING,TMM_DECODEPFRING,TMM_DETECT,TMM_ALERTFASTLOG,TMM_ALERTFASTLOG4,TMM_ALERTFASTLOG6,TMM_ALERTUNIFIEDLOG,TMM_ALERTUNIFIEDALERT,TMM_ALERTUNIFIED2ALERT,TMM_ALERTPRELUDE,TMM_ALERTDEBUGLOG,TMM_ALERTSYSLOG,TMM_LOGDROPLOG,TMM_ALERTSYSLOG4,TMM_ALERTSYSLOG6,TMM_RESPONDREJECT,TMM_LOGHTTPLOG,TMM_LOGHTTPLOG4,TMM_LOGHTTPLOG6,TMM_PCAPLOG,TMM_STREAMTCP,TMM_DECODEIPFW,TMM_VERDICTIPFW,TMM_RECEIVEIPFW,TMM_RECEIVEERFFILE,TMM_DECODEERFFILE,TMM_RECEIVEERFDAG,TMM_DECODEERFDAG,threading
1,4,6,172008,0,0,0,0,0,0,47889,0,0,48582,1323,0,0,0,0,1359,0,1134,0,0,0,0,0,8028,0,0,0,49356,0,0,0,0,0,0,0,14337
First line of the file contains labels.
2 example gnuplot scripts added to plot the data.
14 years ago
|
|
|
|
|
|
|
|
#ifdef PROFILING
|
|
|
|
|
int tm_module_id;
|
|
|
|
|
#endif
|
|
|
|
|
} TmSlot;
|
|
|
|
|
|
|
|
|
|
extern ThreadVars *tv_root[TVT_MAX];
|
|
|
|
|
|
|
|
|
|
extern SCMutex tv_root_lock;
|
|
|
|
|
|
|
|
|
|
void TmSlotSetFuncAppend(ThreadVars *, TmModule *, void *);
|
|
|
|
|
|
|
|
|
|
ThreadVars *TmThreadCreate(char *, char *, char *, char *, char *, char *,
|
|
|
|
|
void *(fn_p)(void *), int);
|
|
|
|
|
ThreadVars *TmThreadCreatePacketHandler(char *, char *, char *, char *, char *,
|
|
|
|
|
char *);
|
|
|
|
|
ThreadVars *TmThreadCreateMgmtThread(char *name, void *(fn_p)(void *), int);
|
|
|
|
|
TmEcode TmThreadSpawn(ThreadVars *);
|
|
|
|
|
void TmThreadSetFlags(ThreadVars *, uint8_t);
|
|
|
|
|
void TmThreadSetAOF(ThreadVars *, uint8_t);
|
|
|
|
|
void TmThreadKillThread(ThreadVars *);
|
|
|
|
|
void TmThreadKillThreads(void);
|
|
|
|
|
void TmThreadAppend(ThreadVars *, int);
|
|
|
|
|
void TmThreadRemove(ThreadVars *, int);
|
|
|
|
|
|
|
|
|
|
TmEcode TmThreadsSlotProcessPkt(ThreadVars *, TmSlot *, Packet *);
|
|
|
|
|
|
|
|
|
|
TmEcode TmThreadSetCPUAffinity(ThreadVars *, uint16_t);
|
|
|
|
|
TmEcode TmThreadSetThreadPriority(ThreadVars *, int);
|
|
|
|
|
TmEcode TmThreadSetCPU(ThreadVars *, uint8_t);
|
|
|
|
|
TmEcode TmThreadSetupOptions(ThreadVars *);
|
|
|
|
|
void TmThreadSetPrio(ThreadVars *);
|
|
|
|
|
int TmThreadGetNbThreads(uint8_t type);
|
|
|
|
|
|
|
|
|
|
void TmThreadInitMC(ThreadVars *);
|
|
|
|
|
void TmThreadTestThreadUnPaused(ThreadVars *);
|
|
|
|
|
void TmThreadContinue(ThreadVars *);
|
|
|
|
|
void TmThreadContinueThreads(void);
|
|
|
|
|
void TmThreadPause(ThreadVars *);
|
|
|
|
|
void TmThreadPauseThreads(void);
|
|
|
|
|
void TmThreadCheckThreadState(void);
|
|
|
|
|
TmEcode TmThreadWaitOnThreadInit(void);
|
|
|
|
|
ThreadVars *TmThreadsGetCallingThread(void);
|
|
|
|
|
|
|
|
|
|
int TmThreadsCheckFlag(ThreadVars *, uint8_t);
|
|
|
|
|
void TmThreadsSetFlag(ThreadVars *, uint8_t);
|
|
|
|
|
void TmThreadsUnsetFlag(ThreadVars *, uint8_t);
|
|
|
|
|
|
|
|
|
|
TmEcode TmThreadsSlotVarRun (ThreadVars *tv, Packet *p, TmSlot *slot);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* \brief Process the rest of the functions (if any) and queue.
|
|
|
|
|
*/
|
|
|
|
|
#define TmThreadsSlotProcessPkt(tv, s, p) ({ \
|
|
|
|
|
TmEcode r = TM_ECODE_OK; \
|
|
|
|
|
\
|
|
|
|
|
if ((s) != NULL && \
|
|
|
|
|
TmThreadsSlotVarRun((tv), (p), (s)) == TM_ECODE_FAILED) { \
|
|
|
|
|
TmqhOutputPacketpool((tv), (p)); \
|
|
|
|
|
TmThreadsSetFlag((tv), THV_FAILED); \
|
|
|
|
|
r = TM_ECODE_FAILED; \
|
|
|
|
|
} else { \
|
|
|
|
|
tv->tmqh_out(tv, p); \
|
|
|
|
|
} \
|
|
|
|
|
\
|
|
|
|
|
r; \
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
#endif /* __TM_THREADS_H__ */
|
