|
|
|
/* Copyright (C) 2007-2010 Open Information Security Foundation
|
|
|
|
*
|
|
|
|
* You can copy, redistribute or modify this Program under the terms of
|
|
|
|
* the GNU General Public License version 2 as published by the Free
|
|
|
|
* Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* version 2 along with this program; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
|
|
* 02110-1301, USA.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* \file
|
|
|
|
*
|
|
|
|
* \author Endace Technology Limited, Jason Ish <jason.ish@endace.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef __OUTPUT_H__
|
|
|
|
#define __OUTPUT_H__
|
|
|
|
|
|
|
|
#include "suricata.h"
|
Add per packet profiling.
Per packet profiling uses tick based accounting. It has 2 outputs, a summary
and a csv file that contains per packet stats.
Stats per packet include:
1) total ticks spent
2) ticks spent per individual thread module
3) "threading overhead" which is simply calculated by subtracting (2) of (1).
A number of changes were made to integrate the new code in a clean way:
a number of generic enums are now placed in tm-threads-common.h so we can
include them from any part of the engine.
Code depends on --enable-profiling just like the rule profiling code.
New yaml parameters:
profiling:
# packet profiling
packets:
# Profiling can be disabled here, but it will still have a
# performance impact if compiled in.
enabled: yes
filename: packet_stats.log
append: yes
# per packet csv output
csv:
# Output can be disabled here, but it will still have a
# performance impact if compiled in.
enabled: no
filename: packet_stats.csv
Example output of summary stats:
IP ver Proto cnt min max avg
------ ----- ------ ------ ---------- -------
IPv4 6 19436 11448 5404365 32993
IPv4 256 4 11511 49968 30575
Per Thread module stats:
Thread Module IP ver Proto cnt min max avg
------------------------ ------ ----- ------ ------ ---------- -------
TMM_DECODEPCAPFILE IPv4 6 19434 1242 47889 1770
TMM_DETECT IPv4 6 19436 1107 137241 1504
TMM_ALERTFASTLOG IPv4 6 19436 90 1323 155
TMM_ALERTUNIFIED2ALERT IPv4 6 19436 108 1359 138
TMM_ALERTDEBUGLOG IPv4 6 19436 90 1134 154
TMM_LOGHTTPLOG IPv4 6 19436 414 5392089 7944
TMM_STREAMTCP IPv4 6 19434 828 1299159 19438
The proto 256 is a counter for handling of pseudo/tunnel packets.
Example output of csv:
pcap_cnt,ipver,ipproto,total,TMM_DECODENFQ,TMM_VERDICTNFQ,TMM_RECEIVENFQ,TMM_RECEIVEPCAP,TMM_RECEIVEPCAPFILE,TMM_DECODEPCAP,TMM_DECODEPCAPFILE,TMM_RECEIVEPFRING,TMM_DECODEPFRING,TMM_DETECT,TMM_ALERTFASTLOG,TMM_ALERTFASTLOG4,TMM_ALERTFASTLOG6,TMM_ALERTUNIFIEDLOG,TMM_ALERTUNIFIEDALERT,TMM_ALERTUNIFIED2ALERT,TMM_ALERTPRELUDE,TMM_ALERTDEBUGLOG,TMM_ALERTSYSLOG,TMM_LOGDROPLOG,TMM_ALERTSYSLOG4,TMM_ALERTSYSLOG6,TMM_RESPONDREJECT,TMM_LOGHTTPLOG,TMM_LOGHTTPLOG4,TMM_LOGHTTPLOG6,TMM_PCAPLOG,TMM_STREAMTCP,TMM_DECODEIPFW,TMM_VERDICTIPFW,TMM_RECEIVEIPFW,TMM_RECEIVEERFFILE,TMM_DECODEERFFILE,TMM_RECEIVEERFDAG,TMM_DECODEERFDAG,threading
1,4,6,172008,0,0,0,0,0,0,47889,0,0,48582,1323,0,0,0,0,1359,0,1134,0,0,0,0,0,8028,0,0,0,49356,0,0,0,0,0,0,0,14337
First line of the file contains labels.
2 example gnuplot scripts added to plot the data.
14 years ago
|
|
|
#include "tm-threads.h"
|
|
|
|
|
|
|
|
#define DEFAULT_LOG_MODE_APPEND "yes"
|
|
|
|
#define DEFAULT_LOG_FILETYPE "regular"
|
|
|
|
|
|
|
|
#include "output-packet.h"
|
|
|
|
#include "output-tx.h"
|
|
|
|
#include "output-file.h"
|
|
|
|
#include "output-filedata.h"
|
|
|
|
#include "output-flow.h"
|
|
|
|
#include "output-streaming.h"
|
|
|
|
#include "output-stats.h"
|
|
|
|
|
|
|
|
typedef struct OutputModule_ {
|
|
|
|
const char *name;
|
|
|
|
const char *conf_name;
|
|
|
|
const char *parent_name;
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *);
|
output: introduce concept of sub-modules
To support the 'eve-log' idea, we need to be able to force all log
modules to be enabled by the master eve-log module, and need to be
able to make all logs go into a single file. This didn't fit the
API so far, so added the sub-module concept.
A sub-module is a regular module, that registers itself as a sub-
module of another module:
OutputRegisterTxSubModule("eve-log", "JsonHttpLog", "http",
OutputHttpLogInitSub, ALPROTO_HTTP, JsonHttpLogger);
The first argument is the name of the parent. The 4th argument is
the OutputCtx init function. It differs slightly from the non-sub
one. The different is that in addition to it's ConfNode, it gets
the OutputCtx from the parent. This way it can set the parents
LogFileCtx in it's own OutputCtx.
The runmode setup code will take care of all the extra setup. It's
possible to register a module both as a normal module and as a sub-
module, which can operate at the same time.
Only the TxLogger API is handled in this patch, the rest will be
updated later.
12 years ago
|
|
|
OutputCtx *(*InitSubFunc)(ConfNode *, OutputCtx *parent_ctx);
|
|
|
|
|
|
|
|
TmEcode (*ThreadInit)(ThreadVars *, void *, void **);
|
|
|
|
TmEcode (*ThreadDeinit)(ThreadVars *, void *);
|
|
|
|
void (*ThreadExitPrintStats)(ThreadVars *, void *);
|
|
|
|
|
|
|
|
PacketLogger PacketLogFunc;
|
|
|
|
PacketLogCondition PacketConditionFunc;
|
|
|
|
TxLogger TxLogFunc;
|
|
|
|
TxLoggerCondition TxLogCondition;
|
|
|
|
FileLogger FileLogFunc;
|
|
|
|
FiledataLogger FiledataLogFunc;
|
|
|
|
FlowLogger FlowLogFunc;
|
|
|
|
StreamingLogger StreamingLogFunc;
|
|
|
|
StatsLogger StatsLogFunc;
|
|
|
|
AppProto alproto;
|
|
|
|
enum OutputStreamingType stream_type;
|
|
|
|
int tc_log_progress;
|
|
|
|
int ts_log_progress;
|
|
|
|
|
|
|
|
TAILQ_ENTRY(OutputModule_) entries;
|
|
|
|
} OutputModule;
|
|
|
|
|
|
|
|
typedef TAILQ_HEAD(OutputModuleList_, OutputModule_) OutputModuleList;
|
|
|
|
extern OutputModuleList output_modules;
|
|
|
|
|
|
|
|
void OutputRegisterModule(const char *, const char *, OutputCtx *(*)(ConfNode *));
|
|
|
|
|
|
|
|
void OutputRegisterPacketModule(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *),
|
|
|
|
PacketLogger LogFunc, PacketLogCondition ConditionFunc,
|
|
|
|
ThreadInitFunc, ThreadDeinitFunc, ThreadExitPrintStatsFunc);
|
|
|
|
void OutputRegisterPacketSubModule(const char *parent_name, const char *name,
|
|
|
|
const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
|
output: sub-module support for other log api's
Packets:
void OutputRegisterPacketSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
PacketLogger LogFunc, PacketLogCondition ConditionFunc);
Files:
void OutputRegisterFileSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), FileLogger FileLogFunc);
Filedata:
void OutputRegisterFiledataSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), FiledataLogger FiledataLogFunc);
12 years ago
|
|
|
PacketLogger LogFunc, PacketLogCondition ConditionFunc);
|
|
|
|
|
|
|
|
void OutputRegisterTxModule(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto,
|
|
|
|
TxLogger TxLogFunc, TmEcode (*ThreadInit)(ThreadVars *t, void *, void **),
|
|
|
|
TmEcode (*ThreadDeinit)(ThreadVars *t, void *),
|
|
|
|
void (*ThreadExitPrintStats)(ThreadVars *, void *));
|
|
|
|
|
|
|
|
void OutputRegisterTxSubModule(const char *parent_name, const char *name,
|
|
|
|
const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx),
|
|
|
|
AppProto alproto, TxLogger TxLogFunc);
|
output: sub-module support for other log api's
Packets:
void OutputRegisterPacketSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
PacketLogger LogFunc, PacketLogCondition ConditionFunc);
Files:
void OutputRegisterFileSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), FileLogger FileLogFunc);
Filedata:
void OutputRegisterFiledataSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), FiledataLogger FiledataLogFunc);
12 years ago
|
|
|
|
|
|
|
void OutputRegisterTxModuleWithCondition(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto,
|
|
|
|
TxLogger TxLogFunc, TxLoggerCondition TxLogCondition);
|
|
|
|
void OutputRegisterTxSubModuleWithCondition(const char *parent_name,
|
|
|
|
const char *name, const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *,
|
|
|
|
OutputCtx *parent_ctx), AppProto alproto, TxLogger TxLogFunc,
|
|
|
|
TxLoggerCondition TxLogCondition);
|
|
|
|
|
|
|
|
void OutputRegisterTxModuleWithProgress(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *), AppProto alproto,
|
|
|
|
TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress);
|
|
|
|
void OutputRegisterTxSubModuleWithProgress(const char *parent_name, const char *name,
|
|
|
|
const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *parent_ctx),
|
|
|
|
AppProto alproto, TxLogger TxLogFunc, int tc_log_progress, int ts_log_progress);
|
|
|
|
|
|
|
|
void OutputRegisterFileModule(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *), FileLogger FileLogFunc);
|
|
|
|
void OutputRegisterFileSubModule(const char *parent_name, const char *name,
|
|
|
|
const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
|
|
|
|
FileLogger FileLogFunc);
|
output: sub-module support for other log api's
Packets:
void OutputRegisterPacketSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
PacketLogger LogFunc, PacketLogCondition ConditionFunc);
Files:
void OutputRegisterFileSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), FileLogger FileLogFunc);
Filedata:
void OutputRegisterFiledataSubModule(const char *parent_name, char *name, char *conf_name,
OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *), FiledataLogger FiledataLogFunc);
12 years ago
|
|
|
|
|
|
|
void OutputRegisterFiledataModule(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *), FiledataLogger FiledataLogFunc);
|
|
|
|
void OutputRegisterFiledataSubModule(const char *parent_name, const char *name,
|
|
|
|
const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
|
|
|
|
FiledataLogger FiledataLogFunc);
|
|
|
|
|
|
|
|
void OutputRegisterFlowModule(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *), FlowLogger FlowLogFunc);
|
|
|
|
void OutputRegisterFlowSubModule(const char *parent_name, const char *name,
|
|
|
|
const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
|
|
|
|
FlowLogger FlowLogFunc);
|
|
|
|
|
|
|
|
void OutputRegisterStreamingModule(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *), StreamingLogger StreamingLogFunc,
|
|
|
|
enum OutputStreamingType stream_type);
|
|
|
|
void OutputRegisterStreamingSubModule(const char *parent_name, const char *name,
|
|
|
|
const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
|
|
|
|
StreamingLogger StreamingLogFunc, enum OutputStreamingType stream_type);
|
|
|
|
|
|
|
|
void OutputRegisterStatsModule(const char *name, const char *conf_name,
|
|
|
|
OutputCtx *(*InitFunc)(ConfNode *), StatsLogger StatsLogFunc);
|
|
|
|
void OutputRegisterStatsSubModule(const char *parent_name, const char *name,
|
|
|
|
const char *conf_name, OutputCtx *(*InitFunc)(ConfNode *, OutputCtx *),
|
|
|
|
StatsLogger StatsLogFunc);
|
|
|
|
|
|
|
|
OutputModule *OutputGetModuleByConfName(const char *name);
|
|
|
|
void OutputDeregisterAll(void);
|
|
|
|
|
|
|
|
int OutputDropLoggerEnable(void);
|
|
|
|
void OutputDropLoggerDisable(void);
|
|
|
|
|
|
|
|
int OutputSshLoggerEnable(void);
|
|
|
|
void OutputSshLoggerDisable(void);
|
|
|
|
|
|
|
|
void OutputRegisterFileRotationFlag(int *flag);
|
|
|
|
void OutputUnregisterFileRotationFlag(int *flag);
|
|
|
|
void OutputNotifyFileRotation(void);
|
|
|
|
|
|
|
|
#endif /* ! __OUTPUT_H__ */
|