aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove dsize grouping from detection engine grouping reducing memory usage. Store sgh in flow to reduce lookups. Reduce locking in alert handling. Increase default grouping values as we use less memory.

Browse Source
remotes/origin/master-1.0.x
Victor Julien 17 years ago
parent b657705d8c
commit 2fd31a1a11
10 changed files with 482 additions and 718 deletions
Show Stats Download Patch File Download Diff File

8
src/detect-engine-alert.c
Unescape Escape View File

@ -163,15 +163,11 @@ void PacketAlertFinalize(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx
(p->flowflags & FLOW_PKT_TOCLIENT && !(p->flowflags & FLOW_PKT_TOCLIENT_IPONLY_SET))) {
SCLogDebug("testing against \"ip-only\" signatures");
/* save in the flow that we scanned this direction... locking is
* done in the FlowSetIPOnlyFlag function. */
/** \todo locking overhead: locked/unlocked twice */
if (p->flow != NULL) {
FlowSetIPOnlyFlag(p->flow, p->flowflags & FLOW_PKT_TOSERVER ? 1 : 0);
/* Update flow flags for iponly */
SCMutexLock(&p->flow->m);
FlowSetIPOnlyFlagNoLock(p->flow, p->flowflags & FLOW_PKT_TOSERVER ? 1 : 0);
if (s->action & ACTION_DROP)
p->flow->flags |= FLOW_ACTION_DROP;
if (s->action & ACTION_REJECT)

2
src/detect-engine-siggroup.c
Unescape Escape View File

@ -1892,7 +1892,7 @@ static int SigGroupHeadTest10(void)
AddressDebugPrint(&p.dst);
SigGroupHead *sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
SigGroupHead *sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
if (sgh == NULL) {
goto end;
}

161
src/detect-engine.c
Unescape Escape View File

@ -141,17 +141,6 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) {
const char *max_uniq_toserver_sp_groups_str = NULL;
const char *max_uniq_toserver_dp_groups_str = NULL;
const char *max_uniq_small_toclient_src_groups_str = NULL;
const char *max_uniq_small_toclient_dst_groups_str = NULL;
const char *max_uniq_small_toclient_sp_groups_str = NULL;
const char *max_uniq_small_toclient_dp_groups_str = NULL;
const char *max_uniq_small_toserver_src_groups_str = NULL;
const char *max_uniq_small_toserver_dst_groups_str = NULL;
const char *max_uniq_small_toserver_sp_groups_str = NULL;
const char *max_uniq_small_toserver_dp_groups_str = NULL;
ConfNode *de_ctx_custom = ConfGetNode("detect-engine");
ConfNode *opt = NULL;
@ -191,33 +180,19 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) {
de_ctx->max_uniq_toserver_dst_groups = 2;
de_ctx->max_uniq_toserver_sp_groups = 2;
de_ctx->max_uniq_toserver_dp_groups = 3;
de_ctx->max_uniq_small_toclient_src_groups = 2;
de_ctx->max_uniq_small_toclient_dst_groups = 2;
de_ctx->max_uniq_small_toclient_sp_groups = 2;
de_ctx->max_uniq_small_toclient_dp_groups = 3;
de_ctx->max_uniq_small_toserver_src_groups = 2;
de_ctx->max_uniq_small_toserver_dst_groups = 2;
de_ctx->max_uniq_small_toserver_sp_groups = 2;
de_ctx->max_uniq_small_toserver_dp_groups = 3;
break;
break;
case ENGINE_PROFILE_HIGH:
de_ctx->max_uniq_toclient_src_groups = 5;
de_ctx->max_uniq_toclient_dst_groups = 5;
de_ctx->max_uniq_toclient_sp_groups = 5;
de_ctx->max_uniq_toclient_dp_groups = 10;
de_ctx->max_uniq_toserver_src_groups = 5;
de_ctx->max_uniq_toserver_dst_groups = 5;
de_ctx->max_uniq_toserver_sp_groups = 5;
de_ctx->max_uniq_toserver_dp_groups = 30;
de_ctx->max_uniq_small_toclient_src_groups = 5;
de_ctx->max_uniq_small_toclient_dst_groups = 5;
de_ctx->max_uniq_small_toclient_sp_groups = 5;
de_ctx->max_uniq_small_toclient_dp_groups = 10;
de_ctx->max_uniq_small_toserver_src_groups = 5;
de_ctx->max_uniq_small_toserver_dst_groups = 5;
de_ctx->max_uniq_small_toserver_sp_groups = 5;
de_ctx->max_uniq_small_toserver_dp_groups = 10;
break;
de_ctx->max_uniq_toclient_src_groups = 15;
de_ctx->max_uniq_toclient_dst_groups = 15;
de_ctx->max_uniq_toclient_sp_groups = 15;
de_ctx->max_uniq_toclient_dp_groups = 20;
de_ctx->max_uniq_toserver_src_groups = 15;
de_ctx->max_uniq_toserver_dst_groups = 15;
de_ctx->max_uniq_toserver_sp_groups = 15;
de_ctx->max_uniq_toserver_dp_groups = 40;
break;
case ENGINE_PROFILE_CUSTOM:
TAILQ_FOREACH(opt, &de_ctx_custom->head, next) {
if (strncmp(opt->val, "custom-values", 3) == 0) {
@ -237,22 +212,6 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) {
(opt->head.tqh_first, "toserver_sp_groups");
max_uniq_toserver_dp_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "toserver_dp_groups");
max_uniq_small_toclient_src_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "small_toclient_src_groups");
max_uniq_small_toclient_dst_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "small_toclient_dst_groups");
max_uniq_small_toclient_sp_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "small_toclient_sp_groups");
max_uniq_small_toclient_dp_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "small_toclient_dp_groups");
max_uniq_small_toserver_src_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "small_toserver_src_groups");
max_uniq_small_toserver_dst_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "small_toserver_dst_groups");
max_uniq_small_toserver_sp_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "small_toserver_sp_groups");
max_uniq_small_toserver_dp_groups_str = ConfNodeLookupChildValue
(opt->head.tqh_first, "small_toserver_dp_groups");
}
}
if (max_uniq_toclient_src_groups_str != NULL) {
@ -319,96 +278,22 @@ static uint8_t DetectEngineCtxLoadConf(DetectEngineCtx *de_ctx) {
} else {
de_ctx->max_uniq_toserver_dp_groups = 2;
}
if (max_uniq_small_toclient_src_groups_str != NULL) {
if (ByteExtractStringUint16(&de_ctx->max_uniq_small_toclient_src_groups, 10,
strlen(max_uniq_small_toclient_src_groups_str),
(const char *)max_uniq_small_toclient_src_groups_str) <= 0)
de_ctx->max_uniq_small_toclient_src_groups = 2;
} else {
de_ctx->max_uniq_small_toclient_src_groups = 2;
}
if (max_uniq_small_toclient_dst_groups_str != NULL) {
if (ByteExtractStringUint16(&de_ctx->max_uniq_small_toclient_dst_groups, 10,
strlen(max_uniq_small_toclient_dst_groups_str),
(const char *)max_uniq_small_toclient_dst_groups_str) <= 0)
de_ctx->max_uniq_small_toclient_dst_groups = 2;
} else {
de_ctx->max_uniq_small_toclient_dst_groups = 2;
}
if (max_uniq_small_toclient_sp_groups_str != NULL) {
if (ByteExtractStringUint16(&de_ctx->max_uniq_small_toclient_sp_groups, 10,
strlen(max_uniq_small_toclient_sp_groups_str),
(const char *)max_uniq_small_toclient_sp_groups_str) <= 0)
de_ctx->max_uniq_small_toclient_sp_groups = 2;
} else {
de_ctx->max_uniq_small_toclient_sp_groups = 2;
}
if (max_uniq_small_toclient_dp_groups_str != NULL) {
if (ByteExtractStringUint16(&de_ctx->max_uniq_small_toclient_dp_groups, 10,
strlen(max_uniq_small_toclient_dp_groups_str),
(const char *)max_uniq_small_toclient_dp_groups_str) <= 0)
de_ctx->max_uniq_small_toclient_dp_groups = 2;
} else {
de_ctx->max_uniq_small_toclient_dp_groups = 2;
}
if (max_uniq_small_toserver_src_groups_str != NULL) {
if (ByteExtractStringUint16(&de_ctx->max_uniq_small_toserver_src_groups, 10,
strlen(max_uniq_small_toserver_src_groups_str),
(const char *)max_uniq_small_toserver_src_groups_str) <= 0)
de_ctx->max_uniq_small_toserver_src_groups = 2;
} else {
de_ctx->max_uniq_small_toserver_src_groups = 2;
}
if (max_uniq_small_toserver_dst_groups_str != NULL) {
if (ByteExtractStringUint16(&de_ctx->max_uniq_small_toserver_dst_groups, 10,
strlen(max_uniq_small_toserver_dst_groups_str),
(const char *)max_uniq_small_toserver_dst_groups_str) <= 0)
de_ctx->max_uniq_small_toserver_dst_groups = 2;
} else {
de_ctx->max_uniq_small_toserver_dst_groups = 2;
}
if (max_uniq_small_toserver_sp_groups_str != NULL) {
if (ByteExtractStringUint16(&de_ctx->max_uniq_small_toserver_sp_groups, 10,
strlen(max_uniq_small_toserver_sp_groups_str),
(const char *)max_uniq_small_toserver_sp_groups_str) <= 0)
de_ctx->max_uniq_small_toserver_sp_groups = 2;
} else {
de_ctx->max_uniq_small_toserver_sp_groups = 2;
}
if (max_uniq_small_toserver_dp_groups_str != NULL) {
if (ByteExtractStringUint16(&de_ctx->max_uniq_small_toserver_dp_groups, 10,
strlen(max_uniq_small_toserver_dp_groups_str),
(const char *)max_uniq_small_toserver_dp_groups_str) <= 0)
de_ctx->max_uniq_small_toserver_dp_groups = 2;
} else {
de_ctx->max_uniq_small_toserver_dp_groups = 2;
}
break;
break;
/* Default (or no config provided) is profile medium */
case ENGINE_PROFILE_MEDIUM:
case ENGINE_PROFILE_UNKNOWN:
default:
de_ctx->max_uniq_toclient_src_groups = 2;
de_ctx->max_uniq_toclient_dst_groups = 2;
de_ctx->max_uniq_toclient_sp_groups = 2;
de_ctx->max_uniq_toclient_dp_groups = 3;
de_ctx->max_uniq_toserver_src_groups = 2;
de_ctx->max_uniq_toserver_dst_groups = 4;
de_ctx->max_uniq_toserver_sp_groups = 2;
de_ctx->max_uniq_toserver_dp_groups = 25;
de_ctx->max_uniq_small_toclient_src_groups = 2;
de_ctx->max_uniq_small_toclient_dst_groups = 2;
de_ctx->max_uniq_small_toclient_sp_groups = 2;
de_ctx->max_uniq_small_toclient_dp_groups = 2;
de_ctx->max_uniq_small_toserver_src_groups = 2;
de_ctx->max_uniq_small_toserver_dst_groups = 2;
de_ctx->max_uniq_small_toserver_sp_groups = 2;
de_ctx->max_uniq_small_toserver_dp_groups = 8;
break;
de_ctx->max_uniq_toclient_src_groups = 4;
de_ctx->max_uniq_toclient_dst_groups = 4;
de_ctx->max_uniq_toclient_sp_groups = 4;
de_ctx->max_uniq_toclient_dp_groups = 6;
de_ctx->max_uniq_toserver_src_groups = 4;
de_ctx->max_uniq_toserver_dst_groups = 8;
de_ctx->max_uniq_toserver_sp_groups = 4;
de_ctx->max_uniq_toserver_dp_groups = 30;
break;
}
if (profile == ENGINE_PROFILE_UNKNOWN)

23
src/detect-fast-pattern.c
Unescape Escape View File

@ -96,11 +96,6 @@ static int DetectFastPatternSetup(DetectEngineCtx *de_ctx, Signature *s, char *n
#ifdef UNITTESTS
SigGroupHead *SigMatchSignaturesGetSgh(ThreadVars *,
DetectEngineCtx *,
DetectEngineThreadCtx *,
Packet *);
/**
* \test Checks if a fast_pattern is registered in a Signature
*/
@ -291,7 +286,7 @@ int DetectFastPatternTest05(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) != 0)
result = 1;
@ -345,7 +340,7 @@ int DetectFastPatternTest06(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) != 0)
result = 1;
@ -400,7 +395,7 @@ int DetectFastPatternTest07(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) == 0)
result = 1;
@ -459,7 +454,7 @@ int DetectFastPatternTest08(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
uint32_t r = PacketPatternSearch(&th_v, det_ctx, &p);
if (r != 1) {
printf("expected 1, got %"PRIu32": ", r);
@ -515,7 +510,7 @@ int DetectFastPatternTest09(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) == 0)
result = 1;
@ -575,7 +570,7 @@ int DetectFastPatternTest10(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
uint32_t r = PacketPatternSearch(&th_v, det_ctx, &p);
if (r != 1) {
printf("expected 1, got %"PRIu32": ", r);
@ -633,7 +628,7 @@ int DetectFastPatternTest11(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) == 0)
result = 1;
@ -689,7 +684,7 @@ int DetectFastPatternTest12(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
if (PacketPatternSearch(&th_v, det_ctx, &p) == 0)
result = 1;
@ -750,7 +745,7 @@ int DetectFastPatternTest13(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
/* start the search phase */
det_ctx->sgh = SigMatchSignaturesGetSgh(&th_v, de_ctx, det_ctx, &p);
det_ctx->sgh = SigMatchSignaturesGetSgh(de_ctx, det_ctx, &p);
uint32_t r = PacketPatternSearch(&th_v, det_ctx, &p);
if (r != 1) {
printf("expected 1 result, got %"PRIu32": ", r);

951
src/detect.c
View File

File diff suppressed because it is too large Load Diff

17
src/detect.h
Unescape Escape View File

@ -331,15 +331,6 @@ typedef struct DetectEngineLookupFlow_ {
* to client
*/
#define FLOW_STATES 2
typedef struct DetectEngineLookupDsize_ {
DetectEngineLookupFlow flow_gh[FLOW_STATES];
} DetectEngineLookupDsize;
/* Dsize states
* <= 100
* >100
*/
#define DSIZE_STATES 2
/* mpm pattern id api */
typedef struct MpmPatternIdStore_ {
@ -381,7 +372,7 @@ typedef struct DetectEngineCtx_ {
HashTable *class_conf_ht;
/* main sigs */
DetectEngineLookupDsize dsize_gh[DSIZE_STATES];
DetectEngineLookupFlow flow_gh[FLOW_STATES];
uint32_t mpm_unique, mpm_reuse, mpm_none,
mpm_uri_unique, mpm_uri_reuse, mpm_uri_none;
@ -431,7 +422,7 @@ typedef struct DetectEngineCtx_ {
uint16_t max_uniq_toserver_dst_groups;
uint16_t max_uniq_toserver_sp_groups;
uint16_t max_uniq_toserver_dp_groups;
/*
uint16_t max_uniq_small_toclient_src_groups;
uint16_t max_uniq_small_toclient_dst_groups;
uint16_t max_uniq_small_toclient_sp_groups;
@ -441,7 +432,7 @@ typedef struct DetectEngineCtx_ {
uint16_t max_uniq_small_toserver_dst_groups;
uint16_t max_uniq_small_toserver_sp_groups;
uint16_t max_uniq_small_toserver_dp_groups;
*/
/** hash table for looking up patterns for
* id sharing and id tracking. */
MpmPatternIdStore *mpm_pattern_id_store;
@ -719,6 +710,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx,
DetectEngineThreadCtx *det_ctx, Packet *p);
int SignatureIsIPOnly(DetectEngineCtx *de_ctx, Signature *s);
SigGroupHead *SigMatchSignaturesGetSgh(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p);
SigGroupHead *SigMatchSignaturesGetSgh(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p);
#endif /* __DETECT_H__ */

2
src/flow-util.h
Unescape Escape View File

@ -43,6 +43,8 @@
(f)->use_cnt = 0; \
DetectEngineStateFree((f)->de_state); \
(f)->de_state = NULL; \
(f)->sgh_toserver = NULL; \
(f)->sgh_toclient = NULL; \
}
Flow *FlowAlloc(void);

9
src/flow.c
Unescape Escape View File

@ -540,6 +540,15 @@ void FlowSetIPOnlyFlag(Flow *f, char direction) {
SCMutexUnlock(&f->m);
}
/** \brief Set the IPOnly scanned flag for 'direction'.
*
* \param f Flow to set the flag in
* \param direction direction to set the flag in
*/
void FlowSetIPOnlyFlagNoLock(Flow *f, char direction) {
direction ? (f->flags |= FLOW_TOSERVER_IPONLY_SET) : (f->flags |= FLOW_TOCLIENT_IPONLY_SET);
}
/** \brief increase the use cnt of a flow
* \param tv thread vars (\todo unused?)
* \param p packet with flow to decrease use cnt for

13
src/flow.h
Unescape Escape View File

@ -59,6 +59,11 @@
/** All packets in this flow should be accepted */
#define FLOW_ACTION_PASS 0x0400
/** Sgh for toserver direction set (even if it's NULL) */
#define FLOW_SGH_TOSERVER 0x0800
/** Sgh for toclient direction set (even if it's NULL) */
#define FLOW_SGH_TOCLIENT 0x1000
/* pkt flow flags */
#define FLOW_PKT_TOSERVER 0x01
#define FLOW_PKT_TOCLIENT 0x02
@ -160,6 +165,13 @@ typedef struct Flow_
/** detection engine state */
struct DetectEngineState_ *de_state;
/** toclient sgh for this flow. Only use when FLOW_SGH_TOCLIENT flow flag
* has been set. */
struct SigGroupHead_ *sgh_toclient;
/** toserver sgh for this flow. Only use when FLOW_SGH_TOSERVER flow flag
* has been set. */
struct SigGroupHead_ *sgh_toserver;
SCMutex m;
/* list flow ptrs
@ -195,6 +207,7 @@ void FlowInitConfig (char);
void FlowPrintQueueInfo (void);
void FlowShutdown(void);
void FlowSetIPOnlyFlag(Flow *, char);
void FlowSetIPOnlyFlagNoLock(Flow *, char);
void FlowDecrUsecnt(ThreadVars *, Packet *);
uint32_t FlowPruneFlowsCnt(struct timeval *, int);
uint32_t FlowKillFlowsCnt(int);

14
suricata.yaml
Unescape Escape View File

@ -64,7 +64,7 @@ defrag:
prealloc: yes
timeout: 60
# The detection engine build internal groups of signatures. The engine
# The detection engine builds internal groups of signatures. The engine
# allow us to specify the profile to use for them, to manage memory on an
# efficient way keeping a good performance. For the profile keyword you
# can use the words "low", "medium", "high" or "custom". If you use custom
@ -81,14 +81,6 @@ detect-engine:
toserver_dst_groups: 4
toserver_sp_groups: 2
toserver_dp_groups: 25
small_toclient_src_groups: 2
small_toclient_dst_groups: 2
small_toclient_sp_groups: 2
small_toclient_dp_groups: 2
small_toserver_src_groups: 2
small_toserver_dst_groups: 2
small_toserver_sp_groups: 2
small_toserver_dp_groups: 8
# Select the multi pattern algorithm you want to run for scan/search the
@ -455,6 +447,9 @@ libhtp:
- 192.168.10.0/24
personality: IIS_7_0
# rule profiling settings. Only effective if Suricata has been built with the
# the --enable-profiling configure flag.
#
profiling:
rules:
@ -468,3 +463,4 @@ profiling:
# Limit the number of items printed at exit.
limit: 100

Write Preview
Loading…
Cancel
Save