From 3f7195454b05e5fb07cbb13dea115b3d0e5e63cb Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Sun, 19 Oct 2008 11:26:51 +0200 Subject: [PATCH] Big detection engine update. --- src/detect-engine-address-ipv4.c | 46 +- src/detect-engine-address-ipv4.h | 2 + src/detect-engine-address-ipv6.c | 110 +++ src/detect-engine-address-ipv6.h | 2 + src/detect-engine-address.c | 37 +- src/detect-engine-address.h | 4 + src/detect-engine-port.c | 203 ++++- src/detect-engine-port.h | 17 +- src/detect-engine-proto.c | 28 +- src/detect-engine-siggroup.c | 134 +++- src/detect-engine-siggroup.h | 4 + src/detect-parse.c | 37 +- src/detect.c | 1254 +++++++++++++++++++++--------- src/detect.h | 35 +- src/util-mpm-wumanber.c | 127 ++- src/vips.c | 2 +- 16 files changed, 1599 insertions(+), 443 deletions(-) diff --git a/src/detect-engine-address-ipv4.c b/src/detect-engine-address-ipv4.c index 931a890d0b..1f0b4831da 100644 --- a/src/detect-engine-address-ipv4.c +++ b/src/detect-engine-address-ipv4.c @@ -129,6 +129,9 @@ int DetectAddressGroupCutIPv4(DetectAddressGroup *a, DetectAddressGroup *b, Dete DetectPortInsertCopy(&b->port, port); } + tmp_c->cnt += b->cnt; + b->cnt += a->cnt; + /* we have 3 parts: [bbb[baba]aaa] * part a: b_ip1 <-> a_ip1 - 1 * part b: a_ip1 <-> b_ip2 @@ -182,6 +185,13 @@ int DetectAddressGroupCutIPv4(DetectAddressGroup *a, DetectAddressGroup *b, Dete DetectPortInsertCopy(&tmp_c->port, port); } + tmp->cnt += a->cnt; + a->cnt = 0; + tmp_c->cnt += tmp->cnt; + a->cnt += b->cnt; + b->cnt += tmp->cnt; + tmp->cnt = 0; + /* we have 2 or three parts: * * 2 part: [[abab]bbb] or [bbb[baba]] @@ -218,6 +228,7 @@ int DetectAddressGroupCutIPv4(DetectAddressGroup *a, DetectAddressGroup *b, Dete for (port = b->port; port != NULL; port = port->next) { DetectPortInsertCopy(&a->port, port); } + a->cnt += b->cnt; } else if (a_ip2 == b_ip2) { #ifdef DBG @@ -235,6 +246,7 @@ int DetectAddressGroupCutIPv4(DetectAddressGroup *a, DetectAddressGroup *b, Dete for (port = a->port; port != NULL; port = port->next) { DetectPortInsertCopy(&b->port, port); } + b->cnt += a->cnt; } else { #ifdef DBG printf("3\n"); @@ -282,15 +294,12 @@ int DetectAddressGroupCutIPv4(DetectAddressGroup *a, DetectAddressGroup *b, Dete for (port = tmp->port; port != NULL; port = port->next) { DetectPortInsertCopy(&b->port, port); } -#ifdef DBG -SigGroupContainer *sg; -printf("DetectAddressGroupCutIPv4: A "); DetectAddressDataPrint(a->ad); printf(" "); -for(sg = a->sh ? a->sh->head : NULL; sg != NULL; sg = sg->next) printf("%u ", sg->s->id); printf("\n"); -printf("DetectAddressGroupCutIPv4: B "); DetectAddressDataPrint(b->ad); printf(" "); -for(sg = b->sh ? b->sh->head : NULL; sg != NULL; sg = sg->next) printf("%u ", sg->s->id); printf("\n"); -printf("DetectAddressGroupCutIPv4: C "); DetectAddressDataPrint(tmp_c->ad); printf(" "); -for(sg = tmp_c->sh ? b->sh->head : NULL; sg != NULL; sg = sg->next) printf("%u ", sg->s->id); printf("\n\n"); -#endif + tmp->cnt += a->cnt; + a->cnt = 0; + tmp_c->cnt += b->cnt; + a->cnt += b->cnt; + b->cnt += tmp->cnt; + tmp->cnt = 0; } /* we have 2 or three parts: * @@ -338,6 +347,11 @@ for(sg = tmp_c->sh ? b->sh->head : NULL; sg != NULL; sg = sg->next) printf("%u " for (port = tmp->port; port != NULL; port = port->next) { DetectPortInsertCopy(&a->port, port); } + tmp->cnt += b->cnt; + b->cnt = 0; + b->cnt += a->cnt; + a->cnt += tmp->cnt; + tmp->cnt = 0; } else if (a_ip2 == b_ip2) { #ifdef DBG printf("DetectAddressGroupCutIPv4: 2\n"); @@ -354,6 +368,8 @@ for(sg = tmp_c->sh ? b->sh->head : NULL; sg != NULL; sg = sg->next) printf("%u " for (port = a->port; port != NULL; port = port->next) { DetectPortInsertCopy(&b->port, port); } + + b->cnt += a->cnt; } else { #ifdef DBG printf("DetectAddressGroupCutIPv4: 3\n"); @@ -391,6 +407,9 @@ for(sg = tmp_c->sh ? b->sh->head : NULL; sg != NULL; sg = sg->next) printf("%u " for (port = a->port; port != NULL; port = port->next) { DetectPortInsertCopy(&tmp_c->port, port); } + + b->cnt += a->cnt; + tmp_c->cnt += a->cnt; } } @@ -617,4 +636,13 @@ error: return -1; } +int DetectAddressGroupJoinIPv4(DetectAddressGroup *target, DetectAddressGroup *source) { + if (ntohl(source->ad->ip[0]) < ntohl(target->ad->ip[0])) + target->ad->ip[0] = source->ad->ip[0]; + + if (ntohl(source->ad->ip2[0]) > ntohl(target->ad->ip2[0])) + target->ad->ip2[0] = source->ad->ip2[0]; + + return 0; +} diff --git a/src/detect-engine-address-ipv4.h b/src/detect-engine-address-ipv4.h index 6564cefba8..88d8eab568 100644 --- a/src/detect-engine-address-ipv4.h +++ b/src/detect-engine-address-ipv4.h @@ -12,5 +12,7 @@ int DetectAddressCutNotIPv4(DetectAddressData *, DetectAddressData **); int DetectAddressGroupCutIPv4(DetectAddressGroup *, DetectAddressGroup *, DetectAddressGroup **); +int DetectAddressGroupJoinIPv4(DetectAddressGroup *target, DetectAddressGroup *source); + #endif /* __DETECT_ADDRESS_IPV4_H__ */ diff --git a/src/detect-engine-address-ipv6.c b/src/detect-engine-address-ipv6.c index fa02584393..66fa115eec 100644 --- a/src/detect-engine-address-ipv6.c +++ b/src/detect-engine-address-ipv6.c @@ -196,6 +196,7 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete ntohl(b->ad->ip[2]), ntohl(b->ad->ip[3]) }; u_int32_t b_ip2[4] = { ntohl(b->ad->ip2[0]), ntohl(b->ad->ip2[1]), ntohl(b->ad->ip2[2]), ntohl(b->ad->ip2[3]) }; + DetectPort *port = NULL; /* default to NULL */ *c = NULL; @@ -241,6 +242,16 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete SigGroupHeadCopySigs(b->sh,&tmp_c->sh); /* copy old b to c */ SigGroupHeadCopySigs(a->sh,&b->sh); /* copy old b to a */ + for (port = b->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&tmp_c->port, port); + } + for (port = a->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&b->port, port); + } + + tmp_c->cnt += b->cnt; + b->cnt += a->cnt; + /* we have 3 parts: [bbb[baba]aaa] * part a: b_ip1 <-> a_ip1 - 1 * part b: a_ip1 <-> b_ip2 @@ -278,6 +289,26 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete SigGroupHeadClearSigs(tmp->sh); /* clean tmp list */ + for (port = a->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&tmp->port, port); + } + for (port = b->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&a->port, port); + } + for (port = tmp->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&b->port, port); + } + for (port = tmp->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&tmp_c->port, port); + } + + tmp->cnt += a->cnt; + a->cnt = 0; + tmp_c->cnt += tmp->cnt; + a->cnt += b->cnt; + b->cnt += tmp->cnt; + tmp->cnt = 0; + /* we have 2 or three parts: * * 2 part: [[abab]bbb] or [bbb[baba]] @@ -302,6 +333,12 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete /* 'b' overlaps 'a' so 'a' needs the 'b' sigs */ SigGroupHeadCopySigs(b->sh,&a->sh); + + for (port = b->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&a->port, port); + } + a->cnt += b->cnt; + } else if (AddressIPv6Eq(a_ip2, b_ip2) == 1) { AddressCutIPv6Copy(b_ip1, a->ad->ip); AddressCutIPv6CopySubOne(a_ip1, a->ad->ip2); @@ -311,6 +348,11 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete /* 'a' overlaps 'b' so a needs the 'a' sigs */ SigGroupHeadCopySigs(a->sh,&b->sh); + + for (port = a->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&b->port, port); + } + b->cnt += a->cnt; } else { AddressCutIPv6Copy(b_ip1, a->ad->ip); AddressCutIPv6CopySubOne(a_ip1, a->ad->ip2); @@ -342,6 +384,25 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete SigGroupHeadCopySigs(tmp->sh,&b->sh); /* prepend old a before b */ SigGroupHeadClearSigs(tmp->sh); /* clean tmp list */ + + for (port = a->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&tmp->port, port); + } + for (port = b->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&tmp_c->port, port); + } + for (port = b->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&a->port, port); + } + for (port = tmp->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&b->port, port); + } + tmp->cnt += a->cnt; + a->cnt = 0; + tmp_c->cnt += b->cnt; + a->cnt += b->cnt; + b->cnt += tmp->cnt; + tmp->cnt = 0; } /* we have 2 or three parts: * @@ -371,6 +432,21 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete SigGroupHeadCopySigs(a->sh,&b->sh); SigGroupHeadCopySigs(tmp->sh,&a->sh); SigGroupHeadClearSigs(tmp->sh); + + for (port = b->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&tmp->port, b->port); + } + for (port = a->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&b->port, port); + } + for (port = tmp->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&a->port, port); + } + tmp->cnt += b->cnt; + b->cnt = 0; + b->cnt += a->cnt; + a->cnt += tmp->cnt; + tmp->cnt = 0; } else if (AddressIPv6Eq(a_ip2, b_ip2) == 1) { AddressCutIPv6Copy(a_ip1, a->ad->ip); AddressCutIPv6CopySubOne(b_ip1, a->ad->ip2); @@ -380,6 +456,12 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete /* 'a' overlaps 'b' so a needs the 'a' sigs */ SigGroupHeadCopySigs(a->sh,&b->sh); + + for (port = a->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&b->port, port); + } + + b->cnt += a->cnt; } else { AddressCutIPv6Copy(a_ip1, a->ad->ip); AddressCutIPv6CopySubOne(b_ip1, a->ad->ip2); @@ -406,6 +488,16 @@ int DetectAddressGroupCutIPv6(DetectAddressGroup *a, DetectAddressGroup *b, Dete * 'c' gets 'a' sigs */ SigGroupHeadCopySigs(a->sh,&b->sh); SigGroupHeadCopySigs(a->sh,&tmp_c->sh); + + for (port = a->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&b->port, port); + } + for (port = a->port; port != NULL; port = port->next) { + DetectPortInsertCopy(&tmp_c->port, port); + } + + b->cnt += a->cnt; + tmp_c->cnt += a->cnt; } } @@ -634,6 +726,24 @@ error: return -1; } +int DetectAddressGroupJoinIPv6(DetectAddressGroup *target, DetectAddressGroup *source) { + if (AddressIPv6Lt(source->ad->ip,target->ad->ip)) { + target->ad->ip[0] = source->ad->ip[0]; + target->ad->ip[1] = source->ad->ip[1]; + target->ad->ip[2] = source->ad->ip[2]; + target->ad->ip[3] = source->ad->ip[3]; + } + + if (AddressIPv6Gt(source->ad->ip,target->ad->ip)) { + target->ad->ip2[0] = source->ad->ip2[0]; + target->ad->ip2[1] = source->ad->ip2[1]; + target->ad->ip2[2] = source->ad->ip2[2]; + target->ad->ip2[3] = source->ad->ip2[3]; + } + + return 0; +} + /* TESTS */ diff --git a/src/detect-engine-address-ipv6.h b/src/detect-engine-address-ipv6.h index 5a536c2eeb..bc1e6cf26f 100644 --- a/src/detect-engine-address-ipv6.h +++ b/src/detect-engine-address-ipv6.h @@ -18,6 +18,8 @@ int DetectAddressCmpIPv6(DetectAddressData *, DetectAddressData *); int DetectAddressGroupCutIPv6(DetectAddressGroup *, DetectAddressGroup *, DetectAddressGroup **); +int DetectAddressGroupJoinIPv6(DetectAddressGroup *, DetectAddressGroup *); + void DetectAddressIPv6Tests(void); #endif /* __DETECT_ADDRESS_IPV6_H__ */ diff --git a/src/detect-engine-address.c b/src/detect-engine-address.c index a6a02126b8..b2e840caf7 100644 --- a/src/detect-engine-address.c +++ b/src/detect-engine-address.c @@ -81,11 +81,12 @@ void DetectAddressGroupFree(DetectAddressGroup *ag) { if (ag->dst_gh != NULL) { DetectAddressGroupsHeadFree(ag->dst_gh); } + ag->dst_gh = NULL; - if (ag->port != NULL && !(ag->flags & ADDRESS_GROUP_PORTS_COPY)) { + if (ag->port != NULL && !(ag->flags & PORT_GROUP_PORTS_COPY)) { DetectPortCleanupList(ag->port); - ag->port = NULL; } + ag->port = NULL; detect_address_group_memory -= sizeof(DetectAddressGroup); detect_address_group_free_cnt++; @@ -154,12 +155,9 @@ void DetectAddressGroupCleanupList (DetectAddressGroup *head) { for (cur = head; cur != NULL; ) { next = cur->next; - DetectAddressGroupFree(cur); cur = next; } - - head = NULL; } /* do a sorted insert, where the top of the list should be the biggest @@ -278,6 +276,7 @@ int DetectAddressGroupInsert(DetectAddressGroupsHead *gh, DetectAddressGroup *ne DetectPortInsertCopy(&cur->port,port); } SigGroupHeadCopySigs(new->sh,&cur->sh); + cur->cnt += new->cnt; DetectAddressGroupFree(new); return 0; } @@ -397,6 +396,30 @@ error: return -1; } +int DetectAddressGroupJoin(DetectAddressGroup *target, DetectAddressGroup *source) { + if (target == NULL || source == NULL) + return -1; + + if (target->ad->family != source->ad->family) + return -1; + + target->cnt += source->cnt; + SigGroupHeadCopySigs(source->sh,&target->sh); + + DetectPort *port = source->port; + for ( ; port != NULL; port = port->next) { + DetectPortInsertCopy(&target->port, port); + } + + if (target->ad->family == AF_INET) { + return DetectAddressGroupJoinIPv4(target,source); + } else if (target->ad->family == AF_INET6) { + return DetectAddressGroupJoinIPv6(target,source); + } + + return -1; +} + /* * return codes: * 1: inserted @@ -781,9 +804,11 @@ DetectAddressGroupsHead *DetectAddressGroupsHeadInit(void) { void DetectAddressGroupsHeadCleanup(DetectAddressGroupsHead *gh) { if (gh != NULL) { DetectAddressGroupCleanupList(gh->any_head); + gh->any_head = NULL; DetectAddressGroupCleanupList(gh->ipv4_head); + gh->ipv4_head = NULL; DetectAddressGroupCleanupList(gh->ipv6_head); - gh->any_head = gh->ipv4_head = gh->ipv6_head = NULL; + gh->ipv6_head = NULL; } } diff --git a/src/detect-engine-address.h b/src/detect-engine-address.h index 61a8620df3..0d156fe51b 100644 --- a/src/detect-engine-address.h +++ b/src/detect-engine-address.h @@ -18,6 +18,7 @@ enum { #define ADDRESS_GROUP_SIGGROUPHEAD_COPY 0x01 #define ADDRESS_GROUP_PORTS_COPY 0x02 +#define ADDRESS_GROUP_PORTS_NOTUNIQ 0x04 typedef struct DetectAddressData_ { /* XXX convert to use a Address datatype to replace family, ip,ip2*/ @@ -34,6 +35,7 @@ typedef struct DetectAddressGroup_ { /* XXX ptr to rules, or PortGroup or whatever */ struct DetectAddressGroupsHead_ *dst_gh; struct DetectPort_ *port; + /* signatures that belong in this group */ struct _SigGroupHead *sh; u_int8_t flags; @@ -42,6 +44,7 @@ typedef struct DetectAddressGroup_ { struct DetectAddressGroup_ *prev; struct DetectAddressGroup_ *next; + u_int32_t cnt; } DetectAddressGroup; typedef struct DetectAddressGroupsHead_ { @@ -72,6 +75,7 @@ void DetectAddressGroupFree(DetectAddressGroup *); int DetectAddressGroupInsert(DetectAddressGroupsHead *, DetectAddressGroup *); void DetectAddressGroupPrintMemory(void); void DetectAddressGroupCleanupList (DetectAddressGroup *); +int DetectAddressGroupJoin(DetectAddressGroup *target, DetectAddressGroup *source); #endif /* __DETECT_ADDRESS_H__ */ diff --git a/src/detect-engine-port.c b/src/detect-engine-port.c index ec913474b5..c5dab93d48 100644 --- a/src/detect-engine-port.c +++ b/src/detect-engine-port.c @@ -75,6 +75,11 @@ void DetectPortFree(DetectPort *dp) { } dp->sh = NULL; + if (dp->dst_ph != NULL && !(dp->flags & PORT_GROUP_PORTS_COPY)) { + DetectPortCleanupList(dp->dst_ph); + } + dp->dst_ph = NULL; + detect_port_memory -= sizeof(DetectPort); detect_port_free_cnt++; free(dp); @@ -186,6 +191,9 @@ int DetectPortAdd(DetectPort **head, DetectPort *dp) { int DetectPortInsertCopy(DetectPort **head, DetectPort *new) { DetectPort *copy = DetectPortCopySingle(new); + //printf("new (%p): ", new); DetectPortPrint(new); printf(" "); DbgPrintSigs2(new->sh); + //printf("copy (%p): ",copy); DetectPortPrint(copy); printf(" "); DbgPrintSigs2(copy->sh); + if (copy != NULL) { //printf("DetectPortInsertCopy: "); DetectPortPrint(copy); printf("\n"); } @@ -206,7 +214,9 @@ int DetectPortInsert(DetectPort **head, DetectPort *new) { if (new == NULL) return 0; + #ifdef DBG + printf("DetectPortInsert: head %p, new %p, new->dp %p\n", head, new, new->dp); printf("DetectPortInsert: inserting (sig %u) ", new->sh ? new->sh->sig_cnt : 0); DetectPortPrint(new); printf("\n"); //DetectPortPrintList(*head); #endif @@ -235,6 +245,7 @@ int DetectPortInsert(DetectPort **head, DetectPort *new) { /* exact overlap/match */ if (cur != new) { SigGroupHeadCopySigs(new->sh,&cur->sh); + cur->cnt += new->cnt; DetectPortFree(new); return 0; } @@ -563,6 +574,9 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { /* default to NULL */ *c = NULL; + //printf("a (%p): ",a); DetectPortPrint(a); printf(" "); DbgPrintSigs2(a->sh); + //printf("b (%p): ",b); DetectPortPrint(b); printf(" "); DbgPrintSigs2(b->sh); + int r = DetectPortCmp(a,b); if (r != PORT_ES && r != PORT_EB && r != PORT_LE && r != PORT_GE) { printf("DetectPortCut: we shouldn't be here\n"); @@ -605,6 +619,9 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { SigGroupHeadCopySigs(b->sh,&tmp_c->sh); /* copy old b to c */ SigGroupHeadCopySigs(a->sh,&b->sh); /* copy old b to a */ + tmp_c->cnt += b->cnt; + b->cnt += a->cnt; + /* we have 3 parts: [bbb[baba]aaa] * part a: b_port1 <-> a_port1 - 1 * part b: a_port1 <-> b_port2 @@ -640,6 +657,14 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { SigGroupHeadCopySigs(tmp->sh,&b->sh); /* prepend old a before b */ SigGroupHeadClearSigs(tmp->sh); /* clean tmp list */ + + tmp->cnt += a->cnt; + a->cnt = 0; + tmp_c->cnt += tmp->cnt; + a->cnt += b->cnt; + b->cnt += tmp->cnt; + tmp->cnt = 0; + /* we have 2 or three parts: * * 2 part: [[abab]bbb] or [bbb[baba]] @@ -672,6 +697,7 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { /* 'b' overlaps 'a' so 'a' needs the 'b' sigs */ SigGroupHeadCopySigs(b->sh,&a->sh); + a->cnt += b->cnt; } else if (a_port2 == b_port2) { #ifdef DBG @@ -685,6 +711,8 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { /* 'a' overlaps 'b' so a needs the 'a' sigs */ SigGroupHeadCopySigs(a->sh,&b->sh); + b->cnt += a->cnt; + } else { #ifdef DBG printf("DetectPortCut: 3\n"); @@ -715,6 +743,13 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { SigGroupHeadCopySigs(tmp->sh,&b->sh); /* prepend old a before b */ SigGroupHeadClearSigs(tmp->sh); /* clean tmp list */ + + tmp->cnt += a->cnt; + a->cnt = 0; + tmp_c->cnt += b->cnt; + a->cnt += b->cnt; + b->cnt += tmp->cnt; + tmp->cnt = 0; } /* we have 2 or three parts: * @@ -753,6 +788,15 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { SigGroupHeadCopySigs(tmp->sh,&a->sh); SigGroupHeadClearSigs(tmp->sh); + + tmp->cnt += b->cnt; + b->cnt = 0; + b->cnt += a->cnt; + a->cnt += tmp->cnt; + tmp->cnt = 0; + + //printf("2a (%p): ",a); DetectPortPrint(a); printf(" "); DbgPrintSigs2(a->sh); + //printf("2b (%p): ",b); DetectPortPrint(b); printf(" "); DbgPrintSigs2(b->sh); } else if (a_port2 == b_port2) { #ifdef DBG printf("DetectPortCut: 2\n"); @@ -765,6 +809,9 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { /* 'a' overlaps 'b' so a needs the 'a' sigs */ SigGroupHeadCopySigs(a->sh,&b->sh); + + b->cnt += a->cnt; + } else { #ifdef DBG printf("DetectPortCut: 3\n"); @@ -787,6 +834,9 @@ int DetectPortCut(DetectPort *a, DetectPort *b, DetectPort **c) { SigGroupHeadCopySigs(a->sh,&b->sh); SigGroupHeadCopySigs(a->sh,&tmp_c->sh); + + b->cnt += a->cnt; + tmp_c->cnt += a->cnt; } } @@ -1028,15 +1078,15 @@ DetectPortLookupGroup(DetectPort *dp, u_int16_t port) { /* XXX eeewww global! move to DetectionEngineCtx once we have that! */ static DetectPort **port_hash; static DetectPort *port_list; -#define HASH_SIZE 65536 +#define PORT_HASH_SIZE 1024 /* XXX dynamic size based on number of sigs? */ int DetectPortHashInit(void) { - port_hash = (DetectPort **)malloc(sizeof(DetectPort) * HASH_SIZE); + port_hash = (DetectPort **)malloc(sizeof(DetectPort *) * PORT_HASH_SIZE); if (port_hash == NULL) { goto error; } - memset(port_hash,0,sizeof(DetectPort) * HASH_SIZE); + memset(port_hash,0,sizeof(DetectPort *) * PORT_HASH_SIZE); port_list = NULL; @@ -1052,7 +1102,7 @@ void DetectPortHashFree(void) { void DetectPortHashReset(void) { if (port_hash != NULL) { - memset(port_hash,0,sizeof(DetectPort) * HASH_SIZE); + memset(port_hash,0,sizeof(DetectPort *) * PORT_HASH_SIZE); } port_list = NULL; } @@ -1066,13 +1116,13 @@ DetectPort *DetectPortHashGetListPtr(void) { } u_int32_t DetectPortHashGetSize(void) { - return HASH_SIZE; + return PORT_HASH_SIZE; } static inline u_int32_t DetectPortHash(DetectPort *p) { u_int32_t hash = p->port * p->port2; - return (hash % HASH_SIZE); + return (hash % PORT_HASH_SIZE); } int DetectPortHashAdd(DetectPort *p) { @@ -1151,6 +1201,147 @@ DetectPort *DetectPortHashLookup(DetectPort *p) { return NULL; } +/* XXX eeewww global! move to DetectionEngineCtx once we have that! */ +static DetectPort **sport_hash; +static DetectPort *sport_list; +#define SPORT_HASH_SIZE 1024 + +/* XXX dynamic size based on number of sigs? */ +int DetectPortSpHashInit(void) { + sport_hash = (DetectPort **)malloc(sizeof(DetectPort *) * SPORT_HASH_SIZE); + if (sport_hash == NULL) { + goto error; + } + memset(sport_hash,0,sizeof(DetectPort *) * SPORT_HASH_SIZE); + + sport_list = NULL; + //printf("DetectSPortHashInit: sport_hash %p\n", sport_hash); + return 0; +error: + printf("DetectSPortHashInit: error sport_hash %p\n", sport_hash); + return -1; +} + +void DetectPortSpHashFree(void) { + free(sport_hash); + sport_hash = NULL; +} + +void DetectPortSpHashReset(void) { + if (sport_hash != NULL) { + memset(sport_hash,0,sizeof(DetectPort *) * SPORT_HASH_SIZE); + } + sport_list = NULL; +} + +DetectPort **DetectPortSpHashGetPtr(void) { + return sport_hash; +} + +DetectPort *DetectPortSpHashGetListPtr(void) { + return sport_list; +} + +u_int32_t DetectPortSpHashGetSize(void) { + return SPORT_HASH_SIZE; +} + +static inline u_int32_t DetectPortSpHash(DetectPort *p) { + u_int32_t hash = p->port * p->port2; + + return (hash % SPORT_HASH_SIZE); +} + +int DetectPortSpHashAdd(DetectPort *p) { + u_int32_t hash = DetectPortSpHash(p); + + //printf("DetectSPortHashAdd: hash %u\n", hash); + detect_port_hash_add_cnt++; + + /* list */ + p->next = sport_list; + sport_list = p; + + /* easy: no collision */ + if (sport_hash[hash] == NULL) { + sport_hash[hash] = p; + return 0; + } + + detect_port_hash_add_coll_cnt++; + + /* harder: collision */ + DetectPort *h = sport_hash[hash], *ph = NULL; + for ( ; h != NULL; h = h->hnext) { +#if 0 + if (DetectPortCmp(p,h) == PORT_EB) { + if (h == port_hash[hash]) { + p->hnext = h; + port_hash[hash] = p; + } else { + p->hnext = ph->hnext; + ph->hnext = p; + } + detect_port_hash_add_insert_cnt++; + return 0; + } +#endif + ph = h; + } + ph->hnext = p; + + return 0; +} + +DetectPort *DetectPortSpHashLookup(DetectPort *p) { + u_int32_t hash = DetectPortSpHash(p); + + //printf("DetectSPortHashLookup: hash %u, sport_hash %p, size %u port %p\n", hash, sport_hash, SPORT_HASH_SIZE, p); + detect_port_hash_lookup_cnt++; + + /* easy: no sgh at our hash */ + if (sport_hash[hash] == NULL) { + detect_port_hash_lookup_miss_cnt++; + //printf("DetectSPortHashLookup: not found\n"); + return NULL; + } + + /* see if we have the sgh we're looking for */ + DetectPort *h = sport_hash[hash]; + for ( ; h != NULL; h = h->hnext) { + detect_port_hash_lookup_loop_cnt++; + if (DetectPortHashCmp(p,h) == 1) { + //printf("DetectSPortHashLookup: found at %p\n", h); + detect_port_hash_lookup_hit_cnt++; + return h; + } + } + + //printf("DetectSPortHashLookup: not found\n"); + return NULL; +} + +int DetectPortJoin(DetectPort *target, DetectPort *source) { + if (target == NULL || source == NULL) + return -1; + + target->cnt += source->cnt; + SigGroupHeadCopySigs(source->sh,&target->sh); + + //DetectPort *port = source->port; + //for ( ; port != NULL; port = port->next) { + // DetectPortInsertCopy(&target->port, port); + //} + + if (source->port < target->port) + target->port = source->port; + + if (source->port2 > target->port2) + target->port2 = source->port2; + + return -1; +} + /* TESTS */ int PortTestParse01 (void) { diff --git a/src/detect-engine-port.h b/src/detect-engine-port.h index 7e3946af16..7c0d068386 100644 --- a/src/detect-engine-port.h +++ b/src/detect-engine-port.h @@ -17,6 +17,7 @@ enum { #define PORT_FLAG_NOT 0x2 #define PORT_SIGGROUPHEAD_COPY 0x04 +#define PORT_GROUP_PORTS_COPY 0x08 typedef struct DetectPort_ { u_int8_t flags; @@ -27,13 +28,16 @@ typedef struct DetectPort_ { /* signatures that belong in this group */ struct _SigGroupHead *sh; + struct DetectPort_ *dst_ph; + /* double linked list */ union { struct DetectPort_ *prev; - struct DetectPort_ *hnext; + struct DetectPort_ *hnext; /* hash next */ }; struct DetectPort_ *next; + u_int32_t cnt; } DetectPort; /* prototypes */ @@ -63,5 +67,16 @@ void DetectPortHashFree(void); int DetectPortHashAdd(DetectPort *p); void DetectPortHashReset(void); +DetectPort *DetectPortSpHashLookup(DetectPort *p); +DetectPort **DetectPortSpHashGetPtr(void); +DetectPort *DetectPortSpHashGetListPtr(void); +u_int32_t DetectPortSpHashGetSize(void); +int DetectPortSpHashInit(void); +void DetectPortSpHashFree(void); +int DetectPortSpHashAdd(DetectPort *p); +void DetectPortSpHashReset(void); + +int DetectPortJoin(DetectPort *target, DetectPort *source); + #endif /* __DETECT_PORT_H__ */ diff --git a/src/detect-engine-proto.c b/src/detect-engine-proto.c index b5c99cf662..ad2d88cd4e 100644 --- a/src/detect-engine-proto.c +++ b/src/detect-engine-proto.c @@ -74,7 +74,7 @@ int DetectProtoSetup (Signature *s, SigMatch *m, char *str) /* TESTS */ -int ProtoTestParse01 (void) { +static int ProtoTestParse01 (void) { DetectProto dp; memset(&dp,0,sizeof(DetectProto)); @@ -86,8 +86,34 @@ int ProtoTestParse01 (void) { return 0; } +static int ProtoTestParse02 (void) { + DetectProto dp; + memset(&dp,0,sizeof(DetectProto)); + + int r = DetectProtoParse(&dp, "tcp"); + if (r == 0 && dp.proto[(IPPROTO_TCP/8)] & (1<<(IPPROTO_TCP%8))) { + return 1; + } + + return 0; +} + +static int ProtoTestParse03 (void) { + DetectProto dp; + memset(&dp,0,sizeof(DetectProto)); + + int r = DetectProtoParse(&dp, "ip"); + if (r == 0 && dp.flags & DETECT_PROTO_ANY) { + return 1; + } + + return 0; +} + void DetectProtoTests(void) { UtRegisterTest("ProtoTestParse01", ProtoTestParse01, 1); + UtRegisterTest("ProtoTestParse02", ProtoTestParse02, 1); + UtRegisterTest("ProtoTestParse03", ProtoTestParse03, 1); } diff --git a/src/detect-engine-siggroup.c b/src/detect-engine-siggroup.c index 10403e9ab9..59f6bbb7a9 100644 --- a/src/detect-engine-siggroup.c +++ b/src/detect-engine-siggroup.c @@ -57,12 +57,13 @@ static int SigGroupHeadCmpSigArray(SigGroupHead *a, SigGroupHead *b) { /* hashes */ /* XXX eeewww global! move to DetectionEngineCtx once we have that! */ -static SigGroupHead **sgh_port_hash; -static SigGroupHead **sgh_hash; -static SigGroupHead **sgh_mpm_hash; -static SigGroupHead **sgh_mpm_uri_hash; +static SigGroupHead **sgh_port_hash = NULL; +static SigGroupHead **sgh_sport_hash = NULL; +static SigGroupHead **sgh_hash = NULL; +static SigGroupHead **sgh_mpm_hash = NULL; +static SigGroupHead **sgh_mpm_uri_hash = NULL; -#define HASH_SIZE 65536 +#define HASH_SIZE 262144 /* mpm sgh hash */ @@ -348,6 +349,66 @@ void SigGroupHeadPortHashFree(void) { sgh_port_hash = NULL; } +/* XXX dynamic size based on number of sigs? */ +int SigGroupHeadSPortHashInit(void) { + sgh_sport_hash = (SigGroupHead **)malloc(sizeof(SigGroupHead *) * HASH_SIZE); + if (sgh_sport_hash == NULL) { + goto error; + } + memset(sgh_sport_hash,0,sizeof(SigGroupHead *) * HASH_SIZE); + + return 0; +error: + return -1; +} + +int SigGroupHeadSPortHashAdd(SigGroupHead *sgh) { + u_int32_t hash = SigGroupHeadHash(sgh); + + //printf("SigGroupHeadHashAdd: hash %u\n", hash); + + /* easy: no collision */ + if (sgh_sport_hash[hash] == NULL) { + sgh_sport_hash[hash] = sgh; + return 0; + } + + /* harder: collision */ + SigGroupHead *h = sgh_sport_hash[hash], *ph = NULL; + for ( ; h != NULL; h = h->next) { + ph = h; + } + ph->next = sgh; + + return 0; +} + +SigGroupHead *SigGroupHeadSPortHashLookup(SigGroupHead *sgh) { + u_int32_t hash = SigGroupHeadHash(sgh); + + //printf("SigGroupHeadHashLookup: hash %u\n", hash); + + /* easy: no sgh at our hash */ + if (sgh_sport_hash[hash] == NULL) { + return NULL; + } + + /* see if we have the sgh we're looking for */ + SigGroupHead *h = sgh_sport_hash[hash]; + for ( ; h != NULL; h = h->next) { + if (SigGroupHeadCmpSigArray(sgh,h) == 1) { + return h; + } + } + + return NULL; +} + +void SigGroupHeadSPortHashFree(void) { + free(sgh_sport_hash); + sgh_sport_hash = NULL; +} + /* end hashes */ void SigGroupHeadFreeHeads(void) { @@ -379,25 +440,12 @@ printf("SigGroupHeadFreeHeads: want to free %p\n", b); } } -/* Free the sigarrays in the sgh's. Those are only - * used during the init stage. */ -void SigGroupHeadFreeSigArrays(void) { +static void SigGroupHeadFreeSigArraysHash(SigGroupHead **hashtbl) { SigGroupHead *b; u_int32_t hash = 0; for ( ; hash < HASH_SIZE; hash++) { - b = sgh_hash[hash]; - for ( ; b != NULL; b = b->next) { - if (b->sig_array != NULL) { - detect_siggroup_sigarray_free_cnt++; - detect_siggroup_sigarray_memory -= b->sig_size; - - free(b->sig_array); - b->sig_array = NULL; - b->sig_size = 0; - } - } - b = sgh_port_hash[hash]; + b = hashtbl[hash]; for ( ; b != NULL; b = b->next) { if (b->sig_array != NULL) { detect_siggroup_sigarray_free_cnt++; @@ -411,6 +459,14 @@ void SigGroupHeadFreeSigArrays(void) { } } +/* Free the sigarrays in the sgh's. Those are only + * used during the init stage. */ +void SigGroupHeadFreeSigArrays(void) { + SigGroupHeadFreeSigArraysHash(sgh_hash); + SigGroupHeadFreeSigArraysHash(sgh_port_hash); + SigGroupHeadFreeSigArraysHash(sgh_sport_hash); +} + /* Free the mpm arrays that are only used during the * init stage */ void SigGroupHeadFreeMpmArrays(void) { @@ -444,6 +500,19 @@ void SigGroupHeadFreeMpmArrays(void) { b->uri_content_size = 0; } } + b = sgh_sport_hash[hash]; + for ( ; b != NULL; b = b->next) { + if (b->content_array != NULL) { + free(b->content_array); + b->content_array = NULL; + b->content_size = 0; + } + if (b->uri_content_array != NULL) { + free(b->uri_content_array); + b->uri_content_array = NULL; + b->uri_content_size = 0; + } + } } } @@ -560,16 +629,29 @@ void DetectSigGroupPrintMemory(void) { void SigGroupHeadPrintContent(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { printf("SigGroupHeadPrintContent: "); - u_int32_t sig; - for (sig = 0; sig < sgh->sig_cnt; sig++) { - u_int32_t num = sgh->match_array[sig]; - - Signature *s = de_ctx->sig_array[num]; - printf("%u ", s->id); + u_int32_t i; + for (i = 0; i < DetectContentMaxId(); i++) { + if (sgh->content_array[(i/8)] & (1<<(i%8))) { + printf("%u ", i); + } } + printf("\n"); } +void SigGroupHeadPrintContentCnt(DetectEngineCtx *de_ctx, SigGroupHead *sgh) { + printf("SigGroupHeadPrintContent: "); + + u_int32_t i, cnt = 0; + for (i = 0; i < DetectContentMaxId(); i++) { + if (sgh->content_array[(i/8)] & (1<<(i%8))) { + cnt++; + } + } + + printf("cnt %u\n", cnt); +} + /* load all pattern id's into a single bitarray that we can memcmp * with other bitarrays. A fast and efficient way of comparing pattern * sets. */ diff --git a/src/detect-engine-siggroup.h b/src/detect-engine-siggroup.h index 66831efb58..7ecbb42e09 100644 --- a/src/detect-engine-siggroup.h +++ b/src/detect-engine-siggroup.h @@ -18,22 +18,26 @@ void SigGroupHeadFreeMpmArrays(void); SigGroupHead *SigGroupHeadHashLookup(SigGroupHead *sgh); SigGroupHead *SigGroupHeadPortHashLookup(SigGroupHead *sgh); +SigGroupHead *SigGroupHeadSPortHashLookup(SigGroupHead *sgh); SigGroupHead *SigGroupHeadMpmHashLookup(SigGroupHead *sgh); SigGroupHead *SigGroupHeadMpmUriHashLookup(SigGroupHead *sgh); int SigGroupHeadPortHashAdd(SigGroupHead *sgh); +int SigGroupHeadSPortHashAdd(SigGroupHead *sgh); int SigGroupHeadMpmHashAdd(SigGroupHead *sgh); int SigGroupHeadMpmUriHashAdd(SigGroupHead *sgh); int SigGroupHeadHashAdd(SigGroupHead *sgh); void SigGroupHeadHashFree(void); void SigGroupHeadPortHashFree(void); +void SigGroupHeadSPortHashFree(void); void SigGroupHeadMpmHashFree(void); void SigGroupHeadMpmUriHashFree(void); int SigGroupHeadMpmHashInit(void); int SigGroupHeadMpmUriHashInit(void); int SigGroupHeadPortHashInit(void); +int SigGroupHeadSPortHashInit(void); int SigGroupHeadHashInit(void); void SigGroupHeadSetSigCnt(SigGroupHead *sgh, u_int32_t max_idx); diff --git a/src/detect-parse.c b/src/detect-parse.c index 08d4b50f87..2c2763a49b 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -17,6 +17,9 @@ static pcre_extra *option_pcre_extra = NULL; /* XXX this should be part of the DE */ static u_int32_t signum = 0; +static u_int32_t dbg_srcportany_cnt = 0; +static u_int32_t dbg_dstportany_cnt = 0; + #define CONFIG_PARTS 8 #define CONFIG_ACTION 0 @@ -37,6 +40,14 @@ u_int32_t SigGetMaxId(void) { return signum; } +u_int32_t DbgGetSrcPortAnyCnt(void) { + return dbg_srcportany_cnt; +} + +u_int32_t DbgGetDstPortAnyCnt(void) { + return dbg_dstportany_cnt; +} + void SigResetMaxId(void) { signum = 0; } @@ -250,10 +261,16 @@ int SigParseAddress(Signature *s, const char *addrstr, char flag) { /* pass on to the address(list) parser */ if (flag == 0) { + if (strcasecmp(addrstr,"any") == 0) + s->flags |= SIG_FLAG_SRC_ANY; + if (DetectAddressGroupParse(&s->src,addr) < 0) { goto error; } } else { + if (strcasecmp(addrstr,"any") == 0) + s->flags |= SIG_FLAG_DST_ANY; + if (DetectAddressGroupParse(&s->dst,addr) < 0) { goto error; } @@ -299,10 +316,15 @@ int SigParsePort(Signature *s, const char *portstr, char flag) { } if (flag == 0) { + if (strcasecmp(port,"any") == 0) + s->flags |= SIG_FLAG_SP_ANY; + r = DetectPortParse(&s->sp,(char *)port); } else if (flag == 1) { + if (strcasecmp(port,"any") == 0) + s->flags |= SIG_FLAG_DP_ANY; + r = DetectPortParse(&s->dp,(char *)port); - //DetectPortPrintList(s->dp); } if (r < 0) { printf("SigParsePort: DetectPortParse \"%s\" failed\n", portstr); @@ -372,12 +394,15 @@ int SigParseBasics(Signature *s, char *sigstr, char ***result) { /* Parse Address & Ports */ if (SigParseAddress(s, arr[CONFIG_SRC], 0) < 0) goto error; - if (SigParsePort(s, arr[CONFIG_SP], 0) < 0) - goto error; + if (strcasecmp(arr[CONFIG_PROTO],"tcp") == 0 || + strcasecmp(arr[CONFIG_PROTO],"udp") == 0) { + if (SigParsePort(s, arr[CONFIG_SP], 0) < 0) + goto error; + if (SigParsePort(s, arr[CONFIG_DP], 1) < 0) + goto error; + } if (SigParseAddress(s, arr[CONFIG_DST], 1) < 0) goto error; - if (SigParsePort(s, arr[CONFIG_DP], 1) < 0) - goto error; *result = (char **)arr; DEBUGPRINT("SigParseBasics: %p %p", arr, *result); @@ -479,8 +504,6 @@ int SigParseTest01 (void) { int result = 1; Signature *sig = NULL; - //SigParsePrepare(); - sig = SigInit("alert tcp 1.2.3.4 any -> !1.2.3.4 any (msg:\"SigParseTest01\"; sid:1;)"); if (sig == NULL) { result = 0; diff --git a/src/detect.c b/src/detect.c index 795d589b60..4c77317f21 100644 --- a/src/detect.c +++ b/src/detect.c @@ -49,6 +49,8 @@ static u_int32_t mpm_memory_size = 0; SigMatch *SigMatchAlloc(void); void SigMatchFree(SigMatch *sm); +int SignatureTupleCmp(SignatureTuple *a, SignatureTuple *b); +int SignatureTupleCmpRaw(DetectAddressGroup *src, DetectAddressGroup *dst, DetectPort *sp, DetectPort *dp, u_int8_t proto, SignatureTuple *b); /* tm module api functions */ int Detect(ThreadVars *, Packet *, void *); @@ -74,21 +76,21 @@ void SigLoadSignatures (void) /* The next 3 rules handle HTTP header capture. */ /* http_uri -- for uricontent */ - sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP URI cap\"; flow:to_server; content:\"GET \"; depth:4; pcre:\"/^GET (?P.*) HTTP\\/\\d\\.\\d\\r\\n/G\"; depth:400; noalert; sid:1;)"); + sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP URI cap\"; flow:to_server; content:\"GET \"; depth:4; pcre:\"/^GET (?P.*) HTTP\\/\\d\\.\\d\\r\\n/G\"; noalert; sid:1;)"); if (sig) { prevsig = sig; g_de_ctx->sig_list = sig; } /* http_host -- for the log-httplog module */ - sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP host cap\"; flow:to_server; content:\"Host:\"; depth:400; pcre:\"/^Host: (?P.*)\\r\\n/m\"; noalert; sid:2;)"); + sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP host cap\"; flow:to_server; content:\"Host:\"; pcre:\"/^Host: (?P.*)\\r\\n/m\"; noalert; sid:2;)"); if (sig == NULL) return; prevsig->next = sig; prevsig = sig; /* http_ua -- for the log-httplog module */ - sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP UA cap\"; flow:to_server; content:\"User-Agent:\"; depth:400; pcre:\"/^User-Agent: (?P.*)\\r\\n/m\"; noalert; sid:3;)"); + sig = SigInit("alert tcp any any -> any $HTTP_PORTS (msg:\"HTTP UA cap\"; flow:to_server; content:\"User-Agent:\"; pcre:\"/^User-Agent: (?P.*)\\r\\n/m\"; noalert; sid:3;)"); if (sig == NULL) return; prevsig->next = sig; @@ -192,14 +194,15 @@ void SigLoadSignatures (void) //#if 0 int good = 0, bad = 0; - FILE *fp = fopen("/etc/vips/rules/bleeding-all.rules", "r"); + //FILE *fp = fopen("/etc/vips/rules/bleeding-all.rules", "r"); + FILE *fp = fopen("/home/victor/rules/all.rules", "r"); //FILE *fp = fopen("/home/victor/rules/vips-http.sigs", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-dshield.rules", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-web.rules", "r"); //FILE *fp = fopen("/home/victor/rules/emerging-web-small.rules", "r"); //FILE *fp = fopen("/home/victor/rules/web-misc.rules", "r"); + //FILE *fp = fopen("/home/victor/rules/emerging-malware.rules", "r"); //FILE *fp = fopen("/home/victor/rules/vips-all.sigs", "r"); - //FILE *fp = fopen("/home/victor/rules/all.rules", "r"); //FILE *fp = fopen("/home/victor/rules/all_noip.rules", "r"); //FILE *fp = fopen("/home/victor/rules/all_iplists.rules", "r"); //FILE *fp = fopen("/home/victor/rules/funky.rules", "r"); @@ -228,6 +231,7 @@ void SigLoadSignatures (void) } fclose(fp); printf("SigLoadSignatures: %d successfully loaded from file. %d sigs failed to load\n", good, bad); + printf("SigLoadSignatures: %u sigs with dstportany\n", DbgGetDstPortAnyCnt()); //#endif @@ -341,11 +345,14 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) } else { //printf("SigMatchSignatures: we have ports\n"); - DetectPort *port = DetectPortLookupGroup(ag->port,p->dp); - if (port != NULL) { - pmt->mc = port->sh->mpm_ctx; - pmt->mcu = port->sh->mpm_uri_ctx; - sgh = port->sh; + DetectPort *sport = DetectPortLookupGroup(ag->port,p->sp); + if (sport != NULL) { + DetectPort *dport = DetectPortLookupGroup(sport->dst_ph,p->dp); + if (dport != NULL) { + pmt->mc = dport->sh->mpm_ctx; + pmt->mcu = dport->sh->mpm_uri_ctx; + sgh = dport->sh; + } } } } @@ -370,6 +377,30 @@ int SigMatchSignatures(ThreadVars *th_v, PatternMatcherThread *pmt, Packet *p) sig = sgh->match_array[idx]; s = g_de_ctx->sig_array[sig]; + if (!(s->flags & SIG_FLAG_SRC_ANY)) { + DetectAddressGroup *saddr = DetectAddressLookupGroup(&s->src,&p->src); + if (saddr == NULL) + continue; + } + + if (!(s->flags & SIG_FLAG_DST_ANY)) { + DetectAddressGroup *daddr = DetectAddressLookupGroup(&s->dst,&p->dst); + if (daddr == NULL) + continue; + } + /* check the source port in the sig */ + if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP) { + if (!(s->flags & SIG_FLAG_SP_ANY)) { + DetectPort *sport = DetectPortLookupGroup(s->sp,p->sp); + if (sport == NULL) + continue; + } + if (!(s->flags & SIG_FLAG_DP_ANY)) { + DetectPort *dport = DetectPortLookupGroup(s->dp,p->dp); + if (dport == NULL) + continue; + } + } /* XXX maybe a (re)set function? */ pmt->pkt_ptr = NULL; pmt->pkt_off = 0; @@ -534,15 +565,15 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) { for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { de_ctx->sig_array[tmp_s->num] = tmp_s; - //printf(" + Signature %u, internal id %u, ptrs %p %p ", tmp_s->id, tmp_s->num, tmp_s, de_ctx->sig_array[tmp_s->num]); + printf(" + Signature %u, internal id %u, ptrs %p %p ", tmp_s->id, tmp_s->num, tmp_s, de_ctx->sig_array[tmp_s->num]); /* see if the sig is ip only */ if (SignatureIsIPOnly(tmp_s) == 1) { tmp_s->flags |= SIG_FLAG_IPONLY; cnt_iponly++; - //printf("(IP only)\n"); + printf("(IP only)\n"); } else { - //printf("\n"); + printf("\n"); } for (gr = tmp_s->src.ipv4_head; gr != NULL; gr = gr->next) { @@ -621,6 +652,7 @@ static int BuildSourceAddressList(DetectEngineCtx *de_ctx, Signature *s, int fam goto error; } grtmp->ad = adtmp; + grtmp->cnt = 1; SigGroupHeadAppendSig(&grtmp->sh, s); @@ -635,6 +667,7 @@ static int BuildSourceAddressList(DetectEngineCtx *de_ctx, Signature *s, int fam } else { /* our group will only have one sig, this one. So add that. */ SigGroupHeadAppendSig(&lookup_gr->sh, s); + lookup_gr->cnt++; } } } @@ -663,6 +696,7 @@ static int BuildSourceAddressList(DetectEngineCtx *de_ctx, Signature *s, int fam goto error; } grtmp->ad = adtmp; + grtmp->cnt = 1; if (family == AF_INET) { DetectAddressGroupAdd(&de_ctx->io_tmp_gh->ipv4_head, grtmp); @@ -676,6 +710,7 @@ static int BuildSourceAddressList(DetectEngineCtx *de_ctx, Signature *s, int fam } else { /* our group will only have one sig, this one. So add that. */ SigGroupHeadAppendSig(&lookup_gr->sh, s); + lookup_gr->cnt++; } SigGroupHeadFree(gr->sh); @@ -688,6 +723,302 @@ error: return -1; } +static DetectAddressGroup *GetHeadPtr(DetectAddressGroupsHead *head, int family) { + DetectAddressGroup *grhead; + + if (head == NULL) + return NULL; + + if (family == AF_INET) { + grhead = head->ipv4_head; + } else if (family == AF_INET6) { + grhead = head->ipv6_head; + } else { + grhead = head->any_head; + } + + return grhead; +} + +#define MAX_UNIQ_GROUPS 5 + +/* set unique_groups to 0 for no grouping. + * + * srchead is a ordered "inserted" list w/o internal overlap + * + */ +int CreateGroupedAddrList(DetectAddressGroup *srchead, int family, DetectAddressGroupsHead *newhead, u_int32_t unique_groups) { + DetectAddressGroup *tmplist = NULL, *tmplist2 = NULL, *joingr = NULL; + char insert = 0; + DetectAddressGroup *gr, *next_gr; + u_int32_t groups = 0; + + /* insert the addresses into the tmplist, where it will + * be sorted descending on 'cnt'. */ + for (gr = srchead; gr != NULL; gr = gr->next) { + groups++; + + /* alloc a copy */ + DetectAddressGroup *newtmp = DetectAddressGroupInit(); + if (newtmp == NULL) { + goto error; + } + DetectAddressData *adtmp = DetectAddressDataCopy(gr->ad); + if (adtmp == NULL) { + goto error; + } + newtmp->ad = adtmp; + newtmp->cnt = gr->cnt; + + SigGroupHeadCopySigs(gr->sh,&newtmp->sh); + DetectPort *port = gr->port; + for ( ; port != NULL; port = port->next) { + DetectPortInsertCopy(&newtmp->port, port); + } + + /* insert it */ + DetectAddressGroup *tmpgr = tmplist, *prevtmpgr; + if (tmplist == NULL) { + /* empty list, set head */ + tmplist = newtmp; + } else { + /* look for the place to insert */ + for ( ; tmpgr != NULL&&!insert; tmpgr = tmpgr->next) { + if (gr->cnt > tmpgr->cnt) { + if (tmpgr == tmplist) { + newtmp->next = tmplist; + tmplist = newtmp; + } else { + newtmp->next = prevtmpgr->next; + prevtmpgr->next = newtmp; + } + insert = 1; + } + prevtmpgr = tmpgr; + } + if (insert == 0) { + newtmp->next = NULL; + prevtmpgr->next = newtmp; + } + insert = 0; + } + } + + u_int32_t i = unique_groups; + if (i == 0) i = groups; + + for (gr = tmplist; gr != NULL; ) { + if (i == 0) { + if (joingr == NULL) { + joingr = DetectAddressGroupInit(); + if (joingr == NULL) { + goto error; + } + DetectAddressData *adtmp = DetectAddressDataCopy(gr->ad); + if (adtmp == NULL) { + goto error; + } + joingr->ad = adtmp; + joingr->cnt = gr->cnt; + + SigGroupHeadCopySigs(gr->sh,&joingr->sh); + + DetectPort *port = gr->port; + for ( ; port != NULL; port = port->next) { + DetectPortInsertCopy(&joingr->port, port); + } + } else { + DetectAddressGroupJoin(joingr, gr); + } + } else { + DetectAddressGroup *newtmp = DetectAddressGroupInit(); + if (newtmp == NULL) { + goto error; + } + DetectAddressData *adtmp = DetectAddressDataCopy(gr->ad); + if (adtmp == NULL) { + goto error; + } + newtmp->ad = adtmp; + newtmp->cnt = gr->cnt; + + SigGroupHeadCopySigs(gr->sh,&newtmp->sh); + + DetectPort *port = gr->port; + for ( ; port != NULL; port = port->next) { + DetectPortInsertCopy(&newtmp->port, port); + } + + if (tmplist2 == NULL) { + tmplist2 = newtmp; + } else { + newtmp->next = tmplist2; + tmplist2 = newtmp; + } + } + if (i)i--; + + next_gr = gr->next; + DetectAddressGroupFree(gr); + gr = next_gr; + } + + /* we now have a tmplist2 containing the 'unique' groups and + * possibly a joingr that covers the rest. Now build the newhead + * that we will pass back to the caller. + * + * Start with inserting the unique groups */ + for (gr = tmplist2; gr != NULL; ) { + DetectAddressGroup *newtmp = DetectAddressGroupInit(); + if (newtmp == NULL) { + goto error; + } + DetectAddressData *adtmp = DetectAddressDataCopy(gr->ad); + if (adtmp == NULL) { + goto error; + } + newtmp->ad = adtmp; + newtmp->cnt = gr->cnt; + + SigGroupHeadCopySigs(gr->sh,&newtmp->sh); + + DetectPort *port = gr->port; + for ( ; port != NULL; port = port->next) { + DetectPortInsertCopy(&newtmp->port, port); + } + + DetectAddressGroupInsert(newhead,newtmp); + + next_gr = gr->next; +// DetectAddressGroupFree(gr); + gr = next_gr; + } + /* if present, insert the joingr that covers the rest */ + if (joingr != NULL) { + DetectAddressGroupInsert(newhead,joingr); + + /* mark the groups that are not unique */ + DetectAddressGroup *ag = GetHeadPtr(newhead,family); + DetectAddressGroup *agr = NULL; + + for (agr = ag; agr != NULL; agr = agr->next) { + DetectAddressGroup *sgr = tmplist2; + for ( ; sgr != NULL; sgr = sgr->next) { + int r = DetectAddressCmp(agr->ad,sgr->ad); + if (r == ADDRESS_ES || r == ADDRESS_EB) { + printf("AGR "); DetectAddressDataPrint(agr->ad);printf(" -> "); + printf(" sgr "); DetectAddressDataPrint(sgr->ad);printf("\n"); + } + } + } + + } + + return 0; +error: + return -1; +} + +int CreateGroupedPortList(DetectPort *srchead, DetectPort **newhead, u_int32_t unique_groups) { + DetectPort *tmplist = NULL, *tmplist2 = NULL, *joingr = NULL; + char insert = 0; + DetectPort *gr, *next_gr; + + /* insert the addresses into the tmplist, where it will + * be sorted descending on 'cnt'. */ + for (gr = srchead; gr != NULL; gr = gr->next) { + /* alloc a copy */ + DetectPort *newtmp = DetectPortCopySingle(gr); + if (newtmp == NULL) { + goto error; + } + + /* insert it */ + DetectPort *tmpgr = tmplist, *prevtmpgr; + if (tmplist == NULL) { + /* empty list, set head */ + tmplist = newtmp; + } else { + /* look for the place to insert */ + for ( ; tmpgr != NULL&&!insert; tmpgr = tmpgr->next) { + if (gr->cnt > tmpgr->cnt) { + if (tmpgr == tmplist) { + newtmp->next = tmplist; + tmplist = newtmp; + } else { + newtmp->next = prevtmpgr->next; + prevtmpgr->next = newtmp; + } + insert = 1; + } + prevtmpgr = tmpgr; + } + if (insert == 0) { + newtmp->next = NULL; + prevtmpgr->next = newtmp; + } + insert = 0; + } + } + + u_int32_t i = unique_groups; + for (gr = tmplist; gr != NULL; ) { + if (i == 0) { + if (joingr == NULL) { + joingr = DetectPortCopySingle(gr); + if (joingr == NULL) { + goto error; + } + } else { + DetectPortJoin(joingr, gr); + } + } else { + DetectPort *newtmp = DetectPortCopySingle(gr); + if (newtmp == NULL) { + goto error; + } + + if (tmplist2 == NULL) { + tmplist2 = newtmp; + } else { + newtmp->next = tmplist2; + tmplist2 = newtmp; + } + } + if (i)i--; + + next_gr = gr->next; + DetectPortFree(gr); + gr = next_gr; + } + + /* we now have a tmplist2 containing the 'unique' groups and + * possibly a joingr that covers the rest. Now build the newhead + * that we will pass back to the caller. + * + * Start with inserting the unique groups */ + for (gr = tmplist2; gr != NULL; ) { + DetectPort *newtmp = DetectPortCopySingle(gr); + if (newtmp == NULL) { + goto error; + } + + DetectPortInsert(newhead,newtmp); + + next_gr = gr->next; + DetectPortFree(gr); + gr = next_gr; + } + /* if present, insert the joingr that covers the rest */ + if (joingr != NULL) { + DetectPortInsert(newhead,joingr); + } + + return 0; +error: + return -1; +} + /* fill the global src group head, with the sigs included */ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { Signature *tmp_s = NULL; @@ -699,14 +1030,14 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { "building source address list...\n"); } - int i; - for (i = 0; i < 256; i++) { - de_ctx->src_gh[i] = DetectAddressGroupsHeadInit(); - if (de_ctx->src_gh[i] == NULL) { + int proto; + for (proto = 0; proto < 256; proto++) { + de_ctx->src_gh[proto] = DetectAddressGroupsHeadInit(); + if (de_ctx->src_gh[proto] == NULL) { goto error; } - de_ctx->tmp_gh[i] = DetectAddressGroupsHeadInit(); - if (de_ctx->tmp_gh[i] == NULL) { + de_ctx->tmp_gh[proto] = DetectAddressGroupsHeadInit(); + if (de_ctx->tmp_gh[proto] == NULL) { goto error; } } @@ -720,7 +1051,7 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { goto error; } - /* now for every rule add the source group to our temp list */ + /* now for every rule add the source group to our temp lists */ for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { BuildSourceAddressList(de_ctx,tmp_s,AF_INET); BuildSourceAddressList(de_ctx,tmp_s,AF_INET6); @@ -728,53 +1059,20 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) { sigs++; } - //DetectAddressGroupPrintMemory(); - //DetectSigGroupPrintMemory(); + /* create the final src addr list based on the tmplist. */ + for (proto = 0; proto < 256; proto++) { + CreateGroupedAddrList(de_ctx->tmp_gh[proto]->ipv4_head, AF_INET, de_ctx->src_gh[proto], MAX_UNIQ_GROUPS); + CreateGroupedAddrList(de_ctx->tmp_gh[proto]->ipv6_head, AF_INET6, de_ctx->src_gh[proto], MAX_UNIQ_GROUPS); + CreateGroupedAddrList(de_ctx->tmp_gh[proto]->any_head, AF_UNSPEC, de_ctx->src_gh[proto], MAX_UNIQ_GROUPS); - //printf("g_tmp_gh strt\n"); - //DetectAddressGroupPrintList(de_ctx->tmp_gh[6]->ipv4_head); - //printf("g_tmp_gh end\n"); + //DetectAddressGroupsHeadFree(de_ctx->tmp_gh[proto]); + free(de_ctx->tmp_gh[proto]); - for (i = 0; i < 256; i++) { - for (gr = de_ctx->tmp_gh[i]->ipv4_head; gr != NULL; ) { - //printf("Inserting2'ing (proto %3d): ",i); DetectAddressDataPrint(gr->ad); printf("\n"); - DetectAddressGroup *grnext = gr->next; - - gr->next = NULL; - if (DetectAddressGroupInsert(de_ctx->src_gh[i],gr) < 0) - goto error; - - gr = grnext; - } - for (gr = de_ctx->tmp_gh[i]->ipv6_head; gr != NULL; ) { - //printf("Inserting2'ing (proto %3d): ",i); DetectAddressDataPrint(gr->ad); printf("\n"); - DetectAddressGroup *grnext = gr->next; - - gr->next = NULL; - if (DetectAddressGroupInsert(de_ctx->src_gh[i],gr) < 0) - goto error; - - gr = grnext; - } - for (gr = de_ctx->tmp_gh[i]->any_head; gr != NULL; ) { - //printf("Inserting2'ing (proto %3d): ",i); DetectAddressDataPrint(gr->ad); printf("\n"); - DetectAddressGroup *grnext = gr->next; - - gr->next = NULL; - if (DetectAddressGroupInsert(de_ctx->src_gh[i],gr) < 0) - goto error; - - gr = grnext; - } - - //DetectAddressGroupsHeadFree(de_ctx->tmp_gh[i]); - free(de_ctx->tmp_gh[i]); - - //printf("g_src_gh[%d] strt\n", i); - //if (i == 6)DetectAddressGroupPrintList(de_ctx->src_gh[i]->ipv4_head); - //DetectAddressGroupPrintList(de_ctx->src_gh[i]->ipv6_head); - //DetectAddressGroupPrintList(de_ctx->src_gh[i]->any_head); - //printf("g_src_gh[%d] end\n", i); + //printf("g_src_gh[%d] strt\n", proto); + //if (proto == 6)DetectAddressGroupPrintList(de_ctx->src_gh[proto]->ipv4_head); + //DetectAddressGroupPrintList(de_ctx->src_gh[proto]->ipv6_head); + //DetectAddressGroupPrintList(de_ctx->src_gh[proto]->any_head); + //printf("g_src_gh[%d] end\n", proto); } /* IP ONLY */ @@ -883,13 +1181,7 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr DetectAddressGroup *grhead = NULL, *grdsthead = NULL, *grsighead = NULL; /* based on the family, select the list we are using in the head */ - if (family == AF_INET) { - grhead = head->ipv4_head; - } else if (family == AF_INET6) { - grhead = head->ipv6_head; - } else { - grhead = head->any_head; - } + grhead = GetHeadPtr(head,family); /* loop through the global source address list */ for (gr = grhead; gr != NULL; gr = gr->next) { @@ -917,15 +1209,8 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr max_idx = sig; - if (family == AF_INET) { - grsighead = tmp_s->dst.ipv4_head; - } else if (family == AF_INET6) { - grsighead = tmp_s->dst.ipv6_head; - } else { - grsighead = tmp_s->dst.any_head; - } - /* build the temp list */ + grsighead = GetHeadPtr(&tmp_s->dst, family); for (sgr = grsighead; sgr != NULL; sgr = sgr->next) { if ((lookup_gr = DetectAddressGroupLookup(tmp_gr_list,sgr->ad)) == NULL) { DetectAddressGroup *grtmp = DetectAddressGroupInit(); @@ -941,48 +1226,25 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr DetectAddressGroupAdd(&tmp_gr_list,grtmp); SigGroupHeadAppendSig(&grtmp->sh,tmp_s); + grtmp->cnt = 1; } else { /* our group will only have one sig, this one. So add that. */ SigGroupHeadAppendSig(&lookup_gr->sh,tmp_s); + lookup_gr->cnt++; } } } - /* for each address in the tmp list, insert a copy */ - for (sgr = tmp_gr_list; sgr != NULL; sgr = sgr->next) { - DetectAddressGroup *grtmp = DetectAddressGroupInit(); - if (grtmp == NULL) { - goto error; - } - DetectAddressData *adtmp = DetectAddressDataCopy(sgr->ad); - if (adtmp == NULL) { - goto error; - } - grtmp->ad = adtmp; - - SigGroupHeadCopySigs(sgr->sh,&grtmp->sh); - - int r = DetectAddressGroupInsert(gr->dst_gh,grtmp); - if (r < 0) { - printf("DetectAddressGroupInsert failed\n"); - goto error; - } - } - - /* set the right dst ptr to work with */ - if (family == AF_INET) { - grdsthead = gr->dst_gh->ipv4_head; - } else if (family == AF_INET6) { - grdsthead = gr->dst_gh->ipv6_head; - } else { - grdsthead = gr->dst_gh->any_head; - } + /* Create the destination address list, keeping in + * mind the limits we use. */ + CreateGroupedAddrList(tmp_gr_list,family,gr->dst_gh,MAX_UNIQ_GROUPS); /* see if the sig group head of each address group is the * same as an earlier one. If it is, free our head and use * a pointer to the earlier one. This saves _a lot_ of memory. */ + grdsthead = GetHeadPtr(gr->dst_gh, family); for (sgr = grdsthead; sgr != NULL; sgr = sgr->next) { //printf(" * Destination group: "); DetectAddressDataPrint(sgr->ad); printf("\n"); @@ -1040,6 +1302,18 @@ static int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, DetectAddressGr printf("PatternMatchPrepareGroup failed\n"); goto error; } + if (sgr->sh->mpm_ctx != NULL) { + if (de_ctx->mpm_max_patcnt < sgr->sh->mpm_ctx->pattern_cnt) + de_ctx->mpm_max_patcnt = sgr->sh->mpm_ctx->pattern_cnt; + + de_ctx->mpm_tot_patcnt += sgr->sh->mpm_ctx->pattern_cnt; + } + if (sgr->sh->mpm_uri_ctx != NULL) { + if (de_ctx->mpm_uri_max_patcnt < sgr->sh->mpm_uri_ctx->pattern_cnt) + de_ctx->mpm_uri_max_patcnt = sgr->sh->mpm_uri_ctx->pattern_cnt; + + de_ctx->mpm_uri_tot_patcnt += sgr->sh->mpm_uri_ctx->pattern_cnt; + } /* dbg */ if (!(sgr->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && sgr->sh->mpm_ctx) { mpm_memory_size += sgr->sh->mpm_ctx->memory_size; @@ -1071,22 +1345,15 @@ error: return -1; } -static int BuildDestinationAddressHeadsWithPorts(DetectEngineCtx *de_ctx, DetectAddressGroupsHead *head, int family) { +static int BuildDestinationAddressHeadsIPOnly(DetectEngineCtx *de_ctx, DetectAddressGroupsHead *head, int family) { Signature *tmp_s = NULL; DetectAddressGroup *gr = NULL, *sgr = NULL, *lookup_gr = NULL; - DetectAddressGroup *grhead = NULL, *grdsthead = NULL, *grsighead = NULL; u_int32_t max_idx = 0; - //printf("BuildDestinationAddressHeadsWithPorts: start\n"); + DetectAddressGroup *grhead = NULL, *grdsthead = NULL, *grsighead = NULL; /* based on the family, select the list we are using in the head */ - if (family == AF_INET) { - grhead = head->ipv4_head; - } else if (family == AF_INET6) { - grhead = head->ipv6_head; - } else { - grhead = head->any_head; - } + grhead = GetHeadPtr(head,family); /* loop through the global source address list */ for (gr = grhead; gr != NULL; gr = gr->next) { @@ -1114,15 +1381,8 @@ static int BuildDestinationAddressHeadsWithPorts(DetectEngineCtx *de_ctx, Detect max_idx = sig; - if (family == AF_INET) { - grsighead = tmp_s->dst.ipv4_head; - } else if (family == AF_INET6) { - grsighead = tmp_s->dst.ipv6_head; - } else { - grsighead = tmp_s->dst.any_head; - } - /* build the temp list */ + grsighead = GetHeadPtr(&tmp_s->dst, family); for (sgr = grsighead; sgr != NULL; sgr = sgr->next) { if ((lookup_gr = DetectAddressGroupLookup(tmp_gr_list,sgr->ad)) == NULL) { DetectAddressGroup *grtmp = DetectAddressGroupInit(); @@ -1138,205 +1398,371 @@ static int BuildDestinationAddressHeadsWithPorts(DetectEngineCtx *de_ctx, Detect DetectAddressGroupAdd(&tmp_gr_list,grtmp); SigGroupHeadAppendSig(&grtmp->sh,tmp_s); + grtmp->cnt = 1; } else { /* our group will only have one sig, this one. So add that. */ SigGroupHeadAppendSig(&lookup_gr->sh,tmp_s); + lookup_gr->cnt++; } + } + + } + + /* Create the destination address list, keeping in + * mind the limits we use. */ + CreateGroupedAddrList(tmp_gr_list,family,gr->dst_gh,0); + /* see if the sig group head of each address group is the + * same as an earlier one. If it is, free our head and use + * a pointer to the earlier one. This saves _a lot_ of memory. + */ + grdsthead = GetHeadPtr(gr->dst_gh, family); + for (sgr = grdsthead; sgr != NULL; sgr = sgr->next) { + //printf(" * Destination group: "); DetectAddressDataPrint(sgr->ad); printf("\n"); + + /* Because a pattern matcher context uses quite some + * memory, we first check if we can reuse it from + * another group head. */ + SigGroupHead *sgh = SigGroupHeadHashLookup(sgr->sh); + if (sgh == NULL) { + /* put the contents in our sig group head */ + SigGroupHeadSetSigCnt(sgr->sh, max_idx); + SigGroupHeadBuildMatchArray(de_ctx,sgr->sh, max_idx); + + SigGroupHeadHashAdd(sgr->sh); + de_ctx->gh_unique++; + } else { SigGroupHeadFree(sgr->sh); - sgr->sh = NULL; + sgr->sh = sgh; + + de_ctx->gh_reuse++; + sgr->flags |= ADDRESS_GROUP_SIGGROUPHEAD_COPY; } } - /* for each address in the tmp list, insert a copy */ - for (sgr = tmp_gr_list; sgr != NULL; sgr = sgr->next) { - DetectAddressGroup *grtmp = DetectAddressGroupInit(); - if (grtmp == NULL) { - goto error; - } - DetectAddressData *adtmp = DetectAddressDataCopy(sgr->ad); - if (adtmp == NULL) { - goto error; - } - grtmp->ad = adtmp; + /* free the temp list */ + DetectAddressGroupCleanupList(tmp_gr_list); + /* clear now unneeded sig group head */ + SigGroupHeadFree(gr->sh); + gr->sh = NULL; + } - /* copy the sigs */ - SigGroupHeadCopySigs(sgr->sh,&grtmp->sh); + return 0; +error: + return -1; +} - DetectPort *port = sgr->port; - for ( ; port != NULL; port = port->next) { - DetectPortInsertCopy(&grtmp->port, port); - } +static int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, DetectAddressGroupsHead *head, int family) { + Signature *tmp_s = NULL; + DetectAddressGroup *src_gr = NULL, *dst_gr = NULL, *sig_gr = NULL, *lookup_gr = NULL; + DetectAddressGroup *src_gr_head = NULL, *dst_gr_head = NULL, *sig_gr_head = NULL; + u_int32_t max_idx = 0; - int r = DetectAddressGroupInsert(gr->dst_gh,grtmp); - if (r < 0) { - printf("DetectAddressGroupInsert failed\n"); - goto error; - } + /* loop through the global source address list */ + src_gr_head = GetHeadPtr(head,family); + for (src_gr = src_gr_head; src_gr != NULL; src_gr = src_gr->next) { + //printf(" * Source group: "); DetectAddressDataPrint(src_gr->ad); printf("\n"); + + /* initialize the destination group head */ + src_gr->dst_gh = DetectAddressGroupsHeadInit(); + if (src_gr->dst_gh == NULL) { + goto error; } - /* set the right dst ptr to work with */ - if (family == AF_INET) { - grdsthead = gr->dst_gh->ipv4_head; - } else if (family == AF_INET6) { - grdsthead = gr->dst_gh->ipv6_head; - } else { - grdsthead = gr->dst_gh->any_head; + /* use a tmp list for speeding up insertions */ + DetectAddressGroup *tmp_gr_list = NULL; + + /* loop through all signatures in this source address group + * and build the temporary destination address list for it */ + u_int32_t sig; + for (sig = 0; sig < de_ctx->sig_array_len; sig++) { + if (!(src_gr->sh->sig_array[(sig/8)] & (1<<(sig%8)))) + continue; + + tmp_s = de_ctx->sig_array[sig]; + if (tmp_s == NULL) + continue; + + max_idx = sig; + + /* build the temp list */ + sig_gr_head = GetHeadPtr(&tmp_s->dst,family); + for (sig_gr = sig_gr_head; sig_gr != NULL; sig_gr = sig_gr->next) { + if ((lookup_gr = DetectAddressGroupLookup(tmp_gr_list, sig_gr->ad)) == NULL) { + DetectAddressGroup *grtmp = DetectAddressGroupInit(); + if (grtmp == NULL) { + goto error; + } + DetectAddressData *adtmp = DetectAddressDataCopy(sig_gr->ad); + if (adtmp == NULL) { + goto error; + } + grtmp->ad = adtmp; + SigGroupHeadAppendSig(&grtmp->sh,tmp_s); + grtmp->cnt = 1; + + DetectAddressGroupAdd(&tmp_gr_list,grtmp); + } else { + /* our group will only have one sig, this one. So add that. */ + SigGroupHeadAppendSig(&lookup_gr->sh,tmp_s); + lookup_gr->cnt++; + } + + SigGroupHeadFree(sig_gr->sh); + sig_gr->sh = NULL; + } } + /* Create the destination address list, keeping in + * mind the limits we use. */ + CreateGroupedAddrList(tmp_gr_list,family,src_gr->dst_gh,MAX_UNIQ_GROUPS); + /* add the ports to the dst address groups and the sigs * to the ports */ - for (sgr = grdsthead; sgr != NULL; sgr = sgr->next) { - if (sgr->sh == NULL) + dst_gr_head = GetHeadPtr(src_gr->dst_gh,family); + for (dst_gr = dst_gr_head; dst_gr != NULL; dst_gr = dst_gr->next) { + //printf(" * Destination group: "); DetectAddressDataPrint(dst_gr->ad); printf("\n"); + + if (dst_gr->sh == NULL) continue; /* we will reuse address sig group heads at this points, * because if the sigs are the same, the ports will be * the same. Saves memory and a lot of init time. */ - SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(sgr->sh); + SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(dst_gr->sh); if (lookup_sgh == NULL) { - DetectPortHashReset(); + DetectPortSpHashReset(); + u_int32_t sig2; for (sig2 = 0; sig2 < max_idx+1; sig2++) { - if (!(sgr->sh->sig_array[(sig2/8)] & (1<<(sig2%8)))) + if (!(dst_gr->sh->sig_array[(sig2/8)] & (1<<(sig2%8)))) continue; Signature *s = de_ctx->sig_array[sig2]; if (s == NULL) continue; - DetectPort *sdp = s->dp; + DetectPort *sdp = s->sp; for ( ; sdp != NULL; sdp = sdp->next) { - DetectPort *lookup_port = DetectPortHashLookup(sdp); + DetectPort *lookup_port = DetectPortSpHashLookup(sdp); if (lookup_port == NULL) { DetectPort *port = DetectPortCopySingle(sdp); if (port == NULL) goto error; SigGroupHeadAppendSig(&port->sh,s); - DetectPortHashAdd(port); + DetectPortSpHashAdd(port); + port->cnt = 1; } else { SigGroupHeadAppendSig(&lookup_port->sh,s); + lookup_port->cnt++; } } } - DetectPort *p = DetectPortHashGetListPtr(); - for (; p != NULL; ) { - DetectPort *next_p = p->next; - p->next = NULL; p->prev = NULL; - DetectPortInsert(&sgr->port,p); - p = next_p; + DetectPort *p = DetectPortSpHashGetListPtr(); + CreateGroupedPortList(p, &dst_gr->port, MAX_UNIQ_GROUPS); + if (p != NULL) { + DetectPort *next_p; + for (; p != NULL; ) { + next_p = p->next; + DetectPortFree(p); + p = next_p; + } } - SigGroupHeadHashAdd(sgr->sh); + SigGroupHeadHashAdd(dst_gr->sh); - sgr->sh->port = sgr->port; + dst_gr->sh->port = dst_gr->port; /* mark this head for deletion once we no longer need * the hash. We're only using the port ptr, so no problem * when we remove this after initialization is done */ - sgr->sh->flags |= SIG_GROUP_HEAD_FREE; + dst_gr->sh->flags |= SIG_GROUP_HEAD_FREE; /* for each destination port we setup the siggrouphead here */ - DetectPort *dp = sgr->port; - for ( ; dp != NULL; dp = dp->next) { - //printf(" * Port(range): "); DetectPortPrint(dp); printf(" "); - //printf("\n"); + DetectPort *sp = dst_gr->port; + for ( ; sp != NULL; sp = sp->next) { + //printf(" * Port(range): "); DetectPortPrint(sp); printf("\n"); - if (dp->sh == NULL) + if (sp->sh == NULL) continue; - /* Because a pattern matcher context uses quite some - * memory, we first check if we can reuse it from - * another group head. */ - SigGroupHead *sgh = SigGroupHeadPortHashLookup(dp->sh); - if (sgh == NULL) { - SigGroupHeadSetSigCnt(dp->sh, max_idx); - SigGroupHeadBuildMatchArray(de_ctx,dp->sh, max_idx); - - SigGroupHeadLoadContent(de_ctx, dp->sh); - if (dp->sh->content_size == 0) { - de_ctx->mpm_none++; - } else { - /* now have a look if we can reuse a mpm ctx */ - SigGroupHead *mpmsh = SigGroupHeadMpmHashLookup(dp->sh); - if (mpmsh == NULL) { - SigGroupHeadMpmHashAdd(dp->sh); - - de_ctx->mpm_unique++; - } else { - dp->sh->mpm_ctx = mpmsh->mpm_ctx; - dp->sh->flags |= SIG_GROUP_HEAD_MPM_COPY; - SigGroupHeadClearContent(dp->sh); + /* we will reuse address sig group heads at this points, + * because if the sigs are the same, the ports will be + * the same. Saves memory and a lot of init time. */ + SigGroupHead *lookup_sp_sgh = SigGroupHeadSPortHashLookup(sp->sh); + if (lookup_sp_sgh == NULL) { + DetectPortHashReset(); + u_int32_t sig2; + for (sig2 = 0; sig2 < max_idx+1; sig2++) { + if (!(sp->sh->sig_array[(sig2/8)] & (1<<(sig2%8)))) + continue; + + Signature *s = de_ctx->sig_array[sig2]; + if (s == NULL) + continue; + + DetectPort *sdp = s->dp; + for ( ; sdp != NULL; sdp = sdp->next) { + DetectPort *lookup_port = DetectPortHashLookup(sdp); + if (lookup_port == NULL) { + DetectPort *port = DetectPortCopySingle(sdp); + if (port == NULL) + goto error; + + SigGroupHeadAppendSig(&port->sh,s); + DetectPortHashAdd(port); + port->cnt = 1; + } else { + SigGroupHeadAppendSig(&lookup_port->sh,s); + lookup_port->cnt++; + } + } + } - de_ctx->mpm_reuse++; + DetectPort *p = DetectPortHashGetListPtr(); + CreateGroupedPortList(p,&sp->dst_ph,MAX_UNIQ_GROUPS); + if (p != NULL) { + DetectPort *next_p; + for (; p != NULL; ) { + next_p = p->next; + DetectPortFree(p); + p = next_p; } } - SigGroupHeadLoadUricontent(de_ctx, dp->sh); - if (dp->sh->uri_content_size == 0) { - de_ctx->mpm_uri_none++; - } else { - /* now have a look if we can reuse a uri mpm ctx */ - SigGroupHead *mpmsh = SigGroupHeadMpmUriHashLookup(dp->sh); - if (mpmsh == NULL) { - SigGroupHeadMpmUriHashAdd(dp->sh); + SigGroupHeadSPortHashAdd(sp->sh); + + sp->sh->port = sp->dst_ph; + /* mark this head for deletion once we no longer need + * the hash. We're only using the port ptr, so no problem + * when we remove this after initialization is done */ + sp->sh->flags |= SIG_GROUP_HEAD_FREE; + + /* for each destination port we setup the siggrouphead here */ + DetectPort *dp = sp->dst_ph; + for ( ; dp != NULL; dp = dp->next) { + //printf(" * Port(range): "); DetectPortPrint(dp); printf(" "); + //printf("\n"); + + if (dp->sh == NULL) + continue; + + /* Because a pattern matcher context uses quite some + * memory, we first check if we can reuse it from + * another group head. */ + SigGroupHead *lookup_dp_sgh = SigGroupHeadPortHashLookup(dp->sh); + if (lookup_dp_sgh == NULL) { + SigGroupHeadSetSigCnt(dp->sh, max_idx); + SigGroupHeadBuildMatchArray(de_ctx,dp->sh, max_idx); + + SigGroupHeadLoadContent(de_ctx, dp->sh); + if (dp->sh->content_size == 0) { + de_ctx->mpm_none++; + } else { + /* now have a look if we can reuse a mpm ctx */ + SigGroupHead *mpmsh = SigGroupHeadMpmHashLookup(dp->sh); + if (mpmsh == NULL) { + SigGroupHeadMpmHashAdd(dp->sh); + + de_ctx->mpm_unique++; + } else { + dp->sh->mpm_ctx = mpmsh->mpm_ctx; + dp->sh->flags |= SIG_GROUP_HEAD_MPM_COPY; + SigGroupHeadClearContent(dp->sh); + + de_ctx->mpm_reuse++; + } + } + + SigGroupHeadLoadUricontent(de_ctx, dp->sh); + if (dp->sh->uri_content_size == 0) { + de_ctx->mpm_uri_none++; + } else { + /* now have a look if we can reuse a uri mpm ctx */ + SigGroupHead *mpmsh = SigGroupHeadMpmUriHashLookup(dp->sh); + if (mpmsh == NULL) { + SigGroupHeadMpmUriHashAdd(dp->sh); + + de_ctx->mpm_uri_unique++; + } else { + dp->sh->mpm_uri_ctx = mpmsh->mpm_uri_ctx; + dp->sh->flags |= SIG_GROUP_HEAD_MPM_URI_COPY; + SigGroupHeadClearUricontent(dp->sh); + + de_ctx->mpm_uri_reuse++; + } + } + + /* init the pattern matcher, this will respect the copy + * setting */ + if (PatternMatchPrepareGroup(de_ctx, dp->sh) < 0) { + printf("PatternMatchPrepareGroup failed\n"); + goto error; + } + if (dp->sh->mpm_ctx != NULL) { + if (de_ctx->mpm_max_patcnt < dp->sh->mpm_ctx->pattern_cnt) + de_ctx->mpm_max_patcnt = dp->sh->mpm_ctx->pattern_cnt; + + de_ctx->mpm_tot_patcnt += dp->sh->mpm_ctx->pattern_cnt; + } + if (dp->sh->mpm_uri_ctx != NULL) { + if (de_ctx->mpm_uri_max_patcnt < dp->sh->mpm_uri_ctx->pattern_cnt) + de_ctx->mpm_uri_max_patcnt = dp->sh->mpm_uri_ctx->pattern_cnt; + + de_ctx->mpm_uri_tot_patcnt += dp->sh->mpm_uri_ctx->pattern_cnt; + } + /* dbg */ + if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_ctx) { + mpm_memory_size += dp->sh->mpm_ctx->memory_size; + } + if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY) && dp->sh->mpm_uri_ctx) { + mpm_memory_size += dp->sh->mpm_uri_ctx->memory_size; + } - de_ctx->mpm_uri_unique++; + SigGroupHeadPortHashAdd(dp->sh); + de_ctx->gh_unique++; } else { - dp->sh->mpm_uri_ctx = mpmsh->mpm_uri_ctx; - dp->sh->flags |= SIG_GROUP_HEAD_MPM_URI_COPY; - SigGroupHeadClearUricontent(dp->sh); + SigGroupHeadFree(dp->sh); - de_ctx->mpm_uri_reuse++; + dp->sh = lookup_dp_sgh; + dp->flags |= PORT_SIGGROUPHEAD_COPY; + + de_ctx->gh_reuse++; } } + /* sig group head found in hash, free it and use the hashed one */ + } else { + SigGroupHeadFree(sp->sh); - /* init the pattern matcher, this will respect the copy - * setting */ - if (PatternMatchPrepareGroup(de_ctx, dp->sh) < 0) { - printf("PatternMatchPrepareGroup failed\n"); - goto error; - } - /* dbg */ - if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_COPY) && dp->sh->mpm_ctx) { - mpm_memory_size += dp->sh->mpm_ctx->memory_size; - } - if (!(dp->sh->flags & SIG_GROUP_HEAD_MPM_URI_COPY) && dp->sh->mpm_uri_ctx) { - mpm_memory_size += dp->sh->mpm_uri_ctx->memory_size; - } + sp->sh = lookup_sp_sgh; + sp->flags |= PORT_SIGGROUPHEAD_COPY; - SigGroupHeadPortHashAdd(dp->sh); - de_ctx->gh_unique++; - } else { - SigGroupHeadFree(dp->sh); - dp->sh = sgh; + sp->dst_ph = lookup_sp_sgh->port; + sp->flags |= PORT_GROUP_PORTS_COPY; de_ctx->gh_reuse++; - dp->flags |= PORT_SIGGROUPHEAD_COPY; } } - /* sig group head found in hash, free it and use the hashed one */ } else { - SigGroupHeadFree(sgr->sh); - sgr->sh = lookup_sgh; - sgr->flags |= ADDRESS_GROUP_SIGGROUPHEAD_COPY; + SigGroupHeadFree(dst_gr->sh); - sgr->port = lookup_sgh->port; - sgr->flags |= ADDRESS_GROUP_PORTS_COPY; + dst_gr->sh = lookup_sgh; + dst_gr->flags |= ADDRESS_GROUP_SIGGROUPHEAD_COPY; + + dst_gr->port = lookup_sgh->port; + dst_gr->flags |= PORT_GROUP_PORTS_COPY; de_ctx->gh_reuse++; } } - /* free the temp list */ DetectAddressGroupCleanupList(tmp_gr_list); /* clear now unneeded sig group head */ - SigGroupHeadFree(gr->sh); - gr->sh = NULL; + SigGroupHeadFree(src_gr->sh); + src_gr->sh = NULL; } - //printf("BuildDestinationAddressHeadsWithPorts: end\n"); return 0; error: return -1; @@ -1355,48 +1781,47 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) { SigGroupHeadHashInit(); SigGroupHeadPortHashInit(); + SigGroupHeadSPortHashInit(); SigGroupHeadMpmHashInit(); SigGroupHeadMpmUriHashInit(); + DetectPortHashInit(); + DetectPortSpHashInit(); -//#if 0 - /* XXX ipv6 & any */ - r = BuildDestinationAddressHeadsWithPorts(de_ctx, de_ctx->src_gh[6],AF_INET); + r = BuildDestinationAddressHeadsWithBothPorts(de_ctx, de_ctx->src_gh[6],AF_INET); if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[6],AF_INET) failed\n"); goto error; } //#if 0 - r = BuildDestinationAddressHeadsWithPorts(de_ctx, de_ctx->src_gh[17],AF_INET); + r = BuildDestinationAddressHeadsWithBothPorts(de_ctx, de_ctx->src_gh[17],AF_INET); if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[17],AF_INET) failed\n"); goto error; } - r = BuildDestinationAddressHeadsWithPorts(de_ctx, de_ctx->src_gh[6],AF_INET6); + r = BuildDestinationAddressHeadsWithBothPorts(de_ctx, de_ctx->src_gh[6],AF_INET6); if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[6],AF_INET) failed\n"); goto error; } -//#if 0 - r = BuildDestinationAddressHeadsWithPorts(de_ctx, de_ctx->src_gh[17],AF_INET6); + r = BuildDestinationAddressHeadsWithBothPorts(de_ctx, de_ctx->src_gh[17],AF_INET6); if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[17],AF_INET) failed\n"); goto error; } - r = BuildDestinationAddressHeadsWithPorts(de_ctx, de_ctx->src_gh[6],AF_UNSPEC); + r = BuildDestinationAddressHeadsWithBothPorts(de_ctx, de_ctx->src_gh[6],AF_UNSPEC); if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[6],AF_INET) failed\n"); goto error; } -//#if 0 - r = BuildDestinationAddressHeadsWithPorts(de_ctx, de_ctx->src_gh[17],AF_UNSPEC); + r = BuildDestinationAddressHeadsWithBothPorts(de_ctx, de_ctx->src_gh[17],AF_UNSPEC); if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[17],AF_INET) failed\n"); goto error; } -//#endif + for (i = 0; i < 256; i++) { - if (i == 6 || i == 17) + if (i == IPPROTO_TCP || i == IPPROTO_UDP) continue; r = BuildDestinationAddressHeads(de_ctx, de_ctx->src_gh[i],AF_INET); @@ -1414,26 +1839,25 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) { printf ("BuildDestinationAddressHeads(src_gh[%d],AF_UNSPEC) failed\n", i); goto error; } - //printf("Protocol %d, gh: u %u r %u mpm: u %u r %u\n",i, de_ctx->gh_unique, de_ctx->gh_reuse, de_ctx->mpm_unique, de_ctx->mpm_reuse); } /* IP ONLY */ - r = BuildDestinationAddressHeads(de_ctx, de_ctx->io_src_gh,AF_INET); + r = BuildDestinationAddressHeadsIPOnly(de_ctx, de_ctx->io_src_gh,AF_INET); if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[%d],AF_INET) failed\n", i); goto error; } - r = BuildDestinationAddressHeads(de_ctx, de_ctx->io_src_gh,AF_INET6); + r = BuildDestinationAddressHeadsIPOnly(de_ctx, de_ctx->io_src_gh,AF_INET6); if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[%d],AF_INET6) failed\n", i); goto error; } - r = BuildDestinationAddressHeads(de_ctx, de_ctx->io_src_gh,AF_UNSPEC); /* for any */ + r = BuildDestinationAddressHeadsIPOnly(de_ctx, de_ctx->io_src_gh,AF_UNSPEC); /* for any */ if (r < 0) { printf ("BuildDestinationAddressHeads(src_gh[%d],AF_UNSPEC) failed\n", i); goto error; } - +//#endif /* cleanup group head (uri)content_array's */ SigGroupHeadFreeMpmArrays(); /* cleanup group head sig arrays */ @@ -1445,10 +1869,12 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) { /* cleanup the hashes */ SigGroupHeadHashFree(); SigGroupHeadPortHashFree(); + SigGroupHeadSPortHashFree(); SigGroupHeadMpmHashFree(); SigGroupHeadMpmUriHashFree(); DetectPortHashFree(); + DetectPortSpHashFree(); //DetectAddressGroupPrintMemory(); //DetectSigGroupPrintMemory(); @@ -1463,8 +1889,12 @@ int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) { printf("* Signature group heads: unique %u, copies %u.\n", de_ctx->gh_unique, de_ctx->gh_reuse); printf("* MPM instances: %u unique, copies %u (none %u).\n", de_ctx->mpm_unique, de_ctx->mpm_reuse, de_ctx->mpm_none); + //u_int32_t mpm_cnt = de_ctx->mpm_unique + de_ctx->mpm_reuse + de_ctx->mpm_none; printf("* MPM (URI) instances: %u unique, copies %u (none %u).\n", de_ctx->mpm_uri_unique, de_ctx->mpm_uri_reuse, de_ctx->mpm_uri_none); + //u_int32_t mpm_uri_cnt = de_ctx->mpm_uri_unique + de_ctx->mpm_uri_reuse + de_ctx->mpm_uri_none; + printf("* MPM max patcnt %u, avg %u\n", de_ctx->mpm_max_patcnt, de_ctx->mpm_tot_patcnt/de_ctx->mpm_unique); + printf("* MPM (URI) max patcnt %u, avg %u\n", de_ctx->mpm_uri_max_patcnt, de_ctx->mpm_uri_tot_patcnt/de_ctx->mpm_uri_unique); printf("* Building signature grouping structure, stage 3: building destination address lists... done\n"); } return 0; @@ -1482,10 +1912,9 @@ int SigAddressCleanupStage1(DetectEngineCtx *de_ctx) { int i; for (i = 0; i < 256; i++) { - //DetectAddressGroupsHeadCleanup(de_ctx->src_gh[i]); DetectAddressGroupsHeadFree(de_ctx->src_gh[i]); + de_ctx->src_gh[i] = NULL; } - //DetectAddressGroupsHeadCleanup(de_ctx->io_src_gh); DetectAddressGroupsHeadFree(de_ctx->io_src_gh); if (!(de_ctx->flags & DE_QUIET)) { @@ -1494,6 +1923,34 @@ int SigAddressCleanupStage1(DetectEngineCtx *de_ctx) { return 0; } +void DbgPrintSigs(SigGroupHead *sgh) { + if (sgh == NULL) { + printf("\n"); + return; + } + + u_int32_t sig; + for (sig = 0; sig < sgh->sig_cnt; sig++) { + printf("%u ", g_de_ctx->sig_array[sgh->match_array[sig]]->id); + } + printf("\n"); +} + +void DbgPrintSigs2(SigGroupHead *sgh) { + if (sgh == NULL) { + printf("\n"); + return; + } + + u_int32_t sig; + for (sig = 0; sig < SigGetMaxId(); sig++) { + if (sgh->sig_array[(sig/8)] & (1<<(sig%8))) { + printf("%u ", g_de_ctx->sig_array[sig]->id); + } + } + printf("\n"); +} + /* shortcut for debugging. If enabled Stage5 will * print sigid's for all groups */ //#define PRINTSIGS @@ -1502,15 +1959,21 @@ int SigAddressCleanupStage1(DetectEngineCtx *de_ctx) { int SigAddressPrepareStage5(void) { DetectAddressGroupsHead *global_dst_gh = NULL; DetectAddressGroup *global_src_gr = NULL, *global_dst_gr = NULL; + int i; printf("* Building signature grouping structure, stage 5: print...\n"); - int i; - for (i = 0; i < 256; i++) { - for (global_src_gr = g_de_ctx->src_gh[i]->ipv4_head; global_src_gr != NULL; + int proto; + for (proto = 0; proto < 256; proto++) { + if (proto != 6) + continue; + + for (global_src_gr = g_de_ctx->src_gh[proto]->ipv4_head; global_src_gr != NULL; global_src_gr = global_src_gr->next) { - printf("- "); DetectAddressDataPrint(global_src_gr->ad); printf(" (sh %p)\n", global_src_gr->sh); + printf("- "); DetectAddressDataPrint(global_src_gr->ad); + //printf(" (sh %p)\n", global_src_gr->sh); + printf("\n"); global_dst_gh = global_src_gr->dst_gh; if (global_dst_gh == NULL) @@ -1520,56 +1983,72 @@ int SigAddressPrepareStage5(void) { global_dst_gr != NULL; global_dst_gr = global_dst_gr->next) { - printf(" - [%4u] ", global_dst_gr->sh ? global_dst_gr->sh->sig_cnt : 0); DetectAddressDataPrint(global_dst_gr->ad); printf(" (sh %p)\n", global_dst_gr->sh); + printf(" - "); DetectAddressDataPrint(global_dst_gr->ad); + //printf(" (sh %p) ", global_dst_gr->sh); if (global_dst_gr->sh) { - if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { - printf("(COPY)\n"); - } else { - printf("\n"); - } + if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { + printf("(COPY)\n"); + } else { + printf("\n"); + } } + DetectPort *sp = global_dst_gr->port; + for ( ; sp != NULL; sp = sp->next) { + printf(" * Src port(range): "); DetectPortPrint(sp); + //printf(" (sh %p)", sp->sh); + printf("\n"); + DetectPort *dp = sp->dst_ph; + for ( ; dp != NULL; dp = dp->next) { + printf(" * Dst port(range): "); DetectPortPrint(dp); + //printf(" (sh %p)", dp->sh); #ifdef PRINTSIGS - if (global_dst_gr->sh && global_dst_gr->sh->head) { - printf (" - "); - SigGroupContainer *sg; - for (sg = global_dst_gr->sh->head; sg != NULL; sg = sg->next) { - printf("%u", sg->s->id); - if (sg->next) printf(","); - else printf("\n"); + printf(" - "); + for (i = 0; i < dp->sh->sig_cnt; i++) { + Signature *s = g_de_ctx->sig_array[dp->sh->match_array[i]]; + printf("%u ", s->id); + } +#endif + printf("\n"); } } -#endif } for (global_dst_gr = global_dst_gh->any_head; global_dst_gr != NULL; global_dst_gr = global_dst_gr->next) { - printf(" - [%4u] ", global_dst_gr->sh ? global_dst_gr->sh->sig_cnt : 0); DetectAddressDataPrint(global_dst_gr->ad); printf(" (sh %p)\n", global_dst_gr->sh); + printf(" - "); DetectAddressDataPrint(global_dst_gr->ad); + //printf(" (sh %p) ", global_dst_gr->sh); if (global_dst_gr->sh) { - if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { - printf("(COPY)\n"); - } else { - printf("\n"); - } + if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { + printf("(COPY)\n"); + } else { + printf("\n"); + } } + DetectPort *sp = global_dst_gr->port; + for ( ; sp != NULL; sp = sp->next) { + printf(" * Src port(range): "); DetectPortPrint(sp); printf("\n"); + DetectPort *dp = sp->dst_ph; + for ( ; dp != NULL; dp = dp->next) { + printf(" * Dst port(range): "); DetectPortPrint(dp); #ifdef PRINTSIGS - if (global_dst_gr->sh && global_dst_gr->sh->head) { - printf (" - "); - SigGroupContainer *sg; - for (sg = global_dst_gr->sh->head; sg != NULL; sg = sg->next) { - printf("%u", sg->s->id); - if (sg->next) printf(","); - else printf("\n"); + printf(" - "); + for (i = 0; i < dp->sh->sig_cnt; i++) { + Signature *s = g_de_ctx->sig_array[dp->sh->match_array[i]]; + printf("%u ", s->id); + } +#endif + printf("\n"); } } -#endif } } - for (global_src_gr = g_de_ctx->src_gh[i]->ipv6_head; global_src_gr != NULL; + for (global_src_gr = g_de_ctx->src_gh[proto]->ipv6_head; global_src_gr != NULL; global_src_gr = global_src_gr->next) { - printf("- "); DetectAddressDataPrint(global_src_gr->ad); printf(" (sh %p)\n", global_src_gr->sh); + printf("- "); DetectAddressDataPrint(global_src_gr->ad); + //printf(" (sh %p)\n", global_src_gr->sh); global_dst_gh = global_src_gr->dst_gh; if (global_dst_gh == NULL) @@ -1579,56 +2058,69 @@ int SigAddressPrepareStage5(void) { global_dst_gr != NULL; global_dst_gr = global_dst_gr->next) { - printf(" - [%4u] ", global_dst_gr->sh ? global_dst_gr->sh->sig_cnt : 0); DetectAddressDataPrint(global_dst_gr->ad); printf(" (sh %p)\n", global_dst_gr->sh); + printf(" - "); DetectAddressDataPrint(global_dst_gr->ad); + //printf(" (sh %p) ", global_dst_gr->sh); if (global_dst_gr->sh) { - if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { - printf("(COPY)\n"); - } else { - printf("\n"); - } + if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { + printf("(COPY)\n"); + } else { + printf("\n"); + } } + DetectPort *sp = global_dst_gr->port; + for ( ; sp != NULL; sp = sp->next) { + printf(" * Src port(range): "); DetectPortPrint(sp); printf("\n"); + DetectPort *dp = sp->dst_ph; + for ( ; dp != NULL; dp = dp->next) { + printf(" * Dst port(range): "); DetectPortPrint(dp); #ifdef PRINTSIGS - if (global_dst_gr->sh && global_dst_gr->sh->head) { - printf (" - "); - SigGroupContainer *sg; - for (sg = global_dst_gr->sh->head; sg != NULL; sg = sg->next) { - printf("%u", sg->s->id); - if (sg->next) printf(","); - else printf("\n"); + printf(" - "); + for (i = 0; i < dp->sh->sig_cnt; i++) { + Signature *s = g_de_ctx->sig_array[dp->sh->match_array[i]]; + printf("%u ", s->id); + } +#endif + printf("\n"); } } -#endif } for (global_dst_gr = global_dst_gh->any_head; global_dst_gr != NULL; global_dst_gr = global_dst_gr->next) { - printf(" - [%4u] ", global_dst_gr->sh ? global_dst_gr->sh->sig_cnt : 0); DetectAddressDataPrint(global_dst_gr->ad); printf(" (sh %p)\n", global_dst_gr->sh); + printf(" - "); DetectAddressDataPrint(global_dst_gr->ad); + //printf(" (sh %p) ", global_dst_gr->sh); if (global_dst_gr->sh) { - if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { - printf("(COPY)\n"); - } else { - printf("\n"); - } + if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { + printf("(COPY)\n"); + } else { + printf("\n"); + } } + DetectPort *sp = global_dst_gr->port; + for ( ; sp != NULL; sp = sp->next) { + printf(" * Src port(range): "); DetectPortPrint(sp); printf("\n"); + DetectPort *dp = sp->dst_ph; + for ( ; dp != NULL; dp = dp->next) { + printf(" * Dst port(range): "); DetectPortPrint(dp); #ifdef PRINTSIGS - if (global_dst_gr->sh && global_dst_gr->sh->head) { - printf (" - "); - SigGroupContainer *sg; - for (sg = global_dst_gr->sh->head; sg != NULL; sg = sg->next) { - printf("%u", sg->s->id); - if (sg->next) printf(","); - else printf("\n"); + printf(" - "); + for (i = 0; i < dp->sh->sig_cnt; i++) { + Signature *s = g_de_ctx->sig_array[dp->sh->match_array[i]]; + printf("%u ", s->id); + } +#endif + printf("\n"); } } -#endif } } - for (global_src_gr = g_de_ctx->src_gh[i]->any_head; global_src_gr != NULL; + for (global_src_gr = g_de_ctx->src_gh[proto]->any_head; global_src_gr != NULL; global_src_gr = global_src_gr->next) { - printf("- "); DetectAddressDataPrint(global_src_gr->ad); printf(" (sh %p)\n", global_src_gr->sh); + printf("- "); DetectAddressDataPrint(global_src_gr->ad); + //printf(" (sh %p)\n", global_src_gr->sh); global_dst_gh = global_src_gr->dst_gh; if (global_dst_gh == NULL) @@ -1638,73 +2130,91 @@ int SigAddressPrepareStage5(void) { global_dst_gr != NULL; global_dst_gr = global_dst_gr->next) { - printf(" - [%4u] ", global_dst_gr->sh ? global_dst_gr->sh->sig_cnt : 0); DetectAddressDataPrint(global_dst_gr->ad); printf(" (sh %p)\n", global_dst_gr->sh); + printf(" - "); DetectAddressDataPrint(global_dst_gr->ad); + //printf(" (sh %p) ", global_dst_gr->sh); if (global_dst_gr->sh) { - if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { - printf("(COPY)\n"); - } else { - printf("\n"); - } + if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { + printf("(COPY)\n"); + } else { + printf("\n"); + } } + DetectPort *sp = global_dst_gr->port; + for ( ; sp != NULL; sp = sp->next) { + printf(" * Src port(range): "); DetectPortPrint(sp); printf("\n"); + DetectPort *dp = sp->dst_ph; + for ( ; dp != NULL; dp = dp->next) { + printf(" * Dst port(range): "); DetectPortPrint(dp); #ifdef PRINTSIGS - if (global_dst_gr->sh && global_dst_gr->sh->head) { - printf (" - "); - SigGroupContainer *sg; - for (sg = global_dst_gr->sh->head; sg != NULL; sg = sg->next) { - printf("%u", sg->s->id); - if (sg->next) printf(","); - else printf("\n"); + printf(" - "); + for (i = 0; i < dp->sh->sig_cnt; i++) { + Signature *s = g_de_ctx->sig_array[dp->sh->match_array[i]]; + printf("%u ", s->id); + } +#endif + printf("\n"); } } -#endif } for (global_dst_gr = global_dst_gh->ipv4_head; global_dst_gr != NULL; global_dst_gr = global_dst_gr->next) { - printf(" - [%4u] ", global_dst_gr->sh ? global_dst_gr->sh->sig_cnt : 0); DetectAddressDataPrint(global_dst_gr->ad); printf(" (sh %p)\n", global_dst_gr->sh); + printf(" - "); DetectAddressDataPrint(global_dst_gr->ad); + //printf(" (sh %p) ", global_dst_gr->sh); if (global_dst_gr->sh) { - if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { - printf("(COPY)\n"); - } else { - printf("\n"); - } + if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { + printf("(COPY)\n"); + } else { + printf("\n"); + } } + DetectPort *sp = global_dst_gr->port; + for ( ; sp != NULL; sp = sp->next) { + printf(" * Src port(range): "); DetectPortPrint(sp); printf("\n"); + DetectPort *dp = sp->dst_ph; + for ( ; dp != NULL; dp = dp->next) { + printf(" * Dst port(range): "); DetectPortPrint(dp); #ifdef PRINTSIGS - if (global_dst_gr->sh && global_dst_gr->sh->head) { - printf (" - "); - SigGroupContainer *sg; - for (sg = global_dst_gr->sh->head; sg != NULL; sg = sg->next) { - printf("%u", sg->s->id); - if (sg->next) printf(","); - else printf("\n"); + printf(" - "); + for (i = 0; i < dp->sh->sig_cnt; i++) { + Signature *s = g_de_ctx->sig_array[dp->sh->match_array[i]]; + printf("%u ", s->id); + } +#endif + printf("\n"); } } -#endif } for (global_dst_gr = global_dst_gh->ipv6_head; global_dst_gr != NULL; global_dst_gr = global_dst_gr->next) { - printf(" - [%4u] ", global_dst_gr->sh->sig_cnt); DetectAddressDataPrint(global_dst_gr->ad); printf(" (sh %p)\n", global_dst_gr->sh); + printf(" - "); DetectAddressDataPrint(global_dst_gr->ad); + //printf(" (sh %p) ", global_dst_gr->sh); if (global_dst_gr->sh) { - if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { - printf("(COPY)\n"); - } else { - printf("\n"); - } + if (global_dst_gr->sh->flags & ADDRESS_GROUP_SIGGROUPHEAD_COPY) { + printf("(COPY)\n"); + } else { + printf("\n"); + } } + DetectPort *sp = global_dst_gr->port; + for ( ; sp != NULL; sp = sp->next) { + printf(" * Src port(range): "); DetectPortPrint(sp); printf("\n"); + DetectPort *dp = sp->dst_ph; + for ( ; dp != NULL; dp = dp->next) { + printf(" * Dst port(range): "); DetectPortPrint(dp); #ifdef PRINTSIGS - if (global_dst_gr->sh && global_dst_gr->sh->head) { - printf (" - "); - SigGroupContainer *sg; - for (sg = global_dst_gr->sh->head; sg != NULL; sg = sg->next) { - printf("%u", sg->s->id); - if (sg->next) printf(","); - else printf("\n"); + printf(" - "); + for (i = 0; i < dp->sh->sig_cnt; i++) { + Signature *s = g_de_ctx->sig_array[dp->sh->match_array[i]]; + printf("%u ", s->id); + } +#endif + printf("\n"); } } -#endif } } } @@ -1716,7 +2226,11 @@ int SigGroupBuild (DetectEngineCtx *de_ctx) { SigAddressPrepareStage1(de_ctx); SigAddressPrepareStage2(de_ctx); SigAddressPrepareStage3(de_ctx); - //SigAddressPrepareStage5(); +// SigAddressPrepareStage5(); + +// DetectAddressGroupPrintMemory(); +// DetectSigGroupPrintMemory(); +// DetectPortPrintMemory(); return 0; } diff --git a/src/detect.h b/src/detect.h index 7b682a7ba8..a1474d76ae 100644 --- a/src/detect.h +++ b/src/detect.h @@ -9,10 +9,12 @@ #include "detect-uricontent.h" #define SIG_FLAG_RECURSIVE 0x01 -#define SIG_FLAG_SP_ANY 0x02 -#define SIG_FLAG_DP_ANY 0x04 -#define SIG_FLAG_NOALERT 0x08 -#define SIG_FLAG_IPONLY 0x10 /* ip only signature */ +#define SIG_FLAG_SRC_ANY 0x02 +#define SIG_FLAG_DST_ANY 0x04 +#define SIG_FLAG_SP_ANY 0x08 +#define SIG_FLAG_DP_ANY 0x10 +#define SIG_FLAG_NOALERT 0x20 +#define SIG_FLAG_IPONLY 0x40 /* ip only signature */ #define DE_QUIET 0x01 @@ -41,12 +43,13 @@ typedef struct _PatternMatcherThread { } PatternMatcherThread; typedef struct _Signature { + u_int8_t flags; + u_int32_t num; /* signature number */ u_int32_t id; u_int8_t rev; u_int8_t prio; char *msg; - u_int8_t flags; u_int8_t action; DetectAddressGroupsHead src, dst; @@ -99,8 +102,30 @@ typedef struct DetectEngineCtx_ { mpm_uri_unique, mpm_uri_reuse, mpm_uri_none; u_int32_t gh_unique, gh_reuse; + u_int32_t mpm_max_patcnt, + mpm_min_patcnt, + mpm_tot_patcnt, + mpm_uri_max_patcnt, + mpm_uri_min_patcnt, + mpm_uri_tot_patcnt; + } DetectEngineCtx; +typedef struct SignatureTuple_ { + DetectAddressGroup *src; + DetectAddressGroup *dst; + DetectPort *sp; + DetectPort *dp; + u_int8_t proto; + + struct _SigGroupHead *sgh; + + struct SignatureTuple_ *hnext; + struct SignatureTuple_ *next; + + u_int32_t cnt; +} SignatureTuple; + /* container for content matches... we use this to compare * group heads for contents * XXX name */ diff --git a/src/util-mpm-wumanber.c b/src/util-mpm-wumanber.c index 013f4c8d90..42b5d5b79d 100644 --- a/src/util-mpm-wumanber.c +++ b/src/util-mpm-wumanber.c @@ -25,6 +25,8 @@ #define INIT_HASH_SIZE 65535 +#define HASH17_SIZE 131072 +#define HASH17(a,b) (((a)<<9) | (b)) #define HASH16_SIZE 65536 #define HASH16(a,b) (((a)<<8) | (b)) #define HASH15_SIZE 32768 @@ -49,6 +51,7 @@ u_int32_t WmSearch2Hash12(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_ u_int32_t WmSearch2Hash14(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf, u_int16_t buflen); u_int32_t WmSearch2Hash15(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf, u_int16_t buflen); u_int32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf, u_int16_t buflen); +u_int32_t WmSearch2Hash17(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf, u_int16_t buflen); void WmPrintInfo(MpmCtx *mpm_ctx); void WmPrintSearchStats(MpmThreadCtx *mpm_thread_ctx); void WmRegisterTests(void); @@ -139,12 +142,14 @@ void WmPrintInfo(MpmCtx *mpm_ctx) { if (mpm_ctx->Search == WmSearch1) { printf("WmSearch1 (allows single byte patterns)\n"); printf("MBSearch funct: "); - if (wm_ctx->MBSearch == WmSearch2Hash16) printf("WmSearch2Hash16\n"); + if (wm_ctx->MBSearch == WmSearch2Hash17) printf("WmSearch2Hash17\n"); + else if (wm_ctx->MBSearch == WmSearch2Hash16) printf("WmSearch2Hash16\n"); else if (wm_ctx->MBSearch == WmSearch2Hash15) printf("WmSearch2Hash15\n"); else if (wm_ctx->MBSearch == WmSearch2Hash14) printf("WmSearch2Hash14\n"); else if (wm_ctx->MBSearch == WmSearch2Hash12) printf("WmSearch2Hash12\n"); else if (wm_ctx->MBSearch == WmSearch2Hash9) printf("WmSearch2Hash9\n"); } + else if (mpm_ctx->Search == WmSearch2Hash17) printf("WmSearch2Hash17 (only for multibyte patterns)\n"); else if (mpm_ctx->Search == WmSearch2Hash16) printf("WmSearch2Hash16 (only for multibyte patterns)\n"); else if (mpm_ctx->Search == WmSearch2Hash15) printf("WmSearch2Hash15 (only for multibyte patterns)\n"); else if (mpm_ctx->Search == WmSearch2Hash14) printf("WmSearch2Hash14 (only for multibyte patterns)\n"); @@ -423,8 +428,10 @@ static void WmPrepareHash(MpmCtx *mpm_ctx) { idx = HASH14(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); else if (wm_ctx->hash_size == HASH15_SIZE) idx = HASH15(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); - else + else if (wm_ctx->hash_size == HASH16_SIZE) idx = HASH16(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); + else + idx = HASH17(wm_ctx->parray[i]->ci[patlen-1], wm_ctx->parray[i]->ci[patlen-2]); if (wm_ctx->hash[idx] == NULL) { WmHashItem *hi = WmAllocHashItem(mpm_ctx); @@ -499,8 +506,11 @@ static void WmPrepareShiftTable(MpmCtx *mpm_ctx) } else if (wm_ctx->hash_size == HASH15_SIZE) { idx = HASH15(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); //printf("HASH15 idx %u\n", idx); - } else { + } else if (wm_ctx->hash_size == HASH16_SIZE) { idx = HASH16(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); + //printf("HASH15 idx %u\n", idx); + } else { + idx = HASH17(wm_ctx->parray[i]->ci[k+1], wm_ctx->parray[i]->ci[k]); } //idx = ((wm_ctx->parray[i]->ci[k]) | (wm_ctx->parray[i]->ci[k+1]<<8)); if (shift < wm_ctx->shifttable[idx]) { @@ -543,20 +553,20 @@ int WmPreparePatterns(MpmCtx *mpm_ctx) { /* TODO VJ these values are chosen pretty much randomly, so * we should do some performance testing - * - * Right now I've chosen pattern cnt / 5 as max * */ if (wm_ctx->hash_size == 0) { - if (mpm_ctx->pattern_cnt < 100) { + if (mpm_ctx->pattern_cnt < 50) { wm_ctx->hash_size = HASH9_SIZE; - } else if(mpm_ctx->pattern_cnt < 800) { + } else if(mpm_ctx->pattern_cnt < 300) { wm_ctx->hash_size = HASH12_SIZE; - } else if(mpm_ctx->pattern_cnt < 3200) { + } else if(mpm_ctx->pattern_cnt < 1200) { wm_ctx->hash_size = HASH14_SIZE; - } else if(mpm_ctx->pattern_cnt < 6400) { + } else if(mpm_ctx->pattern_cnt < 2400) { wm_ctx->hash_size = HASH15_SIZE; - } else { + } else if(mpm_ctx->pattern_cnt < 4000) { wm_ctx->hash_size = HASH16_SIZE; + } else { + wm_ctx->hash_size = HASH17_SIZE; } } @@ -575,9 +585,12 @@ int WmPreparePatterns(MpmCtx *mpm_ctx) { } else if (wm_ctx->hash_size == HASH15_SIZE) { wm_ctx->MBSearch = WmSearch2Hash15; mpm_ctx->Search = WmSearch2Hash15; - } else { + } else if (wm_ctx->hash_size == HASH16_SIZE) { wm_ctx->MBSearch = WmSearch2Hash16; mpm_ctx->Search = WmSearch2Hash16; + } else { + wm_ctx->MBSearch = WmSearch2Hash17; + mpm_ctx->Search = WmSearch2Hash17; } if (mpm_ctx->minlen == 1) { @@ -1081,6 +1094,98 @@ u_int32_t WmSearch2Hash16(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_ return cnt; } +u_int32_t WmSearch2Hash17(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf, u_int16_t buflen) { + WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; + WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; + + u_int32_t cnt = 0; + u_int8_t *bufmin = buf; + u_int8_t *bufend = buf + buflen - 1; + u_int16_t sl = wm_ctx->shiftlen; + u_int16_t h; + u_int8_t shift; + WmHashItem *thi, *hi; + WmPattern *p; + u_int16_t prefixci_buf; + u_int16_t prefixcs_buf; + + if (buflen == 0) + return 0; + + //printf("BUF(%u) ", buflen); prt(buf,buflen); printf("\n"); + + buf+=(sl-1); + //buf++; + + while (buf <= bufend) { + //h = (wm_tolower(*buf)<<8)+(wm_tolower(*(buf-1))); + h = HASH17(wm_tolower(*buf),(wm_tolower(*(buf-1)))); + shift = wm_ctx->shifttable[h]; + //printf("search: h %u, shift %u\n", h, shift); + + if (shift == 0) { + wm_thread_ctx->stat_shift_null++; + /* get our hash item */ + hi = wm_ctx->hash[h]; + //printf("search: hi %p\n", hi); + if (hi != NULL) { + prefixci_buf = (u_int16_t)(wm_tolower(*(buf-sl+1)) + wm_tolower(*(buf-sl+2))); + prefixcs_buf = (u_int16_t)(*(buf-sl+1) + *(buf-sl+2)); + //printf("WmSearch2: prefixci_buf %u, prefixcs_buf %u\n", prefixci_buf, prefixcs_buf); + for (thi = hi; thi != NULL; thi = thi->nxt) { + p = wm_ctx->parray[thi->idx]; + + //printf("WmSearch2: p->prefix_ci %u, p->prefix_cs %u\n", + // p->prefix_ci, p->prefix_cs); + + if (p->flags & NOCASE) { + if (p->prefix_ci != prefixci_buf || p->len > (bufend-(buf-sl))) + continue; + + if (memcmp_lowercase(p->ci, buf-sl+1, p->len) == 0) { + cnt++; + //printf("CI Exact match: "); prt(p->ci, p->len); printf("\n"); + wm_thread_ctx->stat_loop_match++; + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + MpmMatchAppend(mpm_thread_ctx, em, &mpm_thread_ctx->match[em->id],(buf-sl+1 - bufmin)); + } + + } else { + wm_thread_ctx->stat_loop_no_match++; + } + } else { + if (p->prefix_cs != prefixcs_buf || p->len > (bufend-(buf-sl))) + continue; + if (memcmp(p->cs, buf-sl+1, p->len) == 0) { + cnt++; + //printf("CS Exact match: "); prt(p->cs, p->len); printf("\n"); + wm_thread_ctx->stat_loop_match++; + + MpmEndMatch *em; + for (em = p->em; em; em = em->next) { + //printf("em %p id %u\n", em, em->id); + MpmMatchAppend(mpm_thread_ctx, em, &mpm_thread_ctx->match[em->id],(buf-sl+1 - bufmin)); + } + + } else { + wm_thread_ctx->stat_loop_no_match++; + } + } + } + } + shift = 1; + } else { + wm_thread_ctx->stat_total_shift += shift; + wm_thread_ctx->stat_num_shift++; + } + buf += shift; + } + + return cnt; +} u_int32_t WmSearch1(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, u_int8_t *buf, u_int16_t buflen) { WmCtx *wm_ctx = (WmCtx *)mpm_ctx->ctx; //WmThreadCtx *wm_thread_ctx = (WmThreadCtx *)mpm_thread_ctx->ctx; diff --git a/src/vips.c b/src/vips.c index cffec3b259..9bbfe7949a 100644 --- a/src/vips.c +++ b/src/vips.c @@ -23,7 +23,7 @@ #include "util-binsearch.h" #include "detect-parse.h" -#include "detect-mpm.h" +#include "detect-engine-mpm.h" #include "tm-queuehandlers.h" #include "tm-queues.h"