|
|
|
|
/* Copyright (C) 2007-2013 Open Information Security Foundation
|
|
|
|
|
*
|
|
|
|
|
* You can copy, redistribute or modify this Program under the terms of
|
|
|
|
|
* the GNU General Public License version 2 as published by the Free
|
|
|
|
|
* Software Foundation.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
|
*
|
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
|
* version 2 along with this program; if not, write to the Free Software
|
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
|
|
|
|
* 02110-1301, USA.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* \file
|
|
|
|
|
*
|
|
|
|
|
* \author Victor Julien <victor@inliniac.net>
|
|
|
|
|
*
|
|
|
|
|
* Flow utility functions
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include "suricata-common.h"
|
|
|
|
|
#include "threads.h"
|
|
|
|
|
|
|
|
|
|
#include "flow.h"
|
|
|
|
|
#include "flow-private.h"
|
|
|
|
|
#include "flow-util.h"
|
|
|
|
|
#include "flow-callbacks.h"
|
|
|
|
|
#include "flow-var.h"
|
|
|
|
|
#include "app-layer.h"
|
|
|
|
|
|
|
|
|
|
#include "util-var.h"
|
|
|
|
|
#include "util-debug.h"
|
|
|
|
|
#include "util-macset.h"
|
|
|
|
|
#include "util-flow-rate.h"
|
|
|
|
|
#include "flow-storage.h"
|
|
|
|
|
|
|
|
|
|
#include "detect.h"
|
|
|
|
|
#include "detect-engine-state.h"
|
|
|
|
|
|
|
|
|
|
#include "decode-icmpv4.h"
|
|
|
|
|
|
|
|
|
|
#include "util-validate.h"
|
|
|
|
|
|
|
|
|
|
/** \brief allocate a flow
|
|
|
|
|
*
|
|
|
|
|
* We check against the memuse counter. If it passes that check we increment
|
|
|
|
|
* the counter first, then we try to alloc.
|
|
|
|
|
*
|
|
|
|
|
* \retval f the flow or NULL on out of memory
|
|
|
|
|
*/
|
|
|
|
|
Flow *FlowAlloc(void)
|
|
|
|
|
{
|
|
|
|
|
Flow *f;
|
|
|
|
|
size_t size = sizeof(Flow) + FlowStorageSize();
|
|
|
|
|
|
|
|
|
|
if (!(FLOW_CHECK_MEMCAP(size))) {
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
(void) SC_ATOMIC_ADD(flow_memuse, size);
|
|
|
|
|
|
|
|
|
|
f = SCCalloc(1, size);
|
|
|
|
|
if (unlikely(f == NULL)) {
|
|
|
|
|
(void)SC_ATOMIC_SUB(flow_memuse, size);
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* coverity[missing_lock] */
|
|
|
|
|
FLOW_INITIALIZE(f);
|
|
|
|
|
return f;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* \brief cleanup & free the memory of a flow
|
|
|
|
|
*
|
|
|
|
|
* \param f flow to clear & destroy
|
|
|
|
|
*/
|
|
|
|
|
void FlowFree(Flow *f)
|
|
|
|
|
{
|
|
|
|
|
FLOW_DESTROY(f);
|
|
|
|
|
SCFree(f);
|
|
|
|
|
|
|
|
|
|
size_t size = sizeof(Flow) + FlowStorageSize();
|
|
|
|
|
(void) SC_ATOMIC_SUB(flow_memuse, size);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* \brief Function to map the protocol to the defined FLOW_PROTO_* enumeration.
|
|
|
|
|
*
|
|
|
|
|
* \param proto protocol which is needed to be mapped
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
uint8_t FlowGetProtoMapping(uint8_t proto)
|
|
|
|
|
{
|
|
|
|
|
switch (proto) {
|
|
|
|
|
case IPPROTO_TCP:
|
|
|
|
|
return FLOW_PROTO_TCP;
|
|
|
|
|
case IPPROTO_UDP:
|
|
|
|
|
return FLOW_PROTO_UDP;
|
|
|
|
|
case IPPROTO_ICMP:
|
|
|
|
|
return FLOW_PROTO_ICMP;
|
|
|
|
|
default:
|
|
|
|
|
return FLOW_PROTO_DEFAULT;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
App layer API rewritten. The main files in question are:
app-layer.[ch], app-layer-detect-proto.[ch] and app-layer-parser.[ch].
Things addressed in this commit:
- Brings out a proper separation between protocol detection phase and the
parser phase.
- The dns app layer now is registered such that we don't use "dnstcp" and
"dnsudp" in the rules. A user who previously wrote a rule like this -
"alert dnstcp....." or
"alert dnsudp....."
would now have to use,
alert dns (ipproto:tcp;) or
alert udp (app-layer-protocol:dns;) or
alert ip (ipproto:udp; app-layer-protocol:dns;)
The same rules extend to other another such protocol, dcerpc.
- The app layer parser api now takes in the ipproto while registering
callbacks.
- The app inspection/detection engine also takes an ipproto.
- All app layer parser functions now take direction as STREAM_TOSERVER or
STREAM_TOCLIENT, as opposed to 0 or 1, which was taken by some of the
functions.
- FlowInitialize() and FlowRecycle() now resets proto to 0. This is
needed by unittests, which would try to clean the flow, and that would
call the api, AppLayerParserCleanupParserState(), which would try to
clean the app state, but the app layer now needs an ipproto to figure
out which api to internally call to clean the state, and if the ipproto
is 0, it would return without trying to clean the state.
- A lot of unittests are now updated where if they are using a flow and
they need to use the app layer, we would set a flow ipproto.
- The "app-layer" section in the yaml conf has also been updated as well.
13 years ago
|
|
|
uint8_t FlowGetReverseProtoMapping(uint8_t rproto)
|
|
|
|
|
{
|
|
|
|
|
switch (rproto) {
|
|
|
|
|
case FLOW_PROTO_TCP:
|
|
|
|
|
return IPPROTO_TCP;
|
|
|
|
|
case FLOW_PROTO_UDP:
|
|
|
|
|
return IPPROTO_UDP;
|
|
|
|
|
case FLOW_PROTO_ICMP:
|
|
|
|
|
return IPPROTO_ICMP;
|
|
|
|
|
default:
|
|
|
|
|
exit(EXIT_FAILURE);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void FlowSetICMPv4CounterPart(Flow *f)
|
|
|
|
|
{
|
|
|
|
|
int ctype = ICMPv4GetCounterpart(f->icmp_s.type);
|
|
|
|
|
if (ctype == -1)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
f->icmp_d.type = (uint8_t)ctype;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void FlowSetICMPv6CounterPart(Flow *f)
|
|
|
|
|
{
|
|
|
|
|
int ctype = ICMPv6GetCounterpart(f->icmp_s.type);
|
|
|
|
|
if (ctype == -1)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
f->icmp_d.type = (uint8_t)ctype;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* initialize the flow from the first packet
|
|
|
|
|
* we see from it. */
|
|
|
|
|
void FlowInit(ThreadVars *tv, Flow *f, const Packet *p)
|
|
|
|
|
{
|
|
|
|
|
SCEnter();
|
|
|
|
|
SCLogDebug("flow %p", f);
|
|
|
|
|
|
|
|
|
|
f->proto = p->proto;
|
|
|
|
|
f->recursion_level = p->recursion_level;
|
|
|
|
|
memcpy(&f->vlan_id[0], &p->vlan_id[0], sizeof(f->vlan_id));
|
|
|
|
|
f->vlan_idx = p->vlan_idx;
|
|
|
|
|
|
|
|
|
|
f->thread_id[0] = (FlowThreadId)tv->id;
|
|
|
|
|
|
|
|
|
|
f->livedev = p->livedev;
|
|
|
|
|
|
|
|
|
|
if (PacketIsIPv4(p)) {
|
|
|
|
|
const IPV4Hdr *ip4h = PacketGetIPv4(p);
|
|
|
|
|
FLOW_SET_IPV4_SRC_ADDR_FROM_PACKET(ip4h, &f->src);
|
|
|
|
|
FLOW_SET_IPV4_DST_ADDR_FROM_PACKET(ip4h, &f->dst);
|
|
|
|
|
f->min_ttl_toserver = f->max_ttl_toserver = IPV4_GET_RAW_IPTTL(ip4h);
|
|
|
|
|
f->flags |= FLOW_IPV4;
|
|
|
|
|
} else if (PacketIsIPv6(p)) {
|
|
|
|
|
const IPV6Hdr *ip6h = PacketGetIPv6(p);
|
|
|
|
|
FLOW_SET_IPV6_SRC_ADDR_FROM_PACKET(ip6h, &f->src);
|
|
|
|
|
FLOW_SET_IPV6_DST_ADDR_FROM_PACKET(ip6h, &f->dst);
|
|
|
|
|
f->min_ttl_toserver = f->max_ttl_toserver = IPV6_GET_RAW_HLIM(ip6h);
|
|
|
|
|
f->flags |= FLOW_IPV6;
|
|
|
|
|
} else {
|
|
|
|
|
SCLogDebug("neither IPv4 or IPv6, weird");
|
|
|
|
|
DEBUG_VALIDATE_BUG_ON(1);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (PacketIsTCP(p) || PacketIsUDP(p)) {
|
|
|
|
|
f->sp = p->sp;
|
|
|
|
|
f->dp = p->dp;
|
|
|
|
|
} else if (PacketIsICMPv4(p)) {
|
|
|
|
|
f->icmp_s.type = p->icmp_s.type;
|
|
|
|
|
f->icmp_s.code = p->icmp_s.code;
|
|
|
|
|
FlowSetICMPv4CounterPart(f);
|
|
|
|
|
} else if (PacketIsICMPv6(p)) {
|
|
|
|
|
f->icmp_s.type = p->icmp_s.type;
|
|
|
|
|
f->icmp_s.code = p->icmp_s.code;
|
|
|
|
|
FlowSetICMPv6CounterPart(f);
|
|
|
|
|
} else if (PacketIsSCTP(p)) {
|
|
|
|
|
f->sp = p->sp;
|
|
|
|
|
f->dp = p->dp;
|
|
|
|
|
} else if (PacketIsESP(p)) {
|
|
|
|
|
f->esp.spi = ESP_GET_SPI(PacketGetESP(p));
|
|
|
|
|
} else {
|
|
|
|
|
/* nothing to do for this IP proto. */
|
|
|
|
|
SCLogDebug("no special setup for IP proto %u", p->proto);
|
|
|
|
|
}
|
|
|
|
|
f->startts = p->ts;
|
|
|
|
|
|
|
|
|
|
f->protomap = FlowGetProtoMapping(f->proto);
|
flow: redesign of flow timeout handling
Goals:
- reduce locking
- take advantage of 'hot' caches
- better locality
Locking reduction
New flow spare pool. The global pool is implmented as a list of blocks,
where each block has a 100 spare flows. Worker threads fetch a block at
a time, storing the block in the local thread storage.
Flow Recycler now returns flows to the pool is blocks as well.
Flow Recycler fetches all flows to be processed in one step instead of
one at a time.
Cache 'hot'ness
Worker threads now check the timeout of flows they evaluate during lookup.
The worker will have to read the flow into cache anyway, so the added
overhead of checking the timeout value is minimal. When a flow is considered
timed out, one of 2 things happens:
- if the flow is 'owned' by the thread it is handled locally. Handling means
checking if the flow needs 'timeout' work.
- otherwise, the flow is added to a special 'evicted' list in the flow
bucket where it will be picked up by the flow manager.
Flow Manager timing
By default the flow manager now tries to do passes of the flow hash in
smaller steps, where the goal is to do full pass in 8 x the lowest timeout
value it has to enforce. So if the lowest timeout value is 30s, a full pass
will take 4 minutes. The goal here is to reduce locking overhead and not
get in the way of the workers.
In emergency mode each pass is full, and lower timeouts are used.
Timing of the flow manager is also no longer relying on pthread condition
variables, as these generally cause waking up much quicker than the desired
timout. Instead a simple (u)sleep loop is used.
Both changes reduce the number of hash passes a lot.
Emergency behavior
In emergency mode there a number of changes to the workers. In this scenario
the flow memcap is fully used up and it is unavoidable that some flows won't
be tracked.
1. flow spare pool fetches are reduced to once a second. This avoids locking
overhead, while the chance of success was very low.
2. getting an active flow directly from the hash skips flows that had very
recent activity to avoid the scenario where all flows get only into the
NEW state before getting reused. Rather allow some to have a chance of
completing.
3. TCP packets that are not SYN packets will not get a used flow, unless
stream.midstream is enabled. The goal here is again to avoid evicting
active flows unnecessarily.
Better Localily
Flow Manager injects flows into the worker threads now, instead of one or
two packets. Advantage of this is that the worker threads can get packets
from their local packet pools, avoiding constant overhead of packets returning
to 'foreign' pools.
Counters
A lot of flow counters have been added and some have been renamed.
Overall the worker threads increment 'flow.wrk.*' counters, while the flow
manager increments 'flow.mgr.*'.
Additionally, none of the counters are snapshots anymore, they all increment
over time. The flow.memuse and flow.spare counters are exceptions.
Misc
FlowQueue has been split into a FlowQueuePrivate (unlocked) and FlowQueue.
Flow no longer has 'prev' pointers and used a unified 'next' pointer for
both hash and queue use.
7 years ago
|
|
|
f->timeout_policy = FlowGetTimeoutPolicy(f);
|
|
|
|
|
|
|
|
|
|
if (MacSetFlowStorageEnabled()) {
|
|
|
|
|
DEBUG_VALIDATE_BUG_ON(FlowGetStorageById(f, MacSetGetFlowStorageID()) != NULL);
|
|
|
|
|
MacSet *ms = MacSetInit(10);
|
|
|
|
|
FlowSetStorageById(f, MacSetGetFlowStorageID(), ms);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (FlowRateStorageEnabled()) {
|
|
|
|
|
DEBUG_VALIDATE_BUG_ON(FlowGetStorageById(f, FlowRateGetStorageID()) != NULL);
|
|
|
|
|
FlowRateStore *frs = FlowRateStoreInit();
|
|
|
|
|
FlowSetStorageById(f, FlowRateGetStorageID(), frs);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
SCFlowRunInitCallbacks(tv, f, p);
|
|
|
|
|
|
|
|
|
|
SCReturn;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
FlowStorageId g_bypass_info_id = { .id = -1 };
|
|
|
|
|
|
|
|
|
|
FlowStorageId GetFlowBypassInfoID(void)
|
|
|
|
|
{
|
|
|
|
|
return g_bypass_info_id;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void FlowBypassFree(void *x)
|
|
|
|
|
{
|
|
|
|
|
FlowBypassInfo *fb = (FlowBypassInfo *) x;
|
|
|
|
|
|
|
|
|
|
if (fb == NULL)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
if (fb->bypass_data && fb->BypassFree) {
|
|
|
|
|
fb->BypassFree(fb->bypass_data);
|
|
|
|
|
}
|
|
|
|
|
SCFree(fb);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void RegisterFlowBypassInfo(void)
|
|
|
|
|
{
|
|
|
|
|
g_bypass_info_id = FlowStorageRegister("bypass_counters", sizeof(void *),
|
|
|
|
|
NULL, FlowBypassFree);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void FlowEndCountersRegister(ThreadVars *t, FlowEndCounters *fec)
|
|
|
|
|
{
|
|
|
|
|
for (int i = 0; i < FLOW_STATE_SIZE; i++) {
|
|
|
|
|
const char *name = NULL;
|
|
|
|
|
if (i == FLOW_STATE_NEW) {
|
|
|
|
|
name = "flow.end.state.new";
|
|
|
|
|
} else if (i == FLOW_STATE_ESTABLISHED) {
|
|
|
|
|
name = "flow.end.state.established";
|
|
|
|
|
} else if (i == FLOW_STATE_CLOSED) {
|
|
|
|
|
name = "flow.end.state.closed";
|
|
|
|
|
} else if (i == FLOW_STATE_LOCAL_BYPASSED) {
|
|
|
|
|
name = "flow.end.state.local_bypassed";
|
|
|
|
|
#ifdef CAPTURE_OFFLOAD
|
|
|
|
|
} else if (i == FLOW_STATE_CAPTURE_BYPASSED) {
|
|
|
|
|
name = "flow.end.state.capture_bypassed";
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
if (name) {
|
|
|
|
|
fec->flow_state[i] = StatsRegisterCounter(name, t);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
for (enum TcpState i = TCP_NONE; i <= TCP_CLOSED; i++) {
|
|
|
|
|
const char *name = NULL;
|
|
|
|
|
switch (i) {
|
|
|
|
|
case TCP_NONE:
|
|
|
|
|
name = "flow.end.tcp_state.none";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_SYN_SENT:
|
|
|
|
|
name = "flow.end.tcp_state.syn_sent";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_SYN_RECV:
|
|
|
|
|
name = "flow.end.tcp_state.syn_recv";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_ESTABLISHED:
|
|
|
|
|
name = "flow.end.tcp_state.established";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_FIN_WAIT1:
|
|
|
|
|
name = "flow.end.tcp_state.fin_wait1";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_FIN_WAIT2:
|
|
|
|
|
name = "flow.end.tcp_state.fin_wait2";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_TIME_WAIT:
|
|
|
|
|
name = "flow.end.tcp_state.time_wait";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_LAST_ACK:
|
|
|
|
|
name = "flow.end.tcp_state.last_ack";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_CLOSE_WAIT:
|
|
|
|
|
name = "flow.end.tcp_state.close_wait";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_CLOSING:
|
|
|
|
|
name = "flow.end.tcp_state.closing";
|
|
|
|
|
break;
|
|
|
|
|
case TCP_CLOSED:
|
|
|
|
|
name = "flow.end.tcp_state.closed";
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
fec->flow_tcp_state[i] = StatsRegisterCounter(name, t);
|
|
|
|
|
}
|
|
|
|
|
fec->flow_tcp_liberal = StatsRegisterCounter("flow.end.tcp_liberal", t);
|
|
|
|
|
}
|