aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

detect / stream: new 'raw' stream inspection

Browse Source 
Remove the 'StreamMsg' approach from the engine. In this approach the
stream engine would create a list of chunks for inspection by the
detection engine. There were several issues:

1. the messages had a fixed size, so blocks of data bigger than ~4k
   would be cut into multiple messages

2. it lead to lots of data copying and unnecessary memory use

3. the StreamMsgs used a central pool

The Stream engine switched over to the streaming buffer API, which
means that the reassembled data is always available. This made the
StreamMsg approach even clunkier.

The new approach exposes the streaming buffer data to the detection
engine. It has to pay attention to an important issue though: packet
loss. The data may have gaps. The streaming buffer API tracks the
blocks of continuous data.

To access the data for inspection a callback approach is used. The
'StreamReassembleRaw' function is called with a callback and data.
This way it runs the MPM and individual rule inspection code. At
the end of each detection run the stream engine is notified that it
can move forward it's 'progress'.
pull/2673/head
Victor Julien 10 years ago
parent 564c0bd2c1
commit 971ab18b95
23 changed files with 268 additions and 2226 deletions
Show Stats Download Patch File Download Diff File

105
src/app-layer-detect-proto.c
Unescape Escape View File

@ -3679,110 +3679,6 @@ static int AppLayerProtoDetectTest19(void)
return result;
}
/** \test test if the engine detect the proto and match with it
* and also against a content option */
static int AppLayerProtoDetectTest20(void)
{
int result = 0;
Flow *f = NULL;
uint8_t http_buf1[] = "POST /one HTTP/1.0\r\n"
"User-Agent: Mozilla/1.0\r\n"
"Cookie: hellocatch\r\n\r\n";
uint32_t http_buf1_len = sizeof(http_buf1) - 1;
TcpSession ssn;
Packet *p = NULL;
Signature *s = NULL;
ThreadVars tv;
DetectEngineThreadCtx *det_ctx = NULL;
DetectEngineCtx *de_ctx = NULL;
AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
memset(&tv, 0, sizeof(ThreadVars));
memset(&ssn, 0, sizeof(TcpSession));
p = UTHBuildPacket(http_buf1, http_buf1_len, IPPROTO_TCP);
f = UTHBuildFlow(AF_INET, "1.1.1.1", "2.2.2.2", 1024, 80);
if (f == NULL)
goto end;
f->protoctx = &ssn;
p->flow = f;
p->flowflags |= FLOW_PKT_TOSERVER;
p->flowflags |= FLOW_PKT_ESTABLISHED;
p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
f->alproto = ALPROTO_HTTP;
f->proto = IPPROTO_TCP;
p->flags |= PKT_STREAM_ADD;
p->flags |= PKT_STREAM_EOF;
de_ctx = DetectEngineCtxInit();
if (de_ctx == NULL) {
goto end;
}
StreamTcpInitConfig(TRUE);
StreamMsg *stream_msg = StreamMsgGetFromPool();
if (stream_msg == NULL) {
printf("no stream_msg: ");
goto end;
}
memcpy(stream_msg->data, http_buf1, http_buf1_len);
stream_msg->data_len = http_buf1_len;
ssn.toserver_smsg_head = stream_msg;
ssn.toserver_smsg_tail = stream_msg;
de_ctx->flags |= DE_QUIET;
s = de_ctx->sig_list = SigInit(de_ctx, "alert http any any -> any any "
"(msg:\"Test content option\"; "
"content:\"one\"; sid:1;)");
if (s == NULL) {
goto end;
}
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&tv, (void *)de_ctx, (void *)&det_ctx);
FLOWLOCK_WRLOCK(f);
int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP,
STREAM_TOSERVER, http_buf1, http_buf1_len);
if (r != 0) {
printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
FLOWLOCK_UNLOCK(f);
goto end;
}
FLOWLOCK_UNLOCK(f);
/* do detect */
SigMatchSignatures(&tv, de_ctx, det_ctx, p);
if (!PacketAlertCheck(p, 1)) {
printf("sig 1 didn't alert, but it should: ");
goto end;
}
result = 1;
end:
if (alp_tctx != NULL)
AppLayerParserThreadCtxFree(alp_tctx);
if (det_ctx != NULL)
DetectEngineThreadCtxDeinit(&tv, det_ctx);
if (de_ctx != NULL)
SigGroupCleanup(de_ctx);
if (de_ctx != NULL)
DetectEngineCtxFree(de_ctx);
StreamTcpFreeConfig(TRUE);
UTHFreePackets(&p, 1);
UTHFreeFlow(f);
return result;
}
void AppLayerProtoDetectUnittestsRegister(void)
{
SCEnter();
@ -3806,7 +3702,6 @@ void AppLayerProtoDetectUnittestsRegister(void)
UtRegisterTest("AppLayerProtoDetectTest17", AppLayerProtoDetectTest17);
UtRegisterTest("AppLayerProtoDetectTest18", AppLayerProtoDetectTest18);
UtRegisterTest("AppLayerProtoDetectTest19", AppLayerProtoDetectTest19);
UtRegisterTest("AppLayerProtoDetectTest20", AppLayerProtoDetectTest20);
SCReturn;
}

2
src/app-layer.c
Unescape Escape View File

@ -607,8 +607,6 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto,
flags, data, data_len);
PACKET_PROFILING_APP_END(app_tctx, f->alproto);
} else {
SCLogDebug(" smsg not start, but no l7 data? Weird");
}
}

2
src/decode.h
Unescape Escape View File

@ -1108,6 +1108,8 @@ int DecoderParseDataFromFileSerie(char *fileprefix, DecoderFunc Decoder);
#define PKT_REBUILT_FRAGMENT (1<<25) /**< Packet is rebuilt from
* fragments. */
#define PKT_DETECT_HAS_STREAMDATA (1<<26) /**< Set by Detect() if raw stream data is available. */
/** \brief return 1 if the packet is a pseudo packet */
#define PKT_IS_PSEUDOPKT(p) ((p)->flags & PKT_PSEUDO_STREAM_END)

1
src/detect-engine-alert.c
Unescape Escape View File

@ -184,7 +184,6 @@ int PacketAlertRemove(Packet *p, uint16_t pos)
* \param s the signature that matched
* \param p packet
* \param flags alert flags
* \param alert_msg ptr to StreamMsg object that the signature matched on
*/
int PacketAlertAppend(DetectEngineThreadCtx *det_ctx, const Signature *s,
Packet *p, uint64_t tx_id, uint8_t flags)

106
src/detect-engine-payload.c
Unescape Escape View File

@ -35,6 +35,7 @@
#include "detect-engine-prefilter.h"
#include "stream.h"
#include "stream-tcp.h"
#include "util-debug.h"
#include "util-print.h"
@ -45,27 +46,39 @@
#include "util-mpm-ac.h"
struct StreamMpmData {
DetectEngineThreadCtx *det_ctx;
const MpmCtx *mpm_ctx;
};
static int StreamMpmFunc(void *cb_data, const uint8_t *data, const uint32_t data_len)
{
struct StreamMpmData *smd = cb_data;
if (data_len >= smd->mpm_ctx->minlen) {
(void)mpm_table[smd->mpm_ctx->mpm_type].Search(smd->mpm_ctx,
&smd->det_ctx->mtcs, &smd->det_ctx->pmq,
data, data_len);
}
return 0;
}
static void PrefilterPktStream(DetectEngineThreadCtx *det_ctx,
Packet *p, const void *pectx)
{
SCEnter();
const MpmCtx *mpm_ctx = (MpmCtx *)pectx;
const StreamMsg *smsg = det_ctx->smsg;
/* for established packets inspect any smsg we may have queued up */
if (p->flowflags & FLOW_PKT_ESTABLISHED) {
SCLogDebug("p->flowflags & FLOW_PKT_ESTABLISHED");
for ( ; smsg != NULL; smsg = smsg->next) {
if (smsg->data_len >= mpm_ctx->minlen) {
(void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
&det_ctx->mtcs, &det_ctx->pmq,
smsg->data, smsg->data_len);
}
}
/* for established packets inspect any stream we may have queued up */
if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
struct StreamMpmData stream_mpm_data = { det_ctx, mpm_ctx };
StreamReassembleRaw(p->flow->protoctx, p,
StreamMpmFunc, &stream_mpm_data,
&det_ctx->raw_stream_progress);
SCLogDebug("det_ctx->raw_stream_progress %"PRIu64,
det_ctx->raw_stream_progress);
} else {
SCLogDebug("NOT p->flowflags & FLOW_PKT_ESTABLISHED");
SCLogDebug("NOT p->flags & PKT_DETECT_HAS_STREAMDATA");
}
/* packets that have not been added to the stream will be inspected
@ -152,48 +165,61 @@ int DetectEngineInspectPacketPayload(DetectEngineCtx *de_ctx,
SCReturnInt(0);
}
struct StreamContentInspectData {
DetectEngineCtx *de_ctx;
DetectEngineThreadCtx *det_ctx;
const Signature *s;
Flow *f;
};
static int StreamContentInspectFunc(void *cb_data, const uint8_t *data, const uint32_t data_len)
{
SCEnter();
int r = 0;
struct StreamContentInspectData *smd = cb_data;
smd->det_ctx->buffer_offset = 0;
smd->det_ctx->discontinue_matching = 0;
smd->det_ctx->inspection_recursion_counter = 0;
r = DetectEngineContentInspection(smd->de_ctx, smd->det_ctx,
smd->s, smd->s->sm_arrays[DETECT_SM_LIST_PMATCH],
smd->f, (uint8_t *)data, data_len, 0,
DETECT_ENGINE_CONTENT_INSPECTION_MODE_STREAM, NULL);
if (r == 1) {
SCReturnInt(1);
}
SCReturnInt(0);
}
/**
* \brief Do the content inspection & validation for a signature for a stream chunk
* \brief Do the content inspection & validation for a signature
* on the raw stream
*
* \param de_ctx Detection engine context
* \param det_ctx Detection engine thread context
* \param s Signature to inspect
* \param f flow (for pcre flowvar storage)
* \param payload ptr to the payload to inspect
* \param payload_len length of the payload
*
* \retval 0 no match
* \retval 1 match
*
* \todo we might also pass the packet to this function for the pktvar
* storage. Only, would that be right? We're not inspecting data
* from the current packet here.
*/
int DetectEngineInspectStreamPayload(DetectEngineCtx *de_ctx,
DetectEngineThreadCtx *det_ctx, const Signature *s, Flow *f,
uint8_t *payload, uint32_t payload_len)
DetectEngineThreadCtx *det_ctx, const Signature *s,
Flow *f, Packet *p)
{
SCEnter();
int r = 0;
if (s->sm_arrays[DETECT_SM_LIST_PMATCH] == NULL) {
SCReturnInt(0);
}
det_ctx->buffer_offset = 0;
det_ctx->discontinue_matching = 0;
det_ctx->inspection_recursion_counter = 0;
r = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_arrays[DETECT_SM_LIST_PMATCH],
f, payload, payload_len, 0,
DETECT_ENGINE_CONTENT_INSPECTION_MODE_STREAM, NULL);
if (r == 1) {
SCReturnInt(1);
}
SCReturnInt(0);
uint64_t unused;
struct StreamContentInspectData inspect_data = { de_ctx, det_ctx, s, f };
int r = StreamReassembleRaw(f->protoctx, p,
StreamContentInspectFunc, &inspect_data,
&unused);
return r;
}
#ifdef UNITTESTS
/** \test Not the first but the second occurence of "abc" should be used

2
src/detect-engine-payload.h
Unescape Escape View File

@ -31,7 +31,7 @@ int DetectEngineInspectPacketPayload(DetectEngineCtx *,
DetectEngineThreadCtx *, const Signature *, Flow *, Packet *);
int DetectEngineInspectStreamPayload(DetectEngineCtx *,
DetectEngineThreadCtx *, const Signature *, Flow *,
uint8_t *, uint32_t);
Packet *);
void PayloadRegisterTests(void);

7
src/detect-engine-prefilter.c
Unescape Escape View File

@ -141,7 +141,7 @@ static inline void PrefilterTx(DetectEngineThreadCtx *det_ctx,
}
void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh,
Packet *p, const uint8_t flags, int has_state)
Packet *p, const uint8_t flags, const bool has_state)
{
SCEnter();
@ -165,8 +165,9 @@ void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh,
/* run payload inspecting engines */
if (sgh->payload_engines &&
(p->payload_len > 0 || det_ctx->smsg != NULL) &&
!(p->flags & PKT_NOPAYLOAD_INSPECTION)) {
(p->payload_len || (p->flags & PKT_DETECT_HAS_STREAMDATA)) &&
!(p->flags & PKT_NOPAYLOAD_INSPECTION))
{
PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_PAYLOAD);
PrefilterEngine *engine = sgh->payload_engines;
while (1) {

2
src/detect-engine-prefilter.h
Unescape Escape View File

@ -25,7 +25,7 @@
#define __DETECT_ENGINE_PREFILTER_H__
void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p,
const uint8_t flags, int has_state);
const uint8_t flags, const bool has_state);
int PrefilterAppendEngine(SigGroupHead *sgh,
void (*Prefilter)(DetectEngineThreadCtx *det_ctx, Packet *p, const void *pectx),

100
src/detect-ssl-version.c
Unescape Escape View File

@ -549,104 +549,6 @@ static int DetectSslVersionTestDetect02(void)
PASS;
}
static int DetectSslVersionTestDetect03(void)
{
DetectEngineCtx *de_ctx = NULL;
Flow f;
uint8_t sslbuf1[] = { 0x16 };
uint32_t ssllen1 = sizeof(sslbuf1);
uint8_t sslbuf2[] = { 0x03 };
uint32_t ssllen2 = sizeof(sslbuf2);
uint8_t sslbuf3[] = { 0x01 };
uint32_t ssllen3 = sizeof(sslbuf3);
uint8_t sslbuf4[] = { 0x01, 0x00, 0x00, 0xad, 0x03, 0x02 };
uint32_t ssllen4 = sizeof(sslbuf4);
TcpSession ssn;
Packet *p = NULL;
Signature *s = NULL;
ThreadVars th_v;
DetectEngineThreadCtx *det_ctx = NULL;
AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
memset(&th_v, 0, sizeof(th_v));
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
p = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
p->tcph->th_seq = htonl(1000);
FLOW_INITIALIZE(&f);
f.protoctx = (void *)&ssn;
p->flow = &f;
p->flowflags |= FLOW_PKT_TOSERVER;
p->flowflags |= FLOW_PKT_ESTABLISHED;
p->flags |= PKT_HAS_FLOW | PKT_STREAM_EST;
f.alproto = ALPROTO_TLS;
f.proto = p->proto;
StreamTcpInitConfig(TRUE);
StreamMsg *stream_msg = StreamMsgGetFromPool();
FAIL_IF_NULL(stream_msg);
memcpy(stream_msg->data, sslbuf4, ssllen4);
stream_msg->data_len = ssllen4;
ssn.toserver_smsg_head = stream_msg;
ssn.toserver_smsg_tail = stream_msg;
de_ctx = DetectEngineCtxInit();
FAIL_IF_NULL(de_ctx);
de_ctx->flags |= DE_QUIET;
s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any any (msg:\"TLS\"; ssl_version:tls1.0; content:\"|01 00 00 AD|\"; sid:1;)");
FAIL_IF_NULL(s);
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
FLOWLOCK_WRLOCK(&f);
int r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS,
STREAM_TOSERVER, sslbuf1, ssllen1);
FAIL_IF(r != 0);
r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
sslbuf2, ssllen2);
FAIL_IF(r != 0);
r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
sslbuf3, ssllen3);
FAIL_IF(r != 0);
r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
sslbuf4, ssllen4);
FAIL_IF(r != 0);
FLOWLOCK_UNLOCK(&f);
SSLState *app_state = f.alstate;
FAIL_IF_NULL(app_state);
FAIL_IF(app_state->client_connp.content_type != 0x16);
FAIL_IF(app_state->client_connp.version != TLS_VERSION_10);
/* do detect */
SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
FAIL_IF_NOT(PacketAlertCheck(p, 1));
AppLayerParserThreadCtxFree(alp_tctx);
DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
DetectEngineCtxFree(de_ctx);
StreamTcpFreeConfig(TRUE);
FLOW_DESTROY(&f);
UTHFreePackets(&p, 1);
PASS;
}
#endif /* UNITTESTS */
/**
@ -662,8 +564,6 @@ static void DetectSslVersionRegisterTests(void)
DetectSslVersionTestDetect01);
UtRegisterTest("DetectSslVersionTestDetect02",
DetectSslVersionTestDetect02);
UtRegisterTest("DetectSslVersionTestDetect03",
DetectSslVersionTestDetect03);
#endif /* UNITTESTS */
return;

101
src/detect-tls-version.c
Unescape Escape View File

@ -480,105 +480,6 @@ static int DetectTlsVersionTestDetect02(void)
PASS;
}
static int DetectTlsVersionTestDetect03(void)
{
DetectEngineCtx *de_ctx = NULL;
Flow f;
uint8_t tlsbuf1[] = { 0x16 };
uint32_t tlslen1 = sizeof(tlsbuf1);
uint8_t tlsbuf2[] = { 0x03 };
uint32_t tlslen2 = sizeof(tlsbuf2);
uint8_t tlsbuf3[] = { 0x01 };
uint32_t tlslen3 = sizeof(tlsbuf3);
uint8_t tlsbuf4[] = { 0x01, 0x00, 0x00, 0xad, 0x03, 0x02 };
uint32_t tlslen4 = sizeof(tlsbuf4);
TcpSession ssn;
Packet *p = NULL;
Signature *s = NULL;
ThreadVars th_v;
DetectEngineThreadCtx *det_ctx = NULL;
AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
memset(&th_v, 0, sizeof(th_v));
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
p = UTHBuildPacket(NULL, 0, IPPROTO_TCP);
p->tcph->th_seq = htonl(1000);
FLOW_INITIALIZE(&f);
f.protoctx = (void *)&ssn;
p->flow = &f;
p->flowflags |= FLOW_PKT_TOSERVER;
p->flowflags |= FLOW_PKT_ESTABLISHED;
p->flags |= PKT_HAS_FLOW|PKT_STREAM_EST;
f.alproto = ALPROTO_TLS;
f.proto = p->proto;
StreamTcpInitConfig(TRUE);
StreamMsg *stream_msg = StreamMsgGetFromPool();
FAIL_IF_NULL(stream_msg);
memcpy(stream_msg->data, tlsbuf4, tlslen4);
stream_msg->data_len = tlslen4;
ssn.toserver_smsg_head = stream_msg;
ssn.toserver_smsg_tail = stream_msg;
de_ctx = DetectEngineCtxInit();
FAIL_IF_NULL(de_ctx);
de_ctx->flags |= DE_QUIET;
s = de_ctx->sig_list = SigInit(de_ctx,"alert tcp any any -> any any (msg:\"TLS\"; tls.version:1.0; content:\"|01 00 00 AD|\"; sid:1;)");
FAIL_IF_NULL(s);
SigGroupBuild(de_ctx);
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
FLOWLOCK_WRLOCK(&f);
int r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS,
STREAM_TOSERVER, tlsbuf1, tlslen1);
FAIL_IF(r != 0);
r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
tlsbuf2, tlslen2);
FAIL_IF(r != 0);
r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
tlsbuf3, tlslen3);
FAIL_IF(r != 0);
r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_TLS, STREAM_TOSERVER,
tlsbuf4, tlslen4);
FAIL_IF(r != 0);
FLOWLOCK_UNLOCK(&f);
SSLState *ssl_state = f.alstate;
FAIL_IF_NULL(ssl_state);
FAIL_IF(ssl_state->client_connp.content_type != 0x16);
FAIL_IF(ssl_state->client_connp.version != TLS_VERSION_10);
/* do detect */
SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
FAIL_IF_NOT(PacketAlertCheck(p, 1));
AppLayerParserThreadCtxFree(alp_tctx);
DetectEngineThreadCtxDeinit(&th_v, (void *)det_ctx);
DetectEngineCtxFree(de_ctx);
StreamTcpFreeConfig(TRUE);
FLOW_DESTROY(&f);
UTHFreePackets(&p, 1);
PASS;
}
#endif /* UNITTESTS */
/**
@ -593,8 +494,6 @@ static void DetectTlsVersionRegisterTests(void)
DetectTlsVersionTestDetect01);
UtRegisterTest("DetectTlsVersionTestDetect02",
DetectTlsVersionTestDetect02);
UtRegisterTest("DetectTlsVersionTestDetect03",
DetectTlsVersionTestDetect03);
#endif /* UNITTESTS */
}

26
src/detect-uricontent.c
Unescape Escape View File

@ -998,19 +998,6 @@ static int DetectUriSigTest05(void)
f.proto = p->proto;
StreamTcpInitConfig(TRUE);
StreamMsg *stream_msg = StreamMsgGetFromPool();
if (stream_msg == NULL) {
printf("no stream_msg: ");
goto end;
}
memcpy(stream_msg->data, httpbuf1, httplen1);
stream_msg->data_len = httplen1;
ssn.toserver_smsg_head = stream_msg;
ssn.toserver_smsg_tail = stream_msg;
de_ctx = DetectEngineCtxInit();
if (de_ctx == NULL) {
goto end;
@ -1127,19 +1114,6 @@ static int DetectUriSigTest06(void)
f.proto = p->proto;
StreamTcpInitConfig(TRUE);
StreamMsg *stream_msg = StreamMsgGetFromPool();
if (stream_msg == NULL) {
printf("no stream_msg: ");
goto end;
}
memcpy(stream_msg->data, httpbuf1, httplen1);
stream_msg->data_len = httplen1;
ssn.toserver_smsg_head = stream_msg;
ssn.toserver_smsg_tail = stream_msg;
de_ctx = DetectEngineCtxInit();
if (de_ctx == NULL) {
goto end;

196
src/detect.c
Unescape Escape View File

@ -237,7 +237,7 @@ void DetectExitPrintStats(ThreadVars *tv, void *data);
void DbgPrintSigs(DetectEngineCtx *, SigGroupHead *);
void DbgPrintSigs2(DetectEngineCtx *, SigGroupHead *);
static void PacketCreateMask(Packet *, SignatureMask *, uint16_t, int, StreamMsg *, int);
static void PacketCreateMask(Packet *, SignatureMask *, AppProto, bool, int);
/**
* \brief Create the path if default-rule-path was specified
@ -639,103 +639,6 @@ SigGroupHead *SigMatchSignaturesGetSgh(DetectEngineCtx *de_ctx, DetectEngineThre
SCReturnPtr(sgh, "SigGroupHead");
}
/** \brief Get the smsgs relevant to this packet
*
* \param f LOCKED flow
* \param p packet
* \param flags stream flags
*/
static StreamMsg *SigMatchSignaturesGetSmsg(Flow *f, Packet *p, uint8_t flags)
{
SCEnter();
DEBUG_ASSERT_FLOW_LOCKED(f);
StreamMsg *smsg = NULL;
if (p->proto == IPPROTO_TCP && f->protoctx != NULL && (p->flags & PKT_STREAM_EST)) {
TcpSession *ssn = (TcpSession *)f->protoctx;
/* at stream eof, or in inline mode, inspect all smsg's */
if ((flags & STREAM_EOF) || StreamTcpInlineMode()) {
if (p->flowflags & FLOW_PKT_TOSERVER) {
smsg = ssn->toserver_smsg_head;
/* deref from the ssn */
ssn->toserver_smsg_head = NULL;
ssn->toserver_smsg_tail = NULL;
SCLogDebug("to_server smsg %p at stream eof", smsg);
if (smsg)
SCLogDebug("to_server smsg %p, size %u, SEQ %u", smsg, smsg->data_len, smsg->seq);
} else {
smsg = ssn->toclient_smsg_head;
/* deref from the ssn */
ssn->toclient_smsg_head = NULL;
ssn->toclient_smsg_tail = NULL;
SCLogDebug("to_client smsg %p at stream eof", smsg);
if (smsg)
SCLogDebug("to_client smsg %p, size %u, SEQ %u", smsg, smsg->data_len, smsg->seq);
}
} else {
if (p->flowflags & FLOW_PKT_TOSERVER) {
StreamMsg *head = ssn->toserver_smsg_head;
if (unlikely(head == NULL)) {
SCLogDebug("no smsgs in to_server direction");
goto end;
}
/* if the smsg is bigger than the current packet, we will
* process the smsg in a later run */
if (SEQ_GT((head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len))) {
SCLogDebug("smsg ends beyond current packet, skipping for now %"PRIu32">%"PRIu32,
(head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len));
goto end;
}
smsg = head;
/* deref from the ssn */
ssn->toserver_smsg_head = NULL;
ssn->toserver_smsg_tail = NULL;
SCLogDebug("to_server smsg %p, size %u, SEQ %u", smsg, smsg->data_len, smsg->seq);
} else {
StreamMsg *head = ssn->toclient_smsg_head;
if (unlikely(head == NULL))
goto end;
/* if the smsg is bigger than the current packet, we will
* process the smsg in a later run */
if (SEQ_GT((head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len))) {
SCLogDebug("smsg ends beyond current packet, skipping for now %"PRIu32">%"PRIu32,
(head->seq + head->data_len), (TCP_GET_SEQ(p) + p->payload_len));
goto end;
}
smsg = head;
/* deref from the ssn */
ssn->toclient_smsg_head = NULL;
ssn->toclient_smsg_tail = NULL;
SCLogDebug("to_client smsg %p, size %u, SEQ %u", smsg, smsg->data_len, smsg->seq);
}
}
}
end:
#ifdef DEBUG
if (SCLogDebugEnabled()) {
StreamMsg *m = smsg;
while(m) {
SCLogDebug("m %p size %u, SEQ %u", m, m->data_len, m->seq);
PrintRawDataFp(stdout, m->data, m->data_len);
m = m->next;
}
}
#endif
SCReturnPtr(smsg, "StreamMsg");
}
static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
DetectEngineThreadCtx *det_ctx)
{
@ -863,17 +766,6 @@ static inline void DetectPrefilterMergeSort(DetectEngineCtx *de_ctx,
#define SMS_USE_FLOW_SGH 0x01
#define SMS_USED_PM 0x02
#ifdef DEBUG
static void DebugInspectIds(Packet *p, Flow *f, StreamMsg *smsg)
{
SCLogDebug("pcap_cnt %02"PRIu64", %s, %12s, smsg %s",
p->pcap_cnt, p->flowflags & FLOW_PKT_TOSERVER ? "toserver" : "toclient",
p->flags & PKT_STREAM_EST ? "established" : "stateless",
smsg ? "yes" : "no");
AppLayerParserStatePrintDetails(f->alparser);
}
#endif
static inline void
DetectPrefilterBuildNonPrefilterList(DetectEngineThreadCtx *det_ctx, SignatureMask mask)
{
@ -1000,7 +892,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
int state_alert = 0;
int alerts = 0;
int app_decoder_events = 0;
int has_state = 0; /* do we have an alstate to work with? */
bool has_state = false; /* do we have an alstate to work with? */
SCEnter();
@ -1009,8 +901,8 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
det_ctx->ticker++;
p->alerts.cnt = 0;
det_ctx->filestore_cnt = 0;
det_ctx->smsg = NULL;
det_ctx->base64_decoded_len = 0;
det_ctx->raw_stream_progress = 0;
/* No need to perform any detection on this packet, if the the given flag is set.*/
if (p->flags & PKT_NOPACKET_INSPECTION) {
@ -1083,17 +975,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
sms_runflags |= SMS_USE_FLOW_SGH;
}
PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
det_ctx->smsg = SigMatchSignaturesGetSmsg(pflow, p, flow_flags);
#if 0
StreamMsg *tmpsmsg = smsg;
while (tmpsmsg) {
printf("detect ---start---:\n");
PrintRawDataFp(stdout,tmpsmsg->data.data,tmpsmsg->data.data_len);
printf("detect ---end---:\n");
tmpsmsg = tmpsmsg->next;
}
#endif
}
/* Retrieve the app layer state and protocol and the tcp reassembled
@ -1106,6 +987,10 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
flow_flags = FlowGetDisruptionFlags(pflow, flow_flags);
has_state = (FlowGetAppState(pflow) != NULL);
alproto = FlowGetAppProtocol(pflow);
if (p->proto == IPPROTO_TCP && pflow->protoctx &&
StreamReassembleRawHasDataReady(pflow->protoctx, p)) {
p->flags |= PKT_DETECT_HAS_STREAMDATA;
}
SCLogDebug("alstate %s, alproto %u", has_state ? "true" : "false", alproto);
} else {
SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto);
@ -1151,11 +1036,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH);
}
#ifdef DEBUG
if (pflow) {
DebugInspectIds(p, pflow, det_ctx->smsg);
}
#endif
} else { /* p->flags & PKT_HAS_FLOW */
/* no flow */
@ -1194,8 +1074,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
/* create our prefilter mask */
SignatureMask mask = 0;
PacketCreateMask(p, &mask, alproto, has_state, det_ctx->smsg,
app_decoder_events);
PacketCreateMask(p, &mask, alproto, has_state, app_decoder_events);
/* build and prefilter non_pf list against the mask of the packet */
PACKET_PROFILING_DETECT_START(p, PROF_DETECT_NONMPMLIST);
@ -1382,33 +1261,25 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
/* if we have stream msgs, inspect against those first,
* but not for a "dsize" signature */
if (sflags & SIG_FLAG_REQUIRE_STREAM) {
char pmatch = 0;
if (det_ctx->smsg != NULL) {
StreamMsg *smsg_inspect = det_ctx->smsg;
for ( ; smsg_inspect != NULL; smsg_inspect = smsg_inspect->next) {
if (DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, pflow, smsg_inspect->data, smsg_inspect->data_len) == 1) {
SCLogDebug("match in smsg %p", smsg_inspect);
pmatch = 1;
det_ctx->flags |= DETECT_ENGINE_THREAD_CTX_STREAM_CONTENT_MATCH;
/* Tell the engine that this reassembled stream can drop the
* rest of the pkts with no further inspection */
if (s->action & ACTION_DROP)
alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
alert_flags |= PACKET_ALERT_FLAG_STREAM_MATCH;
break;
}
int pmatch = 0;
if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
pmatch = DetectEngineInspectStreamPayload(de_ctx, det_ctx, s, pflow, p);
if (pmatch) {
det_ctx->flags |= DETECT_ENGINE_THREAD_CTX_STREAM_CONTENT_MATCH;
/* Tell the engine that this reassembled stream can drop the
* rest of the pkts with no further inspection */
if (s->action & ACTION_DROP)
alert_flags |= PACKET_ALERT_FLAG_DROP_FLOW;
alert_flags |= PACKET_ALERT_FLAG_STREAM_MATCH;
}
} /* if (smsg != NULL) */
}
/* no match? then inspect packet payload */
if (pmatch == 0) {
SCLogDebug("no match in smsg, fall back to packet payload");
if (!(sflags & SIG_FLAG_REQUIRE_PACKET)) {
if (p->flags & PKT_STREAM_ADD)
goto next;
if (!(sflags & SIG_FLAG_REQUIRE_PACKET) && (p->flags & PKT_STREAM_ADD)) {
goto next;
}
if (DetectEngineInspectPacketPayload(de_ctx, det_ctx, s, pflow, p) != 1) {
@ -1448,7 +1319,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh
/* consider stateful sig matches */
if (sflags & SIG_FLAG_STATE_MATCH) {
if (has_state == 0) {
if (has_state == false) {
SCLogDebug("state matches but no state, we can't match");
goto next;
}
@ -1542,10 +1413,13 @@ end:
DetectPostInspectFirstSGH(p, pflow, det_ctx->sgh);
}
/* if we had no alerts that involved the smsgs,
* we can get rid of them now. */
StreamMsgReturnListToPool(det_ctx->smsg);
det_ctx->smsg = NULL;
/* update inspected tracker for raw reassembly */
if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL &&
det_ctx->raw_stream_progress > 0)
{
StreamReassembleRawUpdateProgress(pflow->protoctx, p,
det_ctx->raw_stream_progress);
}
}
PACKET_PROFILING_DETECT_END(p, PROF_DETECT_CLEANUP);
@ -2062,13 +1936,15 @@ deonly:
* SIG_MASK_REQUIRE_HTTP_STATE, SIG_MASK_REQUIRE_DCE_STATE
*/
static void
PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, int has_state, StreamMsg *smsg,
int app_decoder_events)
PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto,
bool has_state, int app_decoder_events)
{
/* no payload inspect flag doesn't apply to smsg */
if (smsg != NULL || (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0)) {
if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) {
SCLogDebug("packet has payload");
(*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
} else if (p->flags & PKT_DETECT_HAS_STREAMDATA) {
SCLogDebug("stream data available");
(*mask) |= SIG_MASK_REQUIRE_PAYLOAD;
} else {
SCLogDebug("packet has no payload");
(*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD;

4
src/detect.h
Unescape Escape View File

@ -791,6 +791,8 @@ typedef struct DetectEngineThreadCtx_ {
/* detection engine variables */
uint64_t raw_stream_progress;
/** offset into the payload of the last match by:
* content, pcre, etc */
uint32_t buffer_offset;
@ -864,8 +866,6 @@ typedef struct DetectEngineThreadCtx_ {
MpmThreadCtx mtcs; /**< thread ctx for stream mpm */
PrefilterRuleStore pmq;
StreamMsg *smsg;
/** SPM thread context used for scanning. This has been cloned from the
* prototype held by DetectEngineCtx. */
SpmThreadCtx *spm_thread_ctx;

15
src/stream-tcp-list.c
Unescape Escape View File

@ -584,18 +584,6 @@ int StreamTcpReassembleInsertSegment(ThreadVars *tv, TcpReassemblyThreadCtx *ra_
static inline int SegmentInUse(TcpSession *ssn, TcpStream *stream, TcpSegment *seg)
{
if (stream == &ssn->client && ssn->toserver_smsg_head != NULL) {
/* not (seg is entirely before first smsg, skip) */
if (!(SEQ_LEQ(seg->seq + TCP_SEG_LEN(seg), ssn->toserver_smsg_head->seq))) {
SCReturnInt(1);
}
} else if (stream == &ssn->server && ssn->toclient_smsg_head != NULL) {
/* not (seg is entirely before first smsg, skip) */
if (!(SEQ_LEQ(seg->seq + TCP_SEG_LEN(seg), ssn->toclient_smsg_head->seq))) {
SCReturnInt(1);
}
}
/* if proto detect isn't done, we're not returning */
if (!(stream->flags & STREAMTCP_STREAM_FLAG_GAP)) {
if (!(StreamTcpIsSetStreamFlagAppProtoDetectionCompleted(stream))) {
@ -610,9 +598,6 @@ static inline int SegmentInUse(TcpSession *ssn, TcpStream *stream, TcpSegment *s
/** \internal
* \brief check if we can remove a segment from our segment list
*
* If a segment is entirely before the oldest smsg, we can discard it. Otherwise
* we keep it around to be able to log it.
*
* \retval 1 yes
* \retval 0 no
*/

5
src/stream-tcp-private.h
Unescape Escape View File

@ -232,11 +232,6 @@ typedef struct TcpSession_ {
uint32_t reassembly_depth; /**< reassembly depth for the stream */
TcpStream server;
TcpStream client;
struct StreamMsg_ *toserver_smsg_head; /**< list of stream msgs (for detection inspection) */
struct StreamMsg_ *toserver_smsg_tail; /**< list of stream msgs (for detection inspection) */
struct StreamMsg_ *toclient_smsg_head; /**< list of stream msgs (for detection inspection) */
struct StreamMsg_ *toclient_smsg_tail; /**< list of stream msgs (for detection inspection) */
TcpStateQueue *queue; /**< list of SYN/ACK candidates */
} TcpSession;

1451
src/stream-tcp-reassemble.c
View File

File diff suppressed because it is too large Load Diff

2
src/stream-tcp-reassemble.h
Unescape Escape View File

@ -118,5 +118,7 @@ int StreamTcpAppLayerIsDisabled(Flow *f);
int StreamTcpCheckStreamContents(uint8_t *, uint16_t , TcpStream *);
#endif
bool StreamReassembleRawHasDataReady(TcpSession *ssn, Packet *p);
#endif /* __STREAM_TCP_REASSEMBLE_H__ */

80
src/stream-tcp.c
Unescape Escape View File

@ -187,8 +187,6 @@ void StreamTcpStreamCleanup(TcpStream *stream)
void StreamTcpSessionCleanup(TcpSession *ssn)
{
SCEnter();
StreamMsg *smsg = NULL;
TcpStateQueue *q, *q_next;
if (ssn == NULL)
@ -197,29 +195,6 @@ void StreamTcpSessionCleanup(TcpSession *ssn)
StreamTcpStreamCleanup(&ssn->client);
StreamTcpStreamCleanup(&ssn->server);
/* if we have (a) smsg(s), return to the pool */
smsg = ssn->toserver_smsg_head;
while(smsg != NULL) {
StreamMsg *smsg_next = smsg->next;
SCLogDebug("returning smsg %p to pool", smsg);
smsg->next = NULL;
smsg->prev = NULL;
StreamMsgReturnToPool(smsg);
smsg = smsg_next;
}
ssn->toserver_smsg_head = NULL;
smsg = ssn->toclient_smsg_head;
while(smsg != NULL) {
StreamMsg *smsg_next = smsg->next;
SCLogDebug("returning smsg %p to pool", smsg);
smsg->next = NULL;
smsg->prev = NULL;
StreamMsgReturnToPool(smsg);
smsg = smsg_next;
}
ssn->toclient_smsg_head = NULL;
q = ssn->queue;
while (q != NULL) {
q_next = q->next;
@ -553,9 +528,6 @@ void StreamTcpInitConfig(char quiet)
(int) (stream_config.reassembly_toserver_chunk_size *
(r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
}
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER,
stream_config.reassembly_toserver_chunk_size);
char *temp_stream_reassembly_toclient_chunk_size_str;
if (ConfGet("stream.reassembly.toclient-chunk-size",
&temp_stream_reassembly_toclient_chunk_size_str) == 1) {
@ -578,10 +550,6 @@ void StreamTcpInitConfig(char quiet)
(int) (stream_config.reassembly_toclient_chunk_size *
(r * 1.0 / RAND_MAX - 0.5) * rdrange / 100);
}
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT,
stream_config.reassembly_toclient_chunk_size);
if (!quiet) {
SCLogConfig("stream.reassembly \"toserver-chunk-size\": %"PRIu16,
stream_config.reassembly_toserver_chunk_size);
@ -593,8 +561,6 @@ void StreamTcpInitConfig(char quiet)
if (ConfGetBool("stream.reassembly.raw", &enable_raw) == 1) {
if (!enable_raw) {
stream_config.ssn_init_flags = STREAMTCP_FLAG_DISABLE_RAW;
// TODO how to handle this now?
// stream_config.segment_init_flags = SEGMENTTCP_FLAG_RAW_PROCESSED;
}
} else {
enable_raw = 1;
@ -6170,11 +6136,6 @@ static int StreamTcpTest05 (void)
int ret = 0;
StreamTcpUTInit(&stt.ra_ctx);
/* prevent L7 from kicking in */
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
tcph.th_win = htons(5480);
tcph.th_seq = htonl(10);
tcph.th_ack = htonl(20);
@ -7004,11 +6965,6 @@ static int StreamTcpTest14 (void)
strlcpy(os_policy_name, "linux\0", sizeof(os_policy_name));
ip_addr = StreamTcpParseOSPolicy(os_policy_name);
SCHInfoAddHostOSInfo(os_policy_name, ip_addr, -1);
/* prevent L7 from kicking in */
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
addr.s_addr = inet_addr("192.168.0.1");
tcph.th_win = htons(5480);
tcph.th_seq = htonl(10);
@ -7411,11 +7367,6 @@ static int StreamTcpTest15 (void)
strlcpy(os_policy_name, "linux\0", sizeof(os_policy_name));
ip_addr = StreamTcpParseOSPolicy(os_policy_name);
SCHInfoAddHostOSInfo(os_policy_name, ip_addr, -1);
/* prevent L7 from kicking in */
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
addr.s_addr = inet_addr("192.168.0.20");
tcph.th_win = htons(5480);
tcph.th_seq = htonl(10);
@ -7583,11 +7534,6 @@ static int StreamTcpTest16 (void)
strlcpy(os_policy_name, "linux\0", sizeof(os_policy_name));
ip_addr = StreamTcpParseOSPolicy(os_policy_name);
SCHInfoAddHostOSInfo(os_policy_name, ip_addr, -1);
/* prevent L7 from kicking in */
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
addr.s_addr = inet_addr("192.168.0.1");
tcph.th_win = htons(5480);
tcph.th_seq = htonl(10);
@ -7758,11 +7704,6 @@ static int StreamTcpTest17 (void)
strlcpy(os_policy_name, "linux\0", sizeof(os_policy_name));
ip_addr = StreamTcpParseOSPolicy(os_policy_name);
SCHInfoAddHostOSInfo(os_policy_name, ip_addr, -1);
/* prevent L7 from kicking in */
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
addr.s_addr = inet_addr("192.168.0.1");
tcph.th_win = htons(5480);
tcph.th_seq = htonl(10);
@ -8164,11 +8105,6 @@ static int StreamTcpTest23(void)
StreamTcpUTInit(&stt.ra_ctx);
StreamTcpUTSetupSession(&ssn);
/* prevent L7 from kicking in */
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
FLOW_INITIALIZE(&f);
ssn.client.os_policy = OS_POLICY_BSD;
f.protoctx = &ssn;
@ -8308,11 +8244,6 @@ static int StreamTcpTest25(void)
tcph.th_flags = TH_SYN | TH_CWR;
p->tcph = &tcph;
p->flowflags = FLOW_PKT_TOSERVER;
/* prevent L7 from kicking in */
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOSERVER, 4096);
StreamMsgQueueSetMinChunkLen(FLOW_PKT_TOCLIENT, 4096);
StreamTcpUTInit(&stt.ra_ctx);
if (StreamTcpPacket(&tv, p, &stt, &pq) == -1)
@ -9132,10 +9063,8 @@ static int StreamTcpTest33 (void)
StreamTcpThread stt;
TCPHdr tcph;
TcpReassemblyThreadCtx ra_ctx;
StreamMsgQueue stream_q;
PacketQueue pq;
memset(&pq,0,sizeof(PacketQueue));
memset(&stream_q, 0, sizeof(StreamMsgQueue));
memset(&ra_ctx, 0, sizeof(TcpReassemblyThreadCtx));
memset (&p, 0, SIZE_OF_PACKET);
memset (&f, 0, sizeof(Flow));
@ -9150,7 +9079,6 @@ static int StreamTcpTest33 (void)
p.tcph = &tcph;
p.flowflags = FLOW_PKT_TOSERVER;
int ret = 0;
ra_ctx.stream_q = &stream_q;
stt.ra_ctx = &ra_ctx;
StreamTcpInitConfig(TRUE);
@ -9236,10 +9164,8 @@ static int StreamTcpTest34 (void)
StreamTcpThread stt;
TCPHdr tcph;
TcpReassemblyThreadCtx ra_ctx;
StreamMsgQueue stream_q;
PacketQueue pq;
memset(&pq,0,sizeof(PacketQueue));
memset(&stream_q, 0, sizeof(StreamMsgQueue));
memset(&ra_ctx, 0, sizeof(TcpReassemblyThreadCtx));
memset (&p, 0, SIZE_OF_PACKET);
memset (&f, 0, sizeof(Flow));
@ -9254,7 +9180,6 @@ static int StreamTcpTest34 (void)
p.tcph = &tcph;
p.flowflags = FLOW_PKT_TOSERVER;
int ret = 0;
ra_ctx.stream_q = &stream_q;
stt.ra_ctx = &ra_ctx;
StreamTcpInitConfig(TRUE);
@ -9304,10 +9229,8 @@ static int StreamTcpTest35 (void)
StreamTcpThread stt;
TCPHdr tcph;
TcpReassemblyThreadCtx ra_ctx;
StreamMsgQueue stream_q;
PacketQueue pq;
memset(&pq,0,sizeof(PacketQueue));
memset(&stream_q, 0, sizeof(StreamMsgQueue));
memset(&ra_ctx, 0, sizeof(TcpReassemblyThreadCtx));
memset (&p, 0, SIZE_OF_PACKET);
memset (&f, 0, sizeof(Flow));
@ -9322,7 +9245,6 @@ static int StreamTcpTest35 (void)
p.tcph = &tcph;
p.flowflags = FLOW_PKT_TOSERVER;
int ret = 0;
ra_ctx.stream_q = &stream_q;
stt.ra_ctx = &ra_ctx;
StreamTcpInitConfig(TRUE);
@ -9554,7 +9476,7 @@ static int StreamTcpTest37(void)
}
TcpStream *stream = &(((TcpSession *)p->flow->protoctx)->client);
FAIL_IF(STREAM_RAW_PROGRESS(stream) != 3);
FAIL_IF(STREAM_RAW_PROGRESS(stream) != 0); // no detect no progress update
StreamTcpSessionClear(p->flow->protoctx);

7
src/stream-tcp.h
Unescape Escape View File

@ -45,7 +45,6 @@ typedef struct TcpStreamCnf_ {
uint64_t reassembly_memcap; /**< max memory usage for stream reassembly */
uint32_t ssn_init_flags; /**< new ssn flags will be initialized to this */
uint8_t segment_init_flags; /**< new seg flags will be initialized to this */
uint32_t prealloc_sessions; /**< ssns to prealloc per stream thread */
uint32_t prealloc_segments; /**< segments to prealloc per stream thread */
@ -118,6 +117,12 @@ int StreamTcpSegmentForEach(const Packet *p, uint8_t flag,
void StreamTcpReassembleConfigEnableOverlapCheck(void);
void TcpSessionSetReassemblyDepth(TcpSession *ssn, uint32_t size);
typedef int (*StreamReassembleRawFunc)(void *data, const uint8_t *input, const uint32_t input_len);
int StreamReassembleRaw(TcpSession *ssn, const Packet *p,
StreamReassembleRawFunc Callback, void *cb_data, uint64_t *progress_out);
void StreamReassembleRawUpdateProgress(TcpSession *ssn, Packet *p, uint64_t progress);
/** ------- Inline functions: ------ */
/**

239
src/stream.c
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2007-2013 Open Information Security Foundation
/* Copyright (C) 2007-2017 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -19,8 +19,6 @@
* \file
*
* \author Victor Julien <victor@inliniac.net>
*
* Stream Chunk Handling API
*/
#include "suricata-common.h"
@ -32,241 +30,6 @@
#include "stream-tcp.h"
#include "flow-util.h"
#ifdef DEBUG
static SCMutex stream_pool_memuse_mutex;
static uint64_t stream_pool_memuse = 0;
static uint64_t stream_pool_memcnt = 0;
#endif
/* per queue setting */
static uint16_t toserver_min_chunk_len = 2560;
static uint16_t toclient_min_chunk_len = 2560;
static Pool *stream_msg_pool = NULL;
static SCMutex stream_msg_pool_mutex = SCMUTEX_INITIALIZER;
static void StreamMsgEnqueue (StreamMsgQueue *q, StreamMsg *s)
{
SCEnter();
SCLogDebug("s %p", s);
/* more packets in queue */
if (q->top != NULL) {
s->next = q->top;
q->top->prev = s;
q->top = s;
/* only packet */
} else {
q->top = s;
q->bot = s;
}
q->len++;
#ifdef DBG_PERF
if (q->len > q->dbg_maxlen)
q->dbg_maxlen = q->len;
#endif /* DBG_PERF */
SCReturn;
}
static StreamMsg *StreamMsgDequeue (StreamMsgQueue *q)
{
SCEnter();
/* if the queue is empty there are no packets left.
* In that case we sleep and try again. */
if (q->len == 0) {
SCReturnPtr(NULL, "StreamMsg");
}
/* pull the bottom packet from the queue */
StreamMsg *s = q->bot;
/* more packets in queue */
if (q->bot->prev != NULL) {
q->bot = q->bot->prev;
q->bot->next = NULL;
/* just the one we remove, so now empty */
} else {
q->top = NULL;
q->bot = NULL;
}
q->len--;
s->next = NULL;
s->prev = NULL;
SCReturnPtr(s, "StreamMsg");
}
/* Used by stream reassembler to get msgs */
StreamMsg *StreamMsgGetFromPool(void)
{
SCMutexLock(&stream_msg_pool_mutex);
StreamMsg *s = (StreamMsg *)PoolGet(stream_msg_pool);
SCMutexUnlock(&stream_msg_pool_mutex);
return s;
}
/* Used by l7inspection to return msgs to pool */
void StreamMsgReturnToPool(StreamMsg *s)
{
SCLogDebug("s %p", s);
SCMutexLock(&stream_msg_pool_mutex);
PoolReturn(stream_msg_pool, (void *)s);
SCMutexUnlock(&stream_msg_pool_mutex);
}
/* Used by l7inspection to get msgs with data */
StreamMsg *StreamMsgGetFromQueue(StreamMsgQueue *q)
{
if (q->len > 0) {
StreamMsg *s = StreamMsgDequeue(q);
return s;
} else {
/* return NULL if we have no stream msg. Should only happen on signals. */
return NULL;
}
}
/* Used by stream reassembler to fill the queue for l7inspect reading */
void StreamMsgPutInQueue(StreamMsgQueue *q, StreamMsg *s)
{
StreamMsgEnqueue(q, s);
SCLogDebug("q->len %" PRIu32 "", q->len);
}
#define SIZE 4072
void *StreamMsgPoolAlloc(void)
{
if (StreamTcpReassembleCheckMemcap((uint32_t)(sizeof(StreamMsg)+SIZE)) == 0)
return NULL;
StreamMsg *m = SCCalloc(1, (sizeof(StreamMsg) + SIZE));
if (m != NULL) {
m->data = (uint8_t *)m + sizeof(StreamMsg);
m->data_size = SIZE;
StreamTcpReassembleIncrMemuse((uint32_t)(sizeof(StreamMsg)+SIZE));
}
return m;
}
int StreamMsgInit(void *data, void *initdata)
{
StreamMsg *s = data;
memset(s->data, 0, s->data_size);
#ifdef DEBUG
SCMutexLock(&stream_pool_memuse_mutex);
stream_pool_memuse += (sizeof(StreamMsg) + SIZE);
stream_pool_memcnt ++;
SCMutexUnlock(&stream_pool_memuse_mutex);
#endif
return 1;
}
void StreamMsgPoolFree(void *ptr)
{
if (ptr) {
SCFree(ptr);
StreamTcpReassembleDecrMemuse((uint32_t)(sizeof(StreamMsg)+SIZE));
}
}
void StreamMsgQueuesInit(uint32_t prealloc)
{
#ifdef DEBUG
SCMutexInit(&stream_pool_memuse_mutex, NULL);
#endif
SCMutexLock(&stream_msg_pool_mutex);
stream_msg_pool = PoolInit(0, prealloc, 0,
StreamMsgPoolAlloc,StreamMsgInit,
NULL,NULL,StreamMsgPoolFree);
if (stream_msg_pool == NULL)
exit(EXIT_FAILURE); /* XXX */
SCMutexUnlock(&stream_msg_pool_mutex);
}
void StreamMsgQueuesDeinit(char quiet)
{
if (quiet == FALSE) {
if (stream_msg_pool->max_outstanding > stream_msg_pool->allocated)
SCLogInfo("TCP segment chunk pool had a peak use of %u chunks, "
"more than the prealloc setting of %u",
stream_msg_pool->max_outstanding, stream_msg_pool->allocated);
}
SCMutexLock(&stream_msg_pool_mutex);
PoolFree(stream_msg_pool);
SCMutexUnlock(&stream_msg_pool_mutex);
#ifdef DEBUG
SCMutexDestroy(&stream_pool_memuse_mutex);
if (quiet == FALSE)
SCLogDebug("stream_pool_memuse %"PRIu64", stream_pool_memcnt %"PRIu64"", stream_pool_memuse, stream_pool_memcnt);
#endif
}
/** \brief alloc a stream msg queue
* \retval smq ptr to the queue or NULL */
StreamMsgQueue *StreamMsgQueueGetNew(void)
{
if (StreamTcpReassembleCheckMemcap((uint32_t)sizeof(StreamMsgQueue)) == 0)
return NULL;
StreamMsgQueue *smq = SCMalloc(sizeof(StreamMsgQueue));
if (unlikely(smq == NULL))
return NULL;
StreamTcpReassembleIncrMemuse((uint32_t)sizeof(StreamMsgQueue));
memset(smq, 0x00, sizeof(StreamMsgQueue));
return smq;
}
/** \brief Free a StreamMsgQueue
* \param q the queue to free
* \todo we may want to consider non empty queue's
*/
void StreamMsgQueueFree(StreamMsgQueue *q)
{
SCFree(q);
StreamTcpReassembleDecrMemuse((uint32_t)sizeof(StreamMsgQueue));
}
void StreamMsgQueueSetMinChunkLen(uint8_t dir, uint16_t len)
{
if (dir == FLOW_PKT_TOSERVER) {
toserver_min_chunk_len = len;
} else {
toclient_min_chunk_len = len;
}
}
uint16_t StreamMsgQueueGetMinChunkLen(uint8_t dir)
{
if (dir == FLOW_PKT_TOSERVER) {
return toserver_min_chunk_len;
} else {
return toclient_min_chunk_len;
}
}
/** \brief Return a list of smsgs to the pool */
void StreamMsgReturnListToPool(void *list)
{
/* if we have (a) smsg(s), return to the pool */
StreamMsg *smsg = (StreamMsg *)list;
while (smsg != NULL) {
StreamMsg *smsg_next = smsg->next;
SCLogDebug("returning smsg %p to pool", smsg);
smsg->next = NULL;
smsg->prev = NULL;
StreamMsgReturnToPool(smsg);
smsg = smsg_next;
}
}
/** \brief Run callback for all segments
*
* Must be called under flow lock.

39
src/stream.h
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
/* Copyright (C) 2007-2017 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -33,43 +33,6 @@
#define STREAM_GAP 0x10 /**< data gap encountered */
#define STREAM_DEPTH 0x20 /**< depth reached */
typedef struct StreamMsg_ {
struct StreamMsg_ *next;
struct StreamMsg_ *prev;
uint32_t seq; /**< sequence number */
uint32_t data_len; /**< length of the data */
uint32_t data_size;
uint8_t *data; /**< reassembled data: ptr to after this
* struct */
} StreamMsg;
typedef struct StreamMsgQueue_ {
StreamMsg *top;
StreamMsg *bot;
uint16_t len;
#ifdef DBG_PERF
uint16_t dbg_maxlen;
#endif /* DBG_PERF */
} StreamMsgQueue;
/* prototypes */
void StreamMsgQueuesInit(uint32_t prealloc);
void StreamMsgQueuesDeinit(char);
StreamMsg *StreamMsgGetFromPool(void);
void StreamMsgReturnToPool(StreamMsg *);
StreamMsg *StreamMsgGetFromQueue(StreamMsgQueue *);
void StreamMsgPutInQueue(StreamMsgQueue *, StreamMsg *);
StreamMsgQueue *StreamMsgQueueGetNew(void);
void StreamMsgQueueFree(StreamMsgQueue *);
void StreamMsgQueueSetMinChunkLen(uint8_t dir, uint16_t len);
uint16_t StreamMsgQueueGetMinChunkLen(uint8_t);
void StreamMsgReturnListToPool(void *);
typedef int (*StreamSegmentCallback)(const Packet *, void *, const uint8_t *, uint32_t);
int StreamSegmentForEach(const Packet *p, uint8_t flag,
StreamSegmentCallback CallbackFunc,

1
src/suricata.h
Unescape Escape View File

@ -47,7 +47,6 @@
* - ::Packet: Data relative to an individual packet with information about
* linked structure such as the ::Flow the ::Packet belongs to.
* - ::Flow: Information about a flow for example a TCP session
* - ::StreamMsg: structure containing the reassembled data
*
* \subsection runmode Running mode
*

1
src/util-mpm-ac.c
Unescape Escape View File

@ -1336,7 +1336,6 @@ void CudaReleasePacket(Packet *p)
* tables on priority.
* - Introduce profiling.
* - Retrieve sgh before buffer packet.
* - Buffer smsgs too.
*/
void SCACConstructBoth16and32StateTables(void)

Write Preview
Loading…
Cancel
Save