aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

file extract: split toserver and toclient tracking

Browse Source 
Split toserver and toclient file tracking for the http state.
remotes/origin/master-1.2.x
Victor Julien 14 years ago
parent 04ea70ccf7
commit d59ca75e46
10 changed files with 180 additions and 95 deletions
Show Stats Download Patch File Download Diff File

144
src/app-layer-htp-file.c
Unescape Escape View File

@ -65,32 +65,64 @@
* \brief Open the file with "filename" and pass the first chunk
* of data if any.
*
* \param f flow to store the file in
* \param s http state
* \param filename name of the file
* \param filename_len length of the name
* \param data data chunk (if any)
* \param data_len length of the data portion
* \param direction flow direction
*
* \retval 0 ok
* \retval -1 error
* \retval -2 not handling files on this flow
*/
int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len,
uint8_t *data, uint32_t data_len, uint16_t txid)
uint8_t *data, uint32_t data_len, uint16_t txid, uint8_t direction)
{
int retval = 0;
uint8_t flags = 0;
FileContainer *files = NULL;
if (s == NULL) {
SCReturnInt(-1);
}
if (s->files == NULL) {
s->files = FileContainerAlloc();
if (s->files == NULL) {
retval = -1;
goto end;
if (direction & STREAM_TOCLIENT) {
if (s->files_tc == NULL) {
s->files_tc = FileContainerAlloc();
if (s->files_tc == NULL) {
retval = -1;
goto end;
}
}
/* if the previous file is in the same txid, we
* reset the file part of the stateful detection
* engine. */
if (s->files_tc && s->files_tc->tail && s->files_tc->tail->txid == txid) {
SCLogDebug("new file in same tx, resetting de_state");
DeStateResetFileInspection(s->f);
}
files = s->files_tc;
} else {
if (s->files_ts == NULL) {
s->files_ts = FileContainerAlloc();
if (s->files_ts == NULL) {
retval = -1;
goto end;
}
}
/* if the previous file is in the same txid, we
* reset the file part of the stateful detection
* engine. */
if (s->files_ts && s->files_ts->tail && s->files_ts->tail->txid == txid) {
SCLogDebug("new file in same tx, resetting de_state");
DeStateResetFileInspection(s->f);
}
files = s->files_ts;
}
if (s->f->flags & FLOW_FILE_NO_STORE) {
@ -100,23 +132,15 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len,
flags |= FILE_NOMAGIC;
}
/* if the previous file is in the same txid, we
* reset the file part of the stateful detection
* engine. */
if (s->files && s->files->tail && s->files->tail->txid == txid) {
SCLogDebug("new file in same tx, resetting de_state");
DeStateResetFileInspection(s->f);
}
if (FileOpenFile(s->files, filename, filename_len,
if (FileOpenFile(files, filename, filename_len,
data, data_len, flags) == NULL)
{
retval = -1;
}
FileSetTx(s->files->tail, txid);
FileSetTx(files->tail, txid);
FilePrune(s->files);
FilePrune(files);
end:
SCReturnInt(retval);
}
@ -124,31 +148,41 @@ end:
/**
* \brief Store a chunk of data in the flow
*
* \param f flow to store the file in
* \param s http state
* \param data data chunk (if any)
* \param data_len length of the data portion
* \param direction flow direction
*
* \retval 0 ok
* \retval -1 error
* \retval -2 file doesn't need storing
*/
int HTPFileStoreChunk(HtpState *s, uint8_t *data, uint32_t data_len) {
int HTPFileStoreChunk(HtpState *s, uint8_t *data, uint32_t data_len,
uint8_t direction)
{
SCEnter();
int retval = 0;
int result = 0;
FileContainer *files = NULL;
if (s == NULL) {
SCReturnInt(-1);
}
if (s->files == NULL) {
if (direction & STREAM_TOCLIENT) {
files = s->files_tc;
} else {
files = s->files_ts;
}
if (files == NULL) {
SCLogDebug("no files in state");
retval = -1;
goto end;
}
result = FileAppendData(s->files, data, data_len);
result = FileAppendData(files, data, data_len);
if (result == -1) {
SCLogDebug("appending data failed");
retval = -1;
@ -156,7 +190,7 @@ int HTPFileStoreChunk(HtpState *s, uint8_t *data, uint32_t data_len) {
retval = -2;
}
FilePrune(s->files);
FilePrune(files);
end:
SCReturnInt(retval);
}
@ -164,10 +198,11 @@ end:
/**
* \brief Close the file in the flow
*
* \param f flow to store in
* \param s http state
* \param data data chunk if any
* \param data_len length of the data portion
* \param flags flags to indicate events
* \param direction flow direction
*
* Currently on the FLOW_FILE_TRUNCATED flag is implemented, indicating
* that the file isn't complete but we're stopping storing it.
@ -176,27 +211,38 @@ end:
* \retval -1 error
* \retval -2 not storing files on this flow/tx
*/
int HTPFileClose(HtpState *s, uint8_t *data, uint32_t data_len, uint8_t flags) {
int HTPFileClose(HtpState *s, uint8_t *data, uint32_t data_len,
uint8_t flags, uint8_t direction)
{
SCEnter();
int retval = 0;
int result = 0;
FileContainer *files = NULL;
if (s == NULL) {
SCReturnInt(-1);
}
if (s->files == NULL) {
if (direction & STREAM_TOCLIENT) {
files = s->files_tc;
} else {
files = s->files_ts;
}
if (files == NULL) {
retval = -1;
goto end;
}
result = FileCloseFile(s->files, data, data_len, flags);
result = FileCloseFile(files, data, data_len, flags);
if (result == -1) {
retval = -1;
} else if (result == -2) {
retval = -2;
}
FilePrune(s->files);
FilePrune(files);
end:
SCReturnInt(retval);
}
@ -363,7 +409,8 @@ static int HTPFileParserTest02(void) {
goto end;
}
if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) {
if (http_state->files_ts == NULL || http_state->files_ts->tail == NULL ||
http_state->files_ts->tail->state != FILE_STATE_CLOSED) {
goto end;
}
@ -485,11 +532,12 @@ static int HTPFileParserTest03(void) {
goto end;
}
if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) {
if (http_state->files_ts == NULL || http_state->files_ts->tail == NULL ||
http_state->files_ts->tail->state != FILE_STATE_CLOSED) {
goto end;
}
if (http_state->files->head->chunks_head->len != 11) {
if (http_state->files_ts->head->chunks_head->len != 11) {
goto end;
}
@ -611,7 +659,8 @@ static int HTPFileParserTest04(void) {
goto end;
}
if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) {
if (http_state->files_ts == NULL || http_state->files_ts->tail == NULL ||
http_state->files_ts->tail->state != FILE_STATE_CLOSED) {
goto end;
}
@ -692,39 +741,40 @@ static int HTPFileParserTest05(void) {
goto end;
}
if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) {
if (http_state->files_ts == NULL || http_state->files_ts->tail == NULL ||
http_state->files_ts->tail->state != FILE_STATE_CLOSED) {
goto end;
}
if (http_state->files->head == http_state->files->tail)
if (http_state->files_ts->head == http_state->files_ts->tail)
goto end;
if (http_state->files->head->next != http_state->files->tail)
if (http_state->files_ts->head->next != http_state->files_ts->tail)
goto end;
if (http_state->files->head->chunks_head->len != 11) {
if (http_state->files_ts->head->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead\n",
http_state->files->head->chunks_head->len);
PrintRawDataFp(stdout, http_state->files->head->chunks_head->data,
http_state->files->head->chunks_head->len);
http_state->files_ts->head->chunks_head->len);
PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len);
goto end;
}
if (memcmp("filecontent", http_state->files->head->chunks_head->data,
http_state->files->head->chunks_head->len) != 0) {
if (memcmp("filecontent", http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len) != 0) {
goto end;
}
if (http_state->files->tail->chunks_head->len != 11) {
if (http_state->files_ts->tail->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead\n",
http_state->files->tail->chunks_head->len);
PrintRawDataFp(stdout, http_state->files->tail->chunks_head->data,
http_state->files->tail->chunks_head->len);
http_state->files_ts->tail->chunks_head->len);
PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data,
http_state->files_ts->tail->chunks_head->len);
goto end;
}
if (memcmp("FILECONTENT", http_state->files->tail->chunks_head->data,
http_state->files->tail->chunks_head->len) != 0) {
if (memcmp("FILECONTENT", http_state->files_ts->tail->chunks_head->data,
http_state->files_ts->tail->chunks_head->len) != 0) {
goto end;
}
result = 1;

6
src/app-layer-htp-file.h
Unescape Escape View File

@ -25,9 +25,9 @@
#ifndef __APP_LAYER_HTP_FILE_H__
#define __APP_LAYER_HTP_FILE_H__
int HTPFileOpen(HtpState *, uint8_t *, uint16_t, uint8_t *, uint32_t, uint16_t);
int HTPFileStoreChunk(HtpState *, uint8_t *, uint32_t);
int HTPFileClose(HtpState *, uint8_t *, uint32_t, uint8_t);
int HTPFileOpen(HtpState *, uint8_t *, uint16_t, uint8_t *, uint32_t, uint16_t, uint8_t);
int HTPFileStoreChunk(HtpState *, uint8_t *, uint32_t, uint8_t);
int HTPFileClose(HtpState *, uint8_t *, uint32_t, uint8_t, uint8_t);
void HTPFileParserRegisterTests(void);

46
src/app-layer-htp.c
Unescape Escape View File

@ -233,7 +233,8 @@ void HTPStateFree(void *state)
htp_connp_destroy_all(s->connp);
}
FileContainerFree(s->files);
FileContainerFree(s->files_ts);
FileContainerFree(s->files_tc);
SCFree(s);
#ifdef DEBUG
@ -1096,7 +1097,8 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
printf("FILEDATA (final chunk) END: \n");
#endif
if (!(htud->flags & HTP_DONTSTORE)) {
if (HTPFileClose(hstate, filedata, filedata_len, flags) == -1)
if (HTPFileClose(hstate, filedata, filedata_len, flags,
STREAM_TOSERVER) == -1)
{
goto end;
}
@ -1118,7 +1120,8 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
#endif
if (!(htud->flags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate, filedata, filedata_len);
result = HTPFileStoreChunk(hstate, filedata,
filedata_len, STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1189,13 +1192,14 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
#endif
result = HTPFileOpen(hstate, filename, filename_len,
filedata, filedata_len, hstate->transaction_cnt);
filedata, filedata_len, hstate->transaction_cnt,
STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
htud->flags |= HTP_DONTSTORE;
} else {
if (HTPFileClose(hstate, NULL, 0, 0) == -1) {
if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
goto end;
}
}
@ -1208,7 +1212,8 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
result = HTPFileOpen(hstate, filename, filename_len,
filedata, filedata_len, hstate->transaction_cnt);
filedata, filedata_len, hstate->transaction_cnt,
STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1271,8 +1276,8 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
filename_len = bstr_len(tx->parsed_uri->path);
}
result = HTPFileOpen(hstate, filename, filename_len,
data, data_len, hstate->transaction_cnt);
result = HTPFileOpen(hstate, filename, filename_len, data, data_len,
hstate->transaction_cnt, STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1287,7 +1292,7 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, HtpTxUserData *htud,
/* otherwise, just store the data */
if (!(htud->flags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate, data, data_len);
result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1324,7 +1329,7 @@ int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
}
result = HTPFileOpen(hstate, filename, filename_len,
data, data_len, hstate->transaction_cnt);
data, data_len, hstate->transaction_cnt, STREAM_TOCLIENT);
SCLogDebug("result %d", result);
if (result == -1) {
goto end;
@ -1340,7 +1345,7 @@ int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
/* otherwise, just store the data */
if (!(htud->flags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate, data, data_len);
result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
SCLogDebug("result %d", result);
if (result == -1) {
goto end;
@ -1585,7 +1590,7 @@ static int HTPCallbackRequest(htp_connp_t *connp) {
if (htud != NULL) {
if (htud->flags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
(void)HTPFileClose(hstate, NULL, 0, 0);
(void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER);
htud->flags &= ~HTP_FILENAME_SET;
}
}
@ -1619,7 +1624,7 @@ static int HTPCallbackResponse(htp_connp_t *connp) {
if (htud != NULL) {
if (htud->flags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
(void)HTPFileClose(hstate, NULL, 0, 0);
(void)HTPFileClose(hstate, NULL, 0, 0, STREAM_TOCLIENT);
htud->flags &= ~HTP_FILENAME_SET;
}
}
@ -1974,12 +1979,23 @@ void AppLayerHtpPrintStats(void) {
#endif
}
static FileContainer *HTPStateGetFiles(void *state) {
/** \internal
* \brief get files callback
* \param state state ptr
* \param direction flow direction
* \retval files files ptr
*/
static FileContainer *HTPStateGetFiles(void *state, uint8_t direction) {
if (state == NULL)
return NULL;
HtpState *http_state = (HtpState *)state;
SCReturnPtr(http_state->files, "FileContainer");
if (direction & STREAM_TOCLIENT) {
SCReturnPtr(http_state->files_tc, "FileContainer");
} else {
SCReturnPtr(http_state->files_ts, "FileContainer");
}
}
/**

3
src/app-layer-htp.h
Unescape Escape View File

@ -151,7 +151,8 @@ typedef struct HtpState_ {
uint16_t transaction_done;
uint32_t request_body_limit;
uint32_t response_body_limit;
FileContainer *files;
FileContainer *files_ts;
FileContainer *files_tc;
} HtpState;
void RegisterHTPParsers(void);

7
src/app-layer-parser.c
Unescape Escape View File

@ -75,15 +75,16 @@ static uint32_t al_result_pool_elmts = 0;
/** \brief Get the file container flow
* \param f flow pointer to a LOCKED flow
* \retval files void pointer to the state
* \retval direction flow direction, either STREAM_TOCLIENT or STREAM_TOSERVER
* \retval NULL in case we have no state */
FileContainer *AppLayerGetFilesFromFlow(Flow *f) {
FileContainer *AppLayerGetFilesFromFlow(Flow *f, uint8_t direction) {
uint16_t alproto = f->alproto;
if (alproto == ALPROTO_UNKNOWN)
return NULL;
if (al_proto_table[alproto].StateGetFiles != NULL)
return al_proto_table[alproto].StateGetFiles(AppLayerGetProtoStateFromFlow(f));
return al_proto_table[alproto].StateGetFiles(AppLayerGetProtoStateFromFlow(f), direction);
else
return NULL;
}
@ -635,7 +636,7 @@ void *AppLayerGetProtocolParserLocalStorage(uint16_t proto)
}
void AppLayerRegisterGetFilesFunc(uint16_t proto,
FileContainer *(*StateGetFiles)(void *state))
FileContainer *(*StateGetFiles)(void *, uint8_t))
{
al_proto_table[proto].StateGetFiles = StateGetFiles;
}

6
src/app-layer-parser.h
Unescape Escape View File

@ -52,7 +52,7 @@ typedef struct AppLayerProto_ {
void (*StateTransactionFree)(void *, uint16_t);
void *(*LocalStorageAlloc)(void);
void (*LocalStorageFree)(void *);
FileContainer *(*StateGetFiles)(void *);
FileContainer *(*StateGetFiles)(void *, uint8_t);
} AppLayerProto;
@ -249,7 +249,7 @@ void AppLayerRegisterLocalStorageFunc(uint16_t proto,
void (*LocalStorageFree)(void *));
void *AppLayerGetProtocolParserLocalStorage(uint16_t);
void AppLayerRegisterGetFilesFunc(uint16_t proto,
FileContainer *(*StateGetFile)(void *));
FileContainer *(*StateGetFile)(void *, uint8_t));
void AppLayerRegisterLogger(uint16_t proto);
uint16_t AppLayerGetProtoByName(const char *);
@ -283,6 +283,6 @@ void AppLayerFreeProbingParsersInfo(AppLayerProbingParserInfo *);
void AppLayerPrintProbingParsers(AppLayerProbingParser *);
uint16_t AppLayerGetStateVersion(Flow *f);
FileContainer *AppLayerGetFilesFromFlow(Flow *);
FileContainer *AppLayerGetFilesFromFlow(Flow *, uint8_t);
#endif /* __APP_LAYER_PARSER_H__ */

8
src/detect-engine-file.c
Unescape Escape View File

@ -180,6 +180,7 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *
size_t start_tx = 0;
size_t end_tx = 0;
int match = 0;
FileContainer *ffc;
/* locking the flow, we will inspect the htp state */
SCMutexLock(&f->m);
@ -190,6 +191,11 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *
goto end;
}
if (flags & STREAM_TOCLIENT)
ffc = htp_state->files_tc;
else
ffc = htp_state->files_ts;
if (htp_state->connp != NULL && htp_state->connp->conn != NULL)
{
start_tx = AppLayerTransactionGetInspectId(f);
@ -205,7 +211,7 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *
/* inspect files for this transaction */
det_ctx->tx_id = (uint16_t)idx;
match = DetectFileInspect(tv, det_ctx, f, s, htp_state->files);
match = DetectFileInspect(tv, det_ctx, f, s, ffc);
if (match == 1) {
r = 1;
} else if (match == 2) {

3
src/detect-engine-state.c
Unescape Escape View File

@ -961,7 +961,8 @@ int DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, Dete
if (!(f->de_state->flags & DE_STATE_FILE_STORE_DISABLED)) {
if (DeStateStoreFilestoreSigsCantMatch(det_ctx->sgh, f->de_state, flags) == 1) {
SCLogDebug("disabling file storage for transaction");
FileDisableStoringForTransaction(f, det_ctx->tx_id);
FileDisableStoringForTransaction(f, flags & (STREAM_TOCLIENT|STREAM_TOSERVER),
det_ctx->tx_id);
f->de_state->flags |= DE_STATE_FILE_STORE_DISABLED;
}
}

42
src/detect.c
Unescape Escape View File

@ -1775,31 +1775,33 @@ end:
p->flow->flags |= FLOW_SGH_TOCLIENT;
}
if (p->flow->flags & FLOW_SGH_TOCLIENT && p->flow->flags & FLOW_SGH_TOSERVER) {
/* if we know both sides of the flow have had their sgh check
* and both are null, we will never decide to store. So disable
* storage completely. */
if ((p->flow->sgh_toserver == NULL ||
p->flow->sgh_toserver->filestore_cnt == 0)
&&
(p->flow->sgh_toclient == NULL ||
if (p->flow->flags & FLOW_SGH_TOCLIENT && (p->flow->sgh_toclient == NULL ||
p->flow->sgh_toclient->filestore_cnt == 0))
{
FileDisableStoring(p->flow, STREAM_TOCLIENT);
}
if (p->flow->flags & FLOW_SGH_TOSERVER && (p->flow->sgh_toserver == NULL ||
p->flow->sgh_toserver->filestore_cnt == 0))
{
FileDisableStoring(p->flow, STREAM_TOSERVER);
}
/* check if this flow needs magic, if not disable it */
if (!(FileForceMagic())) {
if (p->flow->flags & FLOW_SGH_TOCLIENT && (p->flow->sgh_toclient == NULL ||
!(p->flow->sgh_toclient->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)))
{
FileDisableStoring(p->flow);
SCLogInfo("disabling magic for flow");
FileDisableMagic(p->flow, STREAM_TOCLIENT);
}
/* check if this flow needs magic, if not disable it */
if (!(FileForceMagic())) {
if ((p->flow->sgh_toserver == NULL ||
!(p->flow->sgh_toserver->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC))
&&
(p->flow->sgh_toclient == NULL ||
!(p->flow->sgh_toclient->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)))
{
SCLogInfo("disabling magic for flow");
FileDisableMagic(p->flow);
}
if (p->flow->flags & FLOW_SGH_TOSERVER && (p->flow->sgh_toserver == NULL ||
!(p->flow->sgh_toserver->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)))
{
SCLogInfo("disabling magic for flow");
FileDisableMagic(p->flow, STREAM_TOSERVER);
}
}
}

10
src/log-file.c
Unescape Escape View File

@ -37,6 +37,8 @@
#include "detect-filemagic.h"
#include "stream.h"
#include "util-print.h"
#include "util-unittest.h"
#include "util-privs.h"
@ -179,17 +181,23 @@ static TmEcode LogFileLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue
{
SCEnter();
LogFileLogThread *aft = (LogFileLogThread *)data;
uint8_t flags = 0;
/* no flow, no htp state */
if (p->flow == NULL) {
SCReturnInt(TM_ECODE_OK);
}
if (p->flowflags & FLOW_PKT_TOCLIENT)
flags |= STREAM_TOCLIENT;
else
flags |= STREAM_TOSERVER;
int file_close = (p->flags & PKT_PSEUDO_STREAM_END) ? 1 : 0;
SCMutexLock(&p->flow->m);
FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow);
FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow, flags);
SCLogDebug("ffc %p", ffc);
if (ffc != NULL) {
File *ff;

Write Preview
Loading…
Cancel
Save