file-extraction: Disconnect file handling from flow and move into the app layer state.

remotes/origin/master-1.2.x
Victor Julien 15 years ago
parent 27645f64c6
commit e1022ee5ae

@ -47,7 +47,6 @@ flow-manager.c flow-manager.h \
flow-queue.c flow-queue.h \
flow-hash.c flow-hash.h \
flow-util.c flow-util.h \
flow-file.c flow-file.h \
util-mem.h \
flow-var.c flow-var.h \
flow-bit.c flow-bit.h \
@ -220,6 +219,7 @@ util-vector.h \
util-device.c util-device.h \
util-checksum.c util-checksum.h \
util-runmodes.c util-runmodes.h \
util-file.c util-file.h \
tm-modules.c tm-modules.h \
tm-queues.c tm-queues.h \
tm-queuehandlers.c tm-queuehandlers.h \

@ -55,7 +55,6 @@
#include "util-unittest.h"
#include "util-unittest-helper.h"
#include "flow-util.h"
#include "flow-file.h"
#include "detect-engine.h"
#include "detect-engine-state.h"

@ -52,7 +52,6 @@
#include "util-unittest.h"
#include "util-unittest-helper.h"
#include "flow-util.h"
#include "flow-file.h"
#include "detect-engine.h"
#include "detect-engine-state.h"
@ -76,44 +75,40 @@
* \retval -1 error
* \retval -2 not handling files on this flow
*/
int HTPFileOpen(Flow *f, uint8_t *filename, uint16_t filename_len,
int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len,
uint8_t *data, uint32_t data_len)
{
int retval = 0;
uint16_t txid;
uint8_t flags = 0;
if (f == NULL) {
if (s == NULL) {
SCReturnInt(-1);
}
txid = AppLayerTransactionGetAvailId(f) - 1;
txid = AppLayerTransactionGetAvailId(s->f) - 1;
SCMutexLock(&f->files_m);
{
if (f->files == NULL) {
f->files = FlowFileContainerAlloc();
if (f->files == NULL) {
retval = -1;
goto end;
}
}
if (f->flags & FLOW_FILE_NO_HANDLING) {
flags |= FLOW_FILE_NOSTORE;
}
if (FlowFileOpenFile(f->files, filename, filename_len,
data, data_len, flags) == NULL)
{
if (s->files == NULL) {
s->files = FileContainerAlloc();
if (s->files == NULL) {
retval = -1;
goto end;
}
}
FlowFileSetTx(f->files->tail, txid);
if (s->f->flags & FLOW_FILE_NO_HANDLING) {
flags |= FILE_NOSTORE;
}
if (FileOpenFile(s->files, filename, filename_len,
data, data_len, flags) == NULL)
{
retval = -1;
}
FileSetTx(s->files->tail, txid);
end:
SCMutexUnlock(&f->files_m);
SCReturnInt(retval);
}
@ -128,35 +123,31 @@ end:
* \retval -1 error
* \retval -2 file doesn't need storing
*/
int HTPFileStoreChunk(Flow *f, uint8_t *data, uint32_t data_len) {
int HTPFileStoreChunk(HtpState *s, uint8_t *data, uint32_t data_len) {
SCEnter();
int retval = 0;
int result = 0;
if (f == NULL) {
if (s == NULL) {
SCReturnInt(-1);
}
SCMutexLock(&f->files_m);
{
if (f->files == NULL) {
SCLogDebug("no files in flow");
retval = -1;
goto end;
}
if (s->files == NULL) {
SCLogDebug("no files in state");
retval = -1;
goto end;
}
result = FlowFileAppendData(f->files, data, data_len);
if (result == -1) {
SCLogDebug("appending data failed");
retval = -1;
} else if (result == -2) {
retval = -2;
}
result = FileAppendData(s->files, data, data_len);
if (result == -1) {
SCLogDebug("appending data failed");
retval = -1;
} else if (result == -2) {
retval = -2;
}
end:
SCMutexUnlock(&f->files_m);
SCReturnInt(retval);
}
@ -175,31 +166,27 @@ end:
* \retval -1 error
* \retval -2 not storing files on this flow/tx
*/
int HTPFileClose(Flow *f, uint8_t *data, uint32_t data_len, uint8_t flags) {
int HTPFileClose(HtpState *s, uint8_t *data, uint32_t data_len, uint8_t flags) {
int retval = 0;
int result = 0;
if (f == NULL) {
if (s == NULL) {
SCReturnInt(-1);
}
SCMutexLock(&f->files_m);
{
if (f->files == NULL) {
retval = -1;
goto end;
}
if (s->files == NULL) {
retval = -1;
goto end;
}
result = FlowFileCloseFile(f->files, data, data_len, flags);
if (result == -1) {
retval = -1;
} else if (result == -2) {
retval = -2;
}
result = FileCloseFile(s->files, data, data_len, flags);
if (result == -1) {
retval = -1;
} else if (result == -2) {
retval = -2;
}
end:
SCMutexUnlock(&f->files_m);
SCReturnInt(retval);
}
@ -365,7 +352,7 @@ static int HTPFileParserTest02(void) {
goto end;
}
if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) {
if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) {
goto end;
}
@ -487,11 +474,11 @@ static int HTPFileParserTest03(void) {
goto end;
}
if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) {
if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) {
goto end;
}
if (f.files->head->chunks_head->len != 11) {
if (http_state->files->head->chunks_head->len != 11) {
goto end;
}
@ -613,7 +600,7 @@ static int HTPFileParserTest04(void) {
goto end;
}
if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) {
if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) {
goto end;
}
@ -694,39 +681,39 @@ static int HTPFileParserTest05(void) {
goto end;
}
if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) {
if (http_state->files == NULL || http_state->files->tail == NULL || http_state->files->tail->state != FILE_STATE_CLOSED) {
goto end;
}
if (f.files->head == f.files->tail)
if (http_state->files->head == http_state->files->tail)
goto end;
if (f.files->head->next != f.files->tail)
if (http_state->files->head->next != http_state->files->tail)
goto end;
if (f.files->head->chunks_head->len != 11) {
if (http_state->files->head->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead\n",
f.files->head->chunks_head->len);
PrintRawDataFp(stdout, f.files->head->chunks_head->data,
f.files->head->chunks_head->len);
http_state->files->head->chunks_head->len);
PrintRawDataFp(stdout, http_state->files->head->chunks_head->data,
http_state->files->head->chunks_head->len);
goto end;
}
if (memcmp("filecontent", f.files->head->chunks_head->data,
f.files->head->chunks_head->len) != 0) {
if (memcmp("filecontent", http_state->files->head->chunks_head->data,
http_state->files->head->chunks_head->len) != 0) {
goto end;
}
if (f.files->tail->chunks_head->len != 11) {
if (http_state->files->tail->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead\n",
f.files->tail->chunks_head->len);
PrintRawDataFp(stdout, f.files->tail->chunks_head->data,
f.files->tail->chunks_head->len);
http_state->files->tail->chunks_head->len);
PrintRawDataFp(stdout, http_state->files->tail->chunks_head->data,
http_state->files->tail->chunks_head->len);
goto end;
}
if (memcmp("FILECONTENT", f.files->tail->chunks_head->data,
f.files->tail->chunks_head->len) != 0) {
if (memcmp("FILECONTENT", http_state->files->tail->chunks_head->data,
http_state->files->tail->chunks_head->len) != 0) {
goto end;
}
result = 1;

@ -25,9 +25,9 @@
#ifndef __APP_LAYER_HTP_FILE_H__
#define __APP_LAYER_HTP_FILE_H__
int HTPFileOpen(Flow *, uint8_t *, uint16_t, uint8_t *, uint32_t);
int HTPFileStoreChunk(Flow *, uint8_t *, uint32_t);
int HTPFileClose(Flow *, uint8_t *, uint32_t, uint8_t);
int HTPFileOpen(HtpState *, uint8_t *, uint16_t, uint8_t *, uint32_t);
int HTPFileStoreChunk(HtpState *, uint8_t *, uint32_t);
int HTPFileClose(HtpState *, uint8_t *, uint32_t, uint8_t);
void HTPFileParserRegisterTests(void);

@ -41,6 +41,7 @@
#include "util-print.h"
#include "util-pool.h"
#include "util-radix-tree.h"
#include "util-file.h"
#include "stream-tcp-private.h"
#include "stream-tcp-reassemble.h"
@ -61,7 +62,6 @@
#include "util-unittest.h"
#include "util-unittest-helper.h"
#include "flow-util.h"
#include "flow-file.h"
#include "detect-engine.h"
#include "detect-engine-state.h"
@ -229,6 +229,7 @@ void HTPStateFree(void *state)
htp_connp_destroy_all(s->connp);
}
FileContainerFree(s->files);
SCFree(s);
#ifdef DEBUG
@ -1065,7 +1066,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud,
filedata_len = form_end - filedata - 2; /* 0d 0a */
} else if (htud->flags & HTP_BODY_COMPLETE) {
filedata_len = chunks_buffer_len;
flags = FLOW_FILE_TRUNCATED;
flags = FILE_TRUNCATED;
}
BUG_ON(filedata_len > chunks_buffer_len);
@ -1075,7 +1076,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud,
printf("FILEDATA (final chunk) END: \n");
#endif
if (!(htud->flags & HTP_DONTSTORE)) {
if (HTPFileClose(hstate->f, filedata, filedata_len, flags) == -1)
if (HTPFileClose(hstate, filedata, filedata_len, flags) == -1)
{
goto end;
}
@ -1097,7 +1098,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud,
#endif
if (!(htud->flags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate->f, filedata, filedata_len);
result = HTPFileStoreChunk(hstate, filedata, filedata_len);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1167,14 +1168,14 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud,
printf("FILEDATA END: \n");
#endif
result = HTPFileOpen(hstate->f, filename, filename_len,
result = HTPFileOpen(hstate, filename, filename_len,
filedata, filedata_len);
if (result == -1) {
goto end;
} else if (result == -2) {
htud->flags |= HTP_DONTSTORE;
} else {
if (HTPFileClose(hstate->f, NULL, 0, 0) == -1) {
if (HTPFileClose(hstate, NULL, 0, 0) == -1) {
goto end;
}
}
@ -1185,7 +1186,7 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, SCHtpTxUserData *htud,
SCLogDebug("offset %u", offset);
htud->body_parsed = offset;
result = HTPFileOpen(hstate->f, filename, filename_len,
result = HTPFileOpen(hstate, filename, filename_len,
filedata, filedata_len);
if (result == -1) {
goto end;
@ -1249,7 +1250,7 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, SCHtpTxUserData *htud,
filename_len = bstr_len(tx->parsed_uri->path);
}
result = HTPFileOpen(hstate->f, filename, filename_len,
result = HTPFileOpen(hstate, filename, filename_len,
data, data_len);
if (result == -1) {
goto end;
@ -1265,7 +1266,7 @@ int HtpRequestBodyHandlePUT(HtpState *hstate, SCHtpTxUserData *htud,
/* otherwise, just store the data */
if (!(htud->flags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate->f, data, data_len);
result = HTPFileStoreChunk(hstate, data, data_len);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1440,7 +1441,7 @@ static int HTPCallbackRequest(htp_connp_t *connp) {
if (htud != NULL) {
if (htud->flags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
(void)HTPFileClose(hstate->f, NULL, 0, 0);
(void)HTPFileClose(hstate, NULL, 0, 0);
htud->flags &= ~HTP_FILENAME_SET;
}
}
@ -1776,6 +1777,14 @@ void AppLayerHtpPrintStats(void) {
#endif
}
static FileContainer *HTPStateGetFiles(void *state) {
if (state == NULL)
return NULL;
HtpState *http_state = (HtpState *)state;
SCReturnPtr(http_state->files, "FileContainer");
}
/**
* \brief Register the HTTP protocol and state handling functions to APP layer
* of the engine.
@ -1802,6 +1811,7 @@ void RegisterHTPParsers(void)
AppLayerRegisterStateFuncs(ALPROTO_HTTP, HTPStateAlloc, HTPStateFree);
AppLayerRegisterTransactionIdFuncs(ALPROTO_HTTP, HTPStateUpdateTransactionId, HTPStateTransactionFree);
AppLayerRegisterGetFilesFunc(ALPROTO_HTTP, HTPStateGetFiles);
AppLayerRegisterProto("http", ALPROTO_HTTP, STREAM_TOSERVER,
HTPHandleRequestData);

@ -34,6 +34,7 @@
#define __APP_LAYER_HTP_H__
#include "util-radix-tree.h"
#include "util-file.h"
#include <htp/htp.h>
@ -147,6 +148,7 @@ typedef struct HtpState_ {
uint16_t transaction_cnt;
uint16_t transaction_done;
uint32_t request_body_limit;
FileContainer *files;
} HtpState;
void RegisterHTPParsers(void);

@ -41,6 +41,7 @@
#include "stream.h"
#include "stream-tcp-reassemble.h"
#include "app-layer.h"
#include "app-layer-protos.h"
#include "app-layer-parser.h"
#include "app-layer-smb.h"
@ -71,6 +72,21 @@ static SCMutex al_result_pool_mutex = PTHREAD_MUTEX_INITIALIZER;
static uint32_t al_result_pool_elmts = 0;
#endif /* DEBUG */
/** \brief Get the file container flow
* \param f flow pointer to a LOCKED flow
* \retval files void pointer to the state
* \retval NULL in case we have no state */
FileContainer *AppLayerGetFilesFromFlow(Flow *f) {
uint16_t alproto = f->alproto;
if (alproto == ALPROTO_UNKNOWN)
return NULL;
if (al_proto_table[alproto].StateGetFiles != NULL)
return al_proto_table[alproto].StateGetFiles(AppLayerGetProtoStateFromFlow(f));
else
return NULL;
}
/** \brief Alloc a AppLayerParserResultElmt func for the pool */
static void *AlpResultElmtPoolAlloc(void *null)
@ -618,6 +634,12 @@ void *AppLayerGetProtocolParserLocalStorage(uint16_t proto)
return NULL;
}
void AppLayerRegisterGetFilesFunc(uint16_t proto,
FileContainer *(*StateGetFiles)(void *state))
{
al_proto_table[proto].StateGetFiles = StateGetFiles;
}
/** \brief Indicate to the app layer parser that a logger is active
* for this protocol.
*/

@ -24,6 +24,8 @@
#ifndef __APP_LAYER_PARSER_H__
#define __APP_LAYER_PARSER_H__
#include "util-file.h"
/** Mapping between local parser id's (e.g. HTTP_FIELD_REQUEST_URI) and
* the dynamically assigned (at registration) global parser id. */
typedef struct AppLayerLocalMap_ {
@ -50,6 +52,7 @@ typedef struct AppLayerProto_ {
void (*StateTransactionFree)(void *, uint16_t);
void *(*LocalStorageAlloc)(void);
void (*LocalStorageFree)(void *);
FileContainer *(*StateGetFiles)(void *);
} AppLayerProto;
@ -245,6 +248,8 @@ void AppLayerRegisterLocalStorageFunc(uint16_t proto,
void *(*LocalStorageAlloc)(void),
void (*LocalStorageFree)(void *));
void *AppLayerGetProtocolParserLocalStorage(uint16_t);
void AppLayerRegisterGetFilesFunc(uint16_t proto,
FileContainer *(*StateGetFile)(void *));
void AppLayerRegisterLogger(uint16_t proto);
uint16_t AppLayerGetProtoByName(const char *);
@ -269,8 +274,6 @@ int AppLayerTransactionGetBaseId(Flow *f);
int AppLayerTransactionGetInspectId(Flow *f);
uint16_t AppLayerTransactionGetAvailId(Flow *f);
uint16_t AppLayerGetStateVersion(Flow *f);
void AppLayerSetEOF(Flow *);
/* cleanup */
@ -279,4 +282,7 @@ void AppLayerFreeProbingParsers(AppLayerProbingParser *);
void AppLayerFreeProbingParsersInfo(AppLayerProbingParserInfo *);
void AppLayerPrintProbingParsers(AppLayerProbingParser *);
uint16_t AppLayerGetStateVersion(Flow *f);
FileContainer *AppLayerGetFilesFromFlow(Flow *);
#endif /* __APP_LAYER_PARSER_H__ */

@ -70,21 +70,20 @@
*
* \note flow is not locked at this time
*/
static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s) {
static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *f, Signature *s, FileContainer *ffc) {
SigMatch *sm = NULL;
int r = 0;
int match = 0;
SCLogDebug("file inspection...");
SCMutexLock(&f->files_m);
if (f->files != NULL) {
FlowFile *file = f->files->head;
if (ffc != NULL) {
File *file = ffc->head;
for (; file != NULL; file = file->next) {
SCLogDebug("file");
if (file->state == FLOWFILE_STATE_NONE) {
SCLogDebug("file state FLOWFILE_STATE_NONE");
if (file->state == FILE_STATE_NONE) {
SCLogDebug("file state FILE_STATE_NONE");
continue;
}
@ -143,7 +142,6 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flo
}
}
SCMutexUnlock(&f->files_m);
SCReturnInt(r);
}
@ -178,7 +176,7 @@ int DetectFileInspectHttp(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Flow *
/* inspect files for this transaction */
det_ctx->tx_id = (uint16_t)idx;
match = DetectFileInspect(tv, det_ctx, f, s);
match = DetectFileInspect(tv, det_ctx, f, s, htp_state->files);
if (match == 1) {
r = 1;
} else if (match == 2) {

@ -883,7 +883,7 @@ next_sig:
if (!(f->de_state->flags & DE_STATE_FILE_STORE_DISABLED)) {
if (DeStateStoreFilestoreSigsCantMatch(det_ctx->sgh, f->de_state, flags) == 1) {
SCLogDebug("disabling file storage for transaction");
FlowFileDisableStoringForTransaction(f, det_ctx->tx_id);
FileDisableStoringForTransaction(f, det_ctx->tx_id);
f->de_state->flags |= DE_STATE_FILE_STORE_DISABLED;
}
}

@ -89,7 +89,7 @@ int DetectFileextMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f,
int ret = 0;
DetectFileextData *fileext = (DetectFileextData *)m->ctx;
FlowFile *file = (FlowFile *)state;
File *file = (File *)state;
if (file->name == NULL)
SCReturnInt(0);

@ -82,7 +82,7 @@ void DetectFilemagicRegister(void) {
* \retval -1 error
* \retval 0 ok
*/
static int FilemagicLookup(FlowFile *file) {
static int FilemagicLookup(File *file) {
if (file == NULL || file->chunks_head == NULL) {
SCReturnInt(-1);
}
@ -94,7 +94,7 @@ static int FilemagicLookup(FlowFile *file) {
uint32_t size = 0;
if (buf != NULL) {
FlowFileData *ffd = file->chunks_head;
FileData *ffd = file->chunks_head;
for ( ; ffd != NULL; ffd = ffd->next) {
uint32_t copy_len = ffd->len;
@ -110,7 +110,7 @@ static int FilemagicLookup(FlowFile *file) {
}
/* file is done but smaller than FILEMAGIC_MIN_SIZE */
if (ffd->next == NULL && file->state >= FLOWFILE_STATE_CLOSED) {
if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) {
file->magic = MagicLookup(buf, size);
break;
}
@ -140,7 +140,7 @@ int DetectFilemagicMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f
int ret = 0;
DetectFilemagicData *filemagic = m->ctx;
FlowFile *file = (FlowFile *)state;
File *file = (File *)state;
if (file->txid < det_ctx->tx_id)
SCReturnInt(0);

@ -88,7 +88,7 @@ int DetectFilenameMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f,
int ret = 0;
DetectFilenameData *filename = m->ctx;
FlowFile *file = (FlowFile *)state;
File *file = (File *)state;
if (file->name == NULL)
SCReturnInt(0);

@ -83,8 +83,8 @@ int DetectFilestoreMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f
uint8_t flags, void *state, Signature *s, SigMatch *m)
{
SCEnter();
FlowFile *file = (FlowFile *)state;
FlowFileStore(file);
File *file = (File *)state;
FileStore(file);
SCReturnInt(1);
}

@ -1785,7 +1785,7 @@ end:
(p->flow->sgh_toclient == NULL ||
p->flow->sgh_toclient->filestore_cnt == 0))
{
FlowFileDisableStoring(p->flow);
FileDisableStoring(p->flow);
}
}
@ -3787,7 +3787,7 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) {
SigGroupHeadBuildHeadArray(de_ctx, sgh);
SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
SigGroupHeadSetFilestoreCount(de_ctx, sgh);
SCLogInfo("filestore count %u", sgh->filestore_cnt);
SCLogDebug("filestore count %u", sgh->filestore_cnt);
}
if (de_ctx->decoder_event_sgh != NULL) {

@ -27,506 +27,8 @@
#include "suricata.h"
#include "debug.h"
#include "flow.h"
#include "flow-file.h"
#include "util-hash.h"
#include "util-debug.h"
#include "util-memcmp.h"
#include "util-print.h"
/* prototypes */
static void FlowFileFree(FlowFile *);
int FlowFileMagicSize(void) {
return 512;
}
/**
* \brief check if we have stored enough
*
* \param ff file
*
* \retval 0 limit not reached yet
* \retval 1 limit reached
*/
static int FlowFileStoreNoStoreCheck(FlowFile *ff) {
SCEnter();
if (ff == NULL) {
SCReturnInt(0);
}
if (ff->store == -1) {
if (ff->state == FLOWFILE_STATE_OPENED &&
ff->size >= (uint64_t)FlowFileMagicSize())
{
SCReturnInt(1);
}
}
SCReturnInt(0);
}
/**
* \brief allocate a FlowFileContainer
*
* \retval new newly allocated FlowFileContainer
* \retval NULL error
*/
FlowFileContainer *FlowFileContainerAlloc(void) {
FlowFileContainer *new = SCMalloc(sizeof(FlowFileContainer));
if (new == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "Error allocating mem");
return NULL;
}
memset(new, 0, sizeof(FlowFileContainer));
new->head = new->tail = NULL;
return new;
}
/**
* \brief Recycle a FlowFileContainer
*
* \param ffc FlowFileContainer
*/
void FlowFileContainerRecycle(FlowFileContainer *ffc) {
if (ffc == NULL)
return;
FlowFile *cur = ffc->head;
FlowFile *next = NULL;
for (;cur != NULL; cur = next) {
next = cur->next;
FlowFileFree(cur);
}
ffc->head = ffc->tail = NULL;
}
/**
* \brief Free a FlowFileContainer
*
* \param ffc FlowFileContainer
*/
void FlowFileContainerFree(FlowFileContainer *ffc) {
if (ffc == NULL)
return;
FlowFile *ptr = ffc->head;
FlowFile *next = NULL;
for (;ptr != NULL; ptr = next) {
next = ptr->next;
FlowFileFree(ptr);
}
ffc->head = ffc->tail = NULL;
SCFree(ffc);
}
/**
* \internal
*
* \brief allocate a FlowFileData chunk and set it up
*
* \param data data chunk to store in the FlowFileData
* \param data_len lenght of the data
*
* \retval new FlowFileData object
*/
static FlowFileData *FlowFileDataAlloc(uint8_t *data, uint32_t data_len) {
FlowFileData *new = SCMalloc(sizeof(FlowFileData));
if (new == NULL) {
return NULL;
}
memset(new, 0, sizeof(FlowFileData));
new->data = SCMalloc(data_len);
if (new->data == NULL) {
SCFree(new);
return NULL;
}
new->len = data_len;
memcpy(new->data, data, data_len);
new->next = NULL;
return new;
}
/**
* \internal
*
* \brief free a FlowFileData object
*
* \param ffd the flow file data object to free
*/
static void FlowFileDataFree(FlowFileData *ffd) {
if (ffd == NULL)
return;
if (ffd->data != NULL) {
SCFree(ffd->data);
}
SCFree(ffd);
}
static int FlowFileAppendFlowFileDataFilePtr(FlowFile *ff, FlowFileData *ffd) {
SCEnter();
if (ff == NULL) {
SCReturnInt(-1);
}
if (ff->chunks_tail == NULL) {
ff->chunks_head = ffd;
ff->chunks_tail = ffd;
} else {
ff->chunks_tail->next = ffd;
ff->chunks_tail = ffd;
}
ff->size += ffd->len;
SCLogDebug("file size %"PRIu64, ff->size);
SCReturnInt(0);
}
static int FlowFileAppendFlowFileData(FlowFileContainer *ffc, FlowFileData *ffd) {
SCEnter();
if (ffc == NULL) {
SCReturnInt(-1);
}
if (FlowFileAppendFlowFileDataFilePtr(ffc->tail, ffd) == -1)
{
SCReturnInt(-1);
}
SCReturnInt(0);
}
/**
* \brief Alloc a new FlowFile
*
* \param name character array containing the name (not a string)
* \param name_len length in bytes of the name
*
* \retval new FlowFile object or NULL on error
*/
static FlowFile *FlowFileAlloc(uint8_t *name, uint16_t name_len) {
FlowFile *new = SCMalloc(sizeof(FlowFile));
if (new == NULL) {
SCLogError(SC_ERR_MEM_ALLOC, "Error allocating mem");
return NULL;
}
memset(new, 0, sizeof(FlowFile));
new->fd = -1;
new->name = SCMalloc(name_len);
if (new->name == NULL) {
SCFree(new);
return NULL;
}
new->name_len = name_len;
memcpy(new->name, name, name_len);
return new;
}
static void FlowFileFree(FlowFile *ff) {
if (ff == NULL)
return;
if (ff->fd != -1)
close(ff->fd);
if (ff->name != NULL)
SCFree(ff->name);
SCFree(ff);
}
void FlowFileContainerAdd(FlowFileContainer *ffc, FlowFile *ff) {
if (ffc->head == NULL) {
ffc->head = ffc->tail = ff;
} else {
ffc->tail->next = ff;
ffc->tail = ff;
}
}
/**
* \brief Open a new FlowFile
*
* \param ffc flow container
* \param name filename character array
* \param name_len filename len
* \param data initial data
* \param data_len initial data len
* \param flags open flags
*
* \retval ff flowfile object
*
* \note filename is not a string, so it's not nul terminated.
*/
FlowFile *FlowFileOpenFile(FlowFileContainer *ffc, uint8_t *name,
uint16_t name_len, uint8_t *data, uint32_t data_len, uint8_t flags)
{
SCEnter();
//PrintRawDataFp(stdout, name, name_len);
FlowFile *ff = FlowFileAlloc(name, name_len);
if (ff == NULL) {
SCReturnPtr(NULL, "FlowFile");
}
if (flags & FLOW_FILE_NOSTORE) {
ff->store = -1;
}
ff->state = FLOWFILE_STATE_OPENED;
SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_OPENED");
FlowFileContainerAdd(ffc, ff);
if (data != NULL) {
//PrintRawDataFp(stdout, data, data_len);
FlowFileData *ffd = FlowFileDataAlloc(data, data_len);
if (ffd == NULL) {
ff->state = FLOWFILE_STATE_ERROR;
SCReturnPtr(NULL, "FlowFile");
}
/* append the data */
if (FlowFileAppendFlowFileData(ffc, ffd) < 0) {
ff->state = FLOWFILE_STATE_ERROR;
FlowFileDataFree(ffd);
SCReturnPtr(NULL, "FlowFile");
}
}
SCReturnPtr(ff, "FlowFile");
}
static int FlowFileCloseFilePtr(FlowFile *ff, uint8_t *data,
uint32_t data_len, uint8_t flags)
{
SCEnter();
if (ff == NULL) {
SCReturnInt(-1);
}
if (ff->state != FLOWFILE_STATE_OPENED) {
SCReturnInt(-1);
}
if (data != NULL) {
//PrintRawDataFp(stdout, data, data_len);
FlowFileData *ffd = FlowFileDataAlloc(data, data_len);
if (ffd == NULL) {
ff->state = FLOWFILE_STATE_ERROR;
SCReturnInt(-1);
}
/* append the data */
if (FlowFileAppendFlowFileDataFilePtr(ff, ffd) < 0) {
ff->state = FLOWFILE_STATE_ERROR;
FlowFileDataFree(ffd);
SCReturnInt(-1);
}
}
if (flags & FLOW_FILE_TRUNCATED) {
ff->state = FLOWFILE_STATE_TRUNCATED;
SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_TRUNCATED");
if (flags & FLOW_FILE_NOSTORE) {
ff->store = -1;
}
} else {
ff->state = FLOWFILE_STATE_CLOSED;
SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_CLOSED");
}
SCReturnInt(0);
}
/**
* \brief Close a FlowFile
*
* \param ffc the container
* \param data final data if any
* \param data_len data len if any
* \param flags flags
*
* \retval 0 ok
* \retval -1 error
*/
int FlowFileCloseFile(FlowFileContainer *ffc, uint8_t *data,
uint32_t data_len, uint8_t flags)
{
SCEnter();
if (ffc == NULL || ffc->tail == NULL) {
SCReturnInt(-1);
}
if (FlowFileCloseFilePtr(ffc->tail, data, data_len, flags) == -1) {
SCReturnInt(-1);
}
SCReturnInt(0);
}
/**
* \brief Store a chunk of file data in the flow. The open "flowfile"
* will be used.
*
* \param ffc the container
* \param data data chunk
* \param data_len data chunk len
*
* \retval 0 ok
* \retval -1 error
* \retval -2 no store for this file
*/
int FlowFileAppendData(FlowFileContainer *ffc, uint8_t *data, uint32_t data_len) {
SCEnter();
if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
SCReturnInt(-1);
}
if (ffc->tail->state != FLOWFILE_STATE_OPENED) {
if (ffc->tail->store == -1) {
SCReturnInt(-2);
}
SCReturnInt(-1);
}
if (FlowFileStoreNoStoreCheck(ffc->tail) == 1) {
ffc->tail->state = FLOWFILE_STATE_CLOSED;
SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_CLOSED");
SCReturnInt(-2);
}
SCLogDebug("appending %"PRIu32" bytes", data_len);
FlowFileData *ffd = FlowFileDataAlloc(data, data_len);
if (ffd == NULL) {
ffc->tail->state = FLOWFILE_STATE_ERROR;
SCReturnInt(-1);
}
/* append the data */
if (FlowFileAppendFlowFileData(ffc, ffd) < 0) {
ffc->tail->state = FLOWFILE_STATE_ERROR;
FlowFileDataFree(ffd);
SCReturnInt(-1);
}
SCReturnInt(0);
}
/**
* \brief Tag a file for storing
*
* \param ff The file to store
*/
int FlowFileStore(FlowFile *ff) {
ff->store = 1;
SCReturnInt(0);
}
/**
* \brief Set the TX id for a file
*
* \param ff The file to store
* \param txid the tx id
*/
int FlowFileSetTx(FlowFile *ff, uint16_t txid) {
ff->txid = txid;
SCReturnInt(0);
}
/**
* \brief disable file storage for a flow
*
* \param f *LOCKED* flow
*/
void FlowFileDisableStoring(Flow *f) {
FlowFile *ptr = NULL;
SCEnter();
f->flags |= FLOW_FILE_NO_HANDLING;
SCMutexLock(&f->files_m);
if (f->files != NULL) {
for (ptr = f->files->head; ptr != NULL; ptr = ptr->next) {
if (ptr->state == FLOWFILE_STATE_OPENED) {
if (ptr->store == 0) {
ptr->store = -1;
}
// ptr->state = FLOWFILE_STATE_CLOSED;
// SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_CLOSED");
}
}
}
SCMutexUnlock(&f->files_m);
SCReturn;
}
/**
* \brief set no store flag, close file if needed
*
* \param ff file
*/
static void FlowFileDisableStoringForFile(FlowFile *ff) {
SCEnter();
if (ff == NULL) {
SCReturn;
}
ff->store = -1;
if (ff->state == FLOWFILE_STATE_OPENED && ff->size >= (uint64_t)FlowFileMagicSize()) {
(void)FlowFileCloseFilePtr(ff, NULL, 0,
(FLOW_FILE_TRUNCATED|FLOW_FILE_NOSTORE));
}
}
/**
* \brief disable file storing for files in a transaction
*
* \param f flow
* \param tx_id transaction id
*/
void FlowFileDisableStoringForTransaction(struct Flow_ *f, uint16_t tx_id) {
FlowFile *ptr = NULL;
SCEnter();
SCMutexLock(&f->files_m);
if (f->files != NULL) {
for (ptr = f->files->head; ptr != NULL; ptr = ptr->next) {
if (ptr->state == FLOWFILE_STATE_OPENED && ptr->txid == tx_id) {
if (ptr->store == 1) {
/* weird, already storing -- let it continue*/
SCLogDebug("file is already being stored");
} else {
FlowFileDisableStoringForFile(ptr);
}
}
}
}
SCMutexUnlock(&f->files_m);
SCReturn;
}

@ -28,125 +28,4 @@
#include "flow.h"
#include "util-hash.h"
#define FLOW_FILE_TRUNCATED 0x01
#define FLOW_FILE_NOSTORE 0x02
typedef enum FlowFileState_ {
FLOWFILE_STATE_NONE = 0, /**< no state */
FLOWFILE_STATE_OPENED, /**< flow file is opened */
FLOWFILE_STATE_CLOSED, /**< flow file is completed,
there will be no more data. */
FLOWFILE_STATE_TRUNCATED, /**< flow file is not complete, but
there will be no more data. */
FLOWFILE_STATE_STORED, /**< all fully written to disk */
FLOWFILE_STATE_ERROR, /**< file is in an error state */
FLOWFILE_STATE_MAX
} FlowFileState;
typedef struct FlowFileData_ {
uint8_t *data;
uint32_t len;
int stored; /* true if this chunk has been stored already
* false otherwise */
struct FlowFileData_ *next;
} FlowFileData;
typedef struct FlowFile_ {
int16_t store; /**< need storing? 0: no, 1: yes, -1: won't */
uint16_t txid; /**< tx this file is part of */
uint8_t *name;
uint16_t name_len;
int16_t state;
int fd; /**< file discriptor for storing files */
uint64_t size; /**< size tracked so far */
const char *magic;
FlowFileData *chunks_head;
FlowFileData *chunks_tail;
struct FlowFile_ *next;
} FlowFile;
typedef struct FlowFileContainer_ {
FlowFile *head;
FlowFile *tail;
} FlowFileContainer;
FlowFileContainer *FlowFileContainerAlloc();
void FlowFileContainerFree(FlowFileContainer *);
void FlowFileContainerRecycle(FlowFileContainer *);
void FlowFileContainerAdd(FlowFileContainer *, FlowFile *);
/**
* \brief Open a new FlowFile
*
* \param ffc flow container
* \param name filename character array
* \param name_len filename len
* \param data initial data
* \param data_len initial data len
* \param flags open flags
*
* \retval ff flowfile object
*
* \note filename is not a string, so it's not nul terminated.
*/
FlowFile *FlowFileOpenFile(FlowFileContainer *, uint8_t *name, uint16_t name_len,
uint8_t *data, uint32_t data_len, uint8_t flags);
/**
* \brief Close a FlowFile
*
* \param ffc the container
* \param data final data if any
* \param data_len data len if any
* \param flags flags
*
* \retval 0 ok
* \retval -1 error
*/
int FlowFileCloseFile(FlowFileContainer *, uint8_t *data, uint32_t data_len, uint8_t flags);
/**
* \brief Store a chunk of file data in the flow. The open "flowfile"
* will be used.
*
* \param ffc the container
* \param data data chunk
* \param data_len data chunk len
*
* \retval 0 ok
* \retval -1 error
*/
int FlowFileAppendData(FlowFileContainer *, uint8_t *data, uint32_t data_len);
/**
* \brief Tag a file for storing
*
* \param ff The file to store
*/
int FlowFileStore(FlowFile *);
/**
* \brief Set the TX id for a file
*
* \param ff The file to store
* \param txid the tx id
*/
int FlowFileSetTx(FlowFile *, uint16_t txid);
/**
* \brief disable file storage for a flow
*
* \param f *LOCKED* flow
*/
void FlowFileDisableStoring(struct Flow_ *);
/**
* \brief disable file storing for a transaction
*
* \param f flow
* \param tx_id transaction id
*/
void FlowFileDisableStoringForTransaction(struct Flow_ *, uint16_t);
#endif /* __FLOW_FILE_H__ */

@ -62,8 +62,6 @@
(f)->lnext = NULL; \
(f)->lprev = NULL; \
RESET_COUNTERS((f)); \
(f)->files = NULL; \
SCMutexInit(&(f)->files_m, NULL); \
} while (0)
/** \brief macro to recycle a flow before it goes into the spare queue for reuse.
@ -94,10 +92,6 @@
GenericVarFree((f)->flowvar); \
(f)->flowvar = NULL; \
RESET_COUNTERS((f)); \
SCMutexLock(&(f)->files_m); \
if ((f)->files != NULL) \
FlowFileContainerRecycle((f)->files); \
SCMutexUnlock(&(f)->files_m); \
} while(0)
#define FLOW_DESTROY(f) do { \
@ -112,13 +106,6 @@
GenericVarFree((f)->flowvar); \
SCMutexDestroy(&(f)->de_state_m); \
(f)->tag_list = NULL; \
SCMutexLock(&(f)->files_m); \
if ((f)->files != NULL) {\
FlowFileContainerFree((f)->files); \
SCMutexDestroy(&(f)->files_m); \
(f)->files = NULL; \
} \
SCMutexUnlock(&(f)->files_m); \
} while(0)
Flow *FlowAlloc(void);

@ -41,7 +41,6 @@
#include "flow-queue.h"
#include "flow-hash.h"
#include "flow-util.h"
#include "flow-file.h"
#include "flow-var.h"
#include "flow-private.h"
#include "flow-timeout.h"

@ -28,7 +28,6 @@
#include "util-var.h"
#include "util-atomic.h"
#include "detect-tag.h"
#include "flow-file.h"
#define FLOW_QUIET TRUE
#define FLOW_VERBOSE FALSE
@ -314,10 +313,6 @@ typedef struct Flow_
uint32_t tosrcpktcnt;
uint64_t bytecnt;
#endif
FlowFileContainer *files;
SCMutex files_m;
} Flow;
enum {

@ -33,6 +33,8 @@
#include "threads.h"
#include "app-layer-parser.h"
#include "util-print.h"
#include "util-unittest.h"
#include "util-privs.h"
@ -100,20 +102,20 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P
SCReturnInt(TM_ECODE_OK);
}
SCMutexLock(&p->flow->files_m);
SCMutexLock(&p->flow->m);
FlowFileContainer *ffc = p->flow->files;
FileContainer *ffc = AppLayerGetFilesFromFlow(p->flow);
if (ffc != NULL) {
FlowFile *ff;
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
if (ff->state == FLOWFILE_STATE_STORED)
if (ff->state == FILE_STATE_STORED)
continue;
if (ff->store != 1) {
continue;
}
FlowFileData *ffd;
FileData *ffd;
for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) {
if (ffd->stored == 1)
continue;
@ -176,12 +178,12 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P
continue;
}
if (ff->state == FLOWFILE_STATE_CLOSED ||
ff->state == FLOWFILE_STATE_TRUNCATED ||
ff->state == FLOWFILE_STATE_ERROR)
if (ff->state == FILE_STATE_CLOSED ||
ff->state == FILE_STATE_TRUNCATED ||
ff->state == FILE_STATE_ERROR)
{
if (ffd->next == NULL) {
ff->state = FLOWFILE_STATE_STORED;
ff->state = FILE_STATE_STORED;
close(ff->fd);
ff->fd = -1;
}
@ -191,7 +193,7 @@ TmEcode LogFileLogIPv4(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, P
}
}
}
SCMutexUnlock(&p->flow->files_m);
SCMutexUnlock(&p->flow->m);
SCReturnInt(TM_ECODE_OK);
}

Loading…
Cancel
Save