aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

file: switch to streaming buffer API

Browse Source 
Make the file storage use the streaming buffer API.

As the individual file chunks were not needed by themselves, this
approach uses a chunkless implementation.
pull/2091/head
Victor Julien 11 years ago
parent e836a750c8
commit e43ce0a9ec
17 changed files with 279 additions and 513 deletions
Show Stats Download Patch File Download Diff File

93
src/app-layer-htp-file.c
Unescape Escape View File

@ -83,6 +83,7 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
uint8_t flags = 0;
FileContainer *files = NULL;
FileContainer *files_opposite = NULL;
const StreamingBufferConfig *sbcfg = NULL;
SCLogDebug("data %p data_len %"PRIu32, data, data_len);
@ -120,6 +121,9 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TC)) {
flags |= FILE_NOSTORE;
}
sbcfg = &s->cfg->response.sbcfg;
} else {
if (s->files_ts == NULL) {
s->files_ts = FileContainerAlloc();
@ -149,6 +153,8 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
if (!(flags & FILE_STORE) && (s->f->flags & FLOW_FILE_NO_STORE_TS)) {
flags |= FILE_NOSTORE;
}
sbcfg = &s->cfg->request.sbcfg;
}
/* if the previous file is in the same txid, we reset the file part of the
@ -176,7 +182,7 @@ int HTPFileOpen(HtpState *s, const uint8_t *filename, uint16_t filename_len,
}
}
if (FileOpenFile(files, filename, filename_len,
if (FileOpenFile(files, sbcfg, filename, filename_len,
data, data_len, flags) == NULL)
{
retval = -1;
@ -629,11 +635,12 @@ static int HTPFileParserTest03(void)
goto end;
}
if (http_state->files_ts->head->chunks_head == NULL ||
http_state->files_ts->head->chunks_head->len != 11)
if (http_state->files_ts->head == NULL ||
FileSize(http_state->files_ts->head) != 11)
{
if (http_state->files_ts->head->chunks_head != NULL)
printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
if (http_state->files_ts->head != NULL)
printf("filedata len not 11 but %"PRIu64": ",
FileSize(http_state->files_ts->head));
goto end;
}
@ -882,31 +889,18 @@ static int HTPFileParserTest05(void)
if (http_state->files_ts->head->next != http_state->files_ts->tail)
goto end;
if (http_state->files_ts->head->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead: ",
http_state->files_ts->head->chunks_head->len);
PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len);
goto end;
}
if (memcmp("filecontent", http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len) != 0) {
if (StreamingBufferCompareRawData(http_state->files_ts->head->sb,
(uint8_t *)"filecontent", 11) != 1)
{
goto end;
}
if (http_state->files_ts->tail->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead: ",
http_state->files_ts->tail->chunks_head->len);
PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data,
http_state->files_ts->tail->chunks_head->len);
if (StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
(uint8_t *)"FILECONTENT", 11) != 1)
{
goto end;
}
if (memcmp("FILECONTENT", http_state->files_ts->tail->chunks_head->data,
http_state->files_ts->tail->chunks_head->len) != 0) {
goto end;
}
result = 1;
end:
if (alp_tctx != NULL)
@ -1007,31 +1001,18 @@ static int HTPFileParserTest06(void)
if (http_state->files_ts->head->next != http_state->files_ts->tail)
goto end;
if (http_state->files_ts->head->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead: ",
http_state->files_ts->head->chunks_head->len);
PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len);
goto end;
}
if (memcmp("filecontent", http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len) != 0) {
if (StreamingBufferCompareRawData(http_state->files_ts->head->sb,
(uint8_t *)"filecontent", 11) != 1)
{
goto end;
}
if (http_state->files_ts->tail->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead: ",
http_state->files_ts->tail->chunks_head->len);
PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data,
http_state->files_ts->tail->chunks_head->len);
if (StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
(uint8_t *)"FILECONTENT", 11) != 1)
{
goto end;
}
if (memcmp("FILECONTENT", http_state->files_ts->tail->chunks_head->data,
http_state->files_ts->tail->chunks_head->len) != 0) {
goto end;
}
result = 1;
end:
if (alp_tctx != NULL)
@ -1116,16 +1097,9 @@ static int HTPFileParserTest07(void)
goto end;
}
if (http_state->files_ts->head->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead: ",
http_state->files_ts->head->chunks_head->len);
PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len);
goto end;
}
if (memcmp("FILECONTENT", http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len) != 0) {
if (StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
(uint8_t *)"FILECONTENT", 11) != 1)
{
goto end;
}
@ -1594,16 +1568,9 @@ static int HTPFileParserTest11(void)
goto end;
}
if (http_state->files_ts->head->chunks_head->len != 11) {
printf("expected 11 but file is %u bytes instead: ",
http_state->files_ts->head->chunks_head->len);
PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len);
goto end;
}
if (memcmp("FILECONTENT", http_state->files_ts->head->chunks_head->data,
http_state->files_ts->head->chunks_head->len) != 0) {
if (StreamingBufferCompareRawData(http_state->files_ts->head->sb,
(uint8_t *)"FILECONTENT", 11) != 1)
{
goto end;
}

119
src/app-layer-smtp.c
Unescape Escape View File

@ -225,7 +225,7 @@ SCEnumCharMap smtp_reply_map[ ] = {
};
/* Create SMTP config structure */
SMTPConfig smtp_config = { 0, { 0, 0, 0, 0, 0 }, 0, 0, 0};
SMTPConfig smtp_config = { 0, { 0, 0, 0, 0, 0 }, 0, 0, 0, STREAMING_BUFFER_CONFIG_INITIALIZER};
static SMTPString *SMTPStringAlloc(void);
@ -319,6 +319,8 @@ static void SMTPConfigure(void) {
}
}
smtp_config.sbcfg.buf_size = content_limit ? content_limit : 256;
SCReturn;
}
@ -360,23 +362,21 @@ static void SMTPPruneFiles(FileContainer *files)
File *file = files->head;
while (file) {
SCLogDebug("file %p, file->chunks_head %p", file, file->chunks_head);
if (file->chunks_head) {
uint32_t window = smtp_config.content_inspect_window;
if (file->chunks_head->stream_offset == 0)
window = MAX(window, smtp_config.content_inspect_min_size);
uint64_t file_size = file->content_len_so_far;
uint64_t data_size = file_size - file->chunks_head->stream_offset;
SCLogDebug("window %"PRIu32", file_size %"PRIu64", data_size %"PRIu64,
window, file_size, data_size);
if (data_size > (window * 3)) {
uint64_t left_edge = file_size - window;
SCLogDebug("file->content_inspected now %"PRIu64, left_edge);
file->content_inspected = left_edge;
}
SCLogDebug("file %p", file);
uint32_t window = smtp_config.content_inspect_window;
if (file->sb->stream_offset == 0)
window = MAX(window, smtp_config.content_inspect_min_size);
uint64_t file_size = FileSize(file);
uint64_t data_size = file_size - file->sb->stream_offset;
SCLogDebug("window %"PRIu32", file_size %"PRIu64", data_size %"PRIu64,
window, file_size, data_size);
if (data_size > (window * 3)) {
uint64_t left_edge = left_edge = file_size - window;
SCLogDebug("file->content_inspected now %"PRIu64, left_edge);
file->content_inspected = left_edge;
}
file = file->next;
@ -451,7 +451,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
flags |= FILE_STORE;
}
if (FileOpenFile(files, (uint8_t *) entity->filename, entity->filename_len,
if (FileOpenFile(files, &smtp_config.sbcfg, (uint8_t *) entity->filename, entity->filename_len,
(uint8_t *) chunk, len, flags|FILE_USE_DETECT) == NULL) {
ret = MIME_DEC_ERR_DATA;
SCLogDebug("FileOpenFile() failed");
@ -1681,10 +1681,21 @@ void RegisterSMTPParsers(void)
#ifdef UNITTESTS
static void SMTPTestInitConfig(void)
{
MimeDecSetConfig(&smtp_config.mime_config);
smtp_config.content_limit = FILEDATA_CONTENT_LIMIT;
smtp_config.content_inspect_window = FILEDATA_CONTENT_INSPECT_WINDOW;
smtp_config.content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE;
smtp_config.sbcfg.buf_size = FILEDATA_CONTENT_INSPECT_WINDOW;
}
/*
* \test Test STARTTLS.
*/
int SMTPParserTest01(void)
static int SMTPParserTest01(void)
{
int result = 0;
Flow f;
@ -1760,6 +1771,7 @@ int SMTPParserTest01(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@ -1873,7 +1885,7 @@ end:
/**
* \test Test multiple DATA commands(full mail transactions).
*/
int SMTPParserTest02(void)
static int SMTPParserTest02(void)
{
int result = 0;
Flow f;
@ -2118,6 +2130,7 @@ int SMTPParserTest02(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@ -2657,7 +2670,7 @@ end:
/**
* \test Testing parsing pipelined commands.
*/
int SMTPParserTest03(void)
static int SMTPParserTest03(void)
{
int result = 0;
Flow f;
@ -2752,6 +2765,7 @@ int SMTPParserTest03(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@ -2860,7 +2874,7 @@ end:
/*
* \test Test smtp with just <LF> delimter instead of <CR><LF>.
*/
int SMTPParserTest04(void)
static int SMTPParserTest04(void)
{
int result = 0;
Flow f;
@ -2899,6 +2913,7 @@ int SMTPParserTest04(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@ -2952,7 +2967,7 @@ end:
/*
* \test Test STARTTLS fail.
*/
int SMTPParserTest05(void)
static int SMTPParserTest05(void)
{
int result = 0;
Flow f;
@ -3046,6 +3061,7 @@ int SMTPParserTest05(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@ -3193,7 +3209,7 @@ end:
/**
* \test Test multiple DATA commands(full mail transactions).
*/
int SMTPParserTest06(void)
static int SMTPParserTest06(void)
{
int result = 0;
Flow f;
@ -3342,6 +3358,7 @@ int SMTPParserTest06(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOCLIENT,
@ -3542,7 +3559,7 @@ end:
/*
* \test Test retrieving lines when frag'ed.
*/
int SMTPParserTest07(void)
static int SMTPParserTest07(void)
{
int result = 0;
Flow f;
@ -3580,6 +3597,7 @@ int SMTPParserTest07(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@ -3655,7 +3673,7 @@ end:
/*
* \test Test retrieving lines when frag'ed.
*/
int SMTPParserTest08(void)
static int SMTPParserTest08(void)
{
int result = 0;
Flow f;
@ -3693,6 +3711,7 @@ int SMTPParserTest08(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@ -3768,7 +3787,7 @@ end:
/*
* \test Test retrieving lines when frag'ed.
*/
int SMTPParserTest09(void)
static int SMTPParserTest09(void)
{
int result = 0;
Flow f;
@ -3806,6 +3825,7 @@ int SMTPParserTest09(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@ -3881,7 +3901,7 @@ end:
/*
* \test Test retrieving lines when frag'ed.
*/
int SMTPParserTest10(void)
static int SMTPParserTest10(void)
{
int result = 0;
Flow f;
@ -3919,6 +3939,7 @@ int SMTPParserTest10(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@ -3994,7 +4015,7 @@ end:
/*
* \test Test retrieving lines when frag'ed.
*/
int SMTPParserTest11(void)
static int SMTPParserTest11(void)
{
int result = 0;
Flow f;
@ -4026,6 +4047,7 @@ int SMTPParserTest11(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER,
@ -4079,7 +4101,7 @@ end:
return result;
}
int SMTPParserTest12(void)
static int SMTPParserTest12(void)
{
int result = 0;
Signature *s = NULL;
@ -4124,6 +4146,7 @@ int SMTPParserTest12(void)
f.alproto = ALPROTO_SMTP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
de_ctx = DetectEngineCtxInit();
if (de_ctx == NULL)
@ -4200,7 +4223,7 @@ end:
return result;
}
int SMTPParserTest13(void)
static int SMTPParserTest13(void)
{
int result = 0;
Signature *s = NULL;
@ -4263,6 +4286,7 @@ int SMTPParserTest13(void)
f.alproto = ALPROTO_SMTP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
de_ctx = DetectEngineCtxInit();
if (de_ctx == NULL)
@ -4361,7 +4385,7 @@ end:
/**
* \test Test DATA command w/MIME message.
*/
int SMTPParserTest14(void)
static int SMTPParserTest14(void)
{
int result = 0;
Flow f;
@ -4541,6 +4565,7 @@ int SMTPParserTest14(void)
f.proto = IPPROTO_TCP;
StreamTcpInitConfig(TRUE);
SMTPTestInitConfig();
SCMutexLock(&f.m);
/* Welcome reply */
@ -4788,8 +4813,8 @@ int SMTPParserTest14(void)
printf("smtp-mime file name is incorrect");
goto end;
}
if(file->size != filesize){
printf("smtp-mime file size %"PRIu64" is incorrect", file->size);
if (FileSize(file) != filesize){
printf("smtp-mime file size %"PRIu64" is incorrect", FileSize(file));
goto end;
}
static uint8_t org_binary[] = {
@ -4810,12 +4835,12 @@ int SMTPParserTest14(void)
0x00, 0x00, 0x00, 0x00, 0x5C, 0x5C, 0x36, 0x36,
0x2E, 0x39, 0x33, 0x2E, 0x36, 0x38, 0x2E, 0x36,
0x5C, 0x7A, 0x00, 0x00, 0x38,};
uint64_t z;
for (z=0; z < filesize; z++){
if(org_binary[z] != file->chunks_head->data[z]){
printf("smtp-mime file data incorrect\n");
goto end;
}
if (StreamingBufferCompareRawData(file->sb,
org_binary, sizeof(org_binary)) != 1)
{
printf("smtp-mime file data incorrect\n");
goto end;
}
}
@ -4883,7 +4908,7 @@ end:
return result;
}
int SMTPProcessDataChunkTest01(void){
static int SMTPProcessDataChunkTest01(void){
Flow f;
FLOW_INITIALIZE(&f);
f.flags = FLOW_FILE_NO_STORE_TS;
@ -4895,7 +4920,7 @@ int SMTPProcessDataChunkTest01(void){
}
int SMTPProcessDataChunkTest02(void){
static int SMTPProcessDataChunkTest02(void){
char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72,
0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E,
0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65,
@ -4953,7 +4978,7 @@ int SMTPProcessDataChunkTest02(void){
int SMTPProcessDataChunkTest03(void){
static int SMTPProcessDataChunkTest03(void){
char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72, };
char mimemsg2[] = {0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E, };
char mimemsg3[] = {0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65, };
@ -5007,7 +5032,7 @@ int SMTPProcessDataChunkTest03(void){
}
int SMTPProcessDataChunkTest04(void){
static int SMTPProcessDataChunkTest04(void){
char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72, };
char mimemsg2[] = {0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E, };
char mimemsg3[] = {0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65, };
@ -5047,7 +5072,7 @@ int SMTPProcessDataChunkTest04(void){
return ret == 0;
}
int SMTPProcessDataChunkTest05(void){
static int SMTPProcessDataChunkTest05(void){
char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72,
0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E,
0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65,
@ -5075,7 +5100,7 @@ int SMTPProcessDataChunkTest05(void){
FAIL_IF(file == NULL);
ret = SMTPProcessDataChunk((uint8_t *)mimemsg, sizeof(mimemsg), state);
FAIL_IF(ret != 0);
FAIL_IF(file->size != 106);
FAIL_IF((uint32_t)FileSize(file) != 106);
SMTPStateFree(smtp_state);
FLOW_DESTROY(&f);
PASS;

3
src/app-layer-smtp.h
Unescape Escape View File

@ -27,6 +27,7 @@
#include "decode-events.h"
#include "util-decode-mime.h"
#include "queue.h"
#include "util-streaming-buffer.h"
enum {
SMTP_DECODER_EVENT_INVALID_REPLY,
@ -94,6 +95,8 @@ typedef struct SMTPConfig {
uint32_t content_limit;
uint32_t content_inspect_min_size;
uint32_t content_inspect_window;
StreamingBufferConfig sbcfg;
} SMTPConfig;
typedef struct SMTPState_ {

6
src/detect-engine-file.c
Unescape Escape View File

@ -88,6 +88,7 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
File *file = ffc->head;
for (; file != NULL; file = file->next) {
SCLogDebug("file");
if (file->state == FILE_STATE_NONE) {
SCLogDebug("file state FILE_STATE_NONE");
continue;
@ -109,13 +110,14 @@ static int DetectFileInspect(ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
break;
}
if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file->chunks_head == NULL) {
uint64_t file_size = FileSize(file);
if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file_size == 0) {
SCLogDebug("sig needs file content, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
break;
}
if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file->chunks_head == NULL) {
if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) {
SCLogDebug("sig needs file content, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
break;

79
src/detect-engine-filedata-smtp.c
Unescape Escape View File

@ -81,7 +81,7 @@ static inline int SMTPCreateSpace(DetectEngineThreadCtx *det_ctx, uint16_t size)
return 0;
}
static uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
static const uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
DetectEngineCtx *de_ctx,
DetectEngineThreadCtx *det_ctx,
Flow *f, File *curr_file,
@ -91,10 +91,10 @@ static uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
{
SCEnter();
int index = 0;
uint8_t *buffer = NULL;
const uint8_t *buffer = NULL;
*buffer_len = 0;
*stream_start_offset = 0;
FileData *curr_chunk = NULL;
uint64_t file_size = FileSize(curr_file);
if (det_ctx->smtp_buffers_list_len == 0) {
if (SMTPCreateSpace(det_ctx, 1) < 0)
@ -129,23 +129,21 @@ static uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
SCLogDebug("smtp_config.content_limit %u, smtp_config.content_inspect_min_size %u",
smtp_config.content_limit, smtp_config.content_inspect_min_size);
SCLogDebug("file %p size %"PRIu64", state %d", curr_file, curr_file->content_len_so_far, curr_file->state);
SCLogDebug("file %p size %"PRIu64", state %d", curr_file, file_size, curr_file->state);
/* no new data */
if (curr_file->content_inspected == curr_file->content_len_so_far) {
if (curr_file->content_inspected == file_size) {
SCLogDebug("no new data");
goto end;
}
curr_chunk = curr_file->chunks_head;
if (curr_chunk == NULL) {
SCLogDebug("no data chunks to inspect for this transaction");
if (file_size == 0) {
SCLogDebug("no data to inspect for this transaction");
goto end;
}
if ((smtp_config.content_limit == 0 ||
curr_file->content_len_so_far < smtp_config.content_limit) &&
curr_file->content_len_so_far < smtp_config.content_inspect_min_size &&
if ((smtp_config.content_limit == 0 || file_size < smtp_config.content_limit) &&
file_size < smtp_config.content_inspect_min_size &&
!(flags & STREAM_EOF) && !(curr_file->state > FILE_STATE_OPENED)) {
SCLogDebug("we still haven't seen the entire content. "
"Let's defer content inspection till we see the "
@ -153,51 +151,16 @@ static uint8_t *DetectEngineSMTPGetBufferForTX(uint64_t tx_id,
goto end;
}
int first = 1;
curr_chunk = curr_file->chunks_head;
while (curr_chunk != NULL) {
/* see if we can filter out chunks */
if (curr_file->content_inspected > 0) {
if (curr_chunk->stream_offset < curr_file->content_inspected) {
if ((curr_file->content_inspected - curr_chunk->stream_offset) > smtp_config.content_inspect_window) {
curr_chunk = curr_chunk->next;
continue;
} else {
/* include this one */
}
} else {
/* include this one */
}
}
if (first) {
det_ctx->smtp[index].offset = curr_chunk->stream_offset;
first = 0;
}
StreamingBufferGetDataAtOffset(curr_file->sb,
&det_ctx->smtp[index].buffer, &det_ctx->smtp[index].buffer_len,
curr_file->content_inspected);
/* see if we need to grow the buffer */
if (det_ctx->smtp[index].buffer == NULL || (det_ctx->smtp[index].buffer_len + curr_chunk->len) > det_ctx->smtp[index].buffer_size) {
void *ptmp;
det_ctx->smtp[index].buffer_size += curr_chunk->len * 2;
if ((ptmp = SCRealloc(det_ctx->smtp[index].buffer, det_ctx->smtp[index].buffer_size)) == NULL) {
SCFree(det_ctx->smtp[index].buffer);
det_ctx->smtp[index].buffer = NULL;
det_ctx->smtp[index].buffer_size = 0;
det_ctx->smtp[index].buffer_len = 0;
goto end;
}
det_ctx->smtp[index].buffer = ptmp;
}
memcpy(det_ctx->smtp[index].buffer + det_ctx->smtp[index].buffer_len, curr_chunk->data, curr_chunk->len);
det_ctx->smtp[index].buffer_len += curr_chunk->len;
curr_chunk = curr_chunk->next;
}
det_ctx->smtp[index].offset = curr_file->content_inspected;
/* updat inspected tracker */
curr_file->content_inspected = curr_file->chunks_tail->stream_offset + curr_file->chunks_tail->len;
SCLogDebug("curr_file->content_inspected now %"PRIu64, curr_file->content_inspected);
curr_file->content_inspected = FileSize(curr_file);
SCLogDebug("content_inspected %u, offset %u", (uint)curr_file->content_inspected, (uint)det_ctx->smtp[index].offset);
buffer = det_ctx->smtp[index].buffer;
*buffer_len = det_ctx->smtp[index].buffer_len;
@ -221,7 +184,7 @@ int DetectEngineInspectSMTPFiledata(ThreadVars *tv,
int match = 0;
uint32_t buffer_len = 0;
uint32_t stream_start_offset = 0;
uint8_t *buffer = 0;
const uint8_t *buffer = 0;
if (ffc != NULL) {
File *file = ffc->head;
@ -240,7 +203,7 @@ int DetectEngineInspectSMTPFiledata(ThreadVars *tv,
det_ctx->inspection_recursion_counter = 0;
match = DetectEngineContentInspection(de_ctx, det_ctx, s, s->sm_lists[DETECT_SM_LIST_FILEDATA],
f,
buffer,
(uint8_t *)buffer,
buffer_len,
stream_start_offset,
DETECT_ENGINE_CONTENT_INSPECTION_MODE_FD_SMTP, NULL);
@ -309,7 +272,7 @@ int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx,
uint32_t cnt = 0;
uint32_t buffer_len = 0;
uint32_t stream_start_offset = 0;
uint8_t *buffer = NULL;
const uint8_t *buffer = NULL;
if (ffc != NULL) {
File *file = ffc->head;
@ -323,7 +286,7 @@ int DetectEngineRunSMTPMpm(DetectEngineCtx *de_ctx,
if (buffer_len == 0)
goto end;
cnt += SMTPFiledataPatternSearch(det_ctx, buffer, buffer_len, flags);
cnt += SMTPFiledataPatternSearch(det_ctx, (uint8_t *)buffer, buffer_len, flags);
}
}
end:
@ -410,7 +373,7 @@ static int DetectEngineSMTPFiledataTest01(void)
DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx);
SCMutexLock(&f.m);
int r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER, mimemsg, mimemsg_len);
int r = AppLayerParserParse(alp_tctx, &f, ALPROTO_SMTP, STREAM_TOSERVER|STREAM_START|STREAM_EOF, mimemsg, mimemsg_len);
if (r != 0) {
printf("AppLayerParse for smtp failed. Returned %d", r);
SCMutexUnlock(&f.m);

5
src/detect-engine.c
Unescape Escape View File

@ -1699,11 +1699,6 @@ void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
/* SMTP */
if (det_ctx->smtp != NULL) {
SCLogDebug("det_ctx smtp %u", det_ctx->smtp_buffers_size);
for (i = 0; i < det_ctx->smtp_buffers_size; i++) {
if (det_ctx->smtp[i].buffer != NULL)
SCFree(det_ctx->smtp[i].buffer);
SCLogDebug("det_ctx->smtp[i].buffer_size %u", det_ctx->smtp[i].buffer_size);
}
SCFree(det_ctx->smtp);
}

87
src/detect-filemagic.c
Unescape Escape View File

@ -90,40 +90,21 @@ void DetectFilemagicRegister(void)
*/
int FilemagicGlobalLookup(File *file)
{
if (file == NULL || file->chunks_head == NULL) {
if (file == NULL || FileSize(file) == 0) {
SCReturnInt(-1);
}
/* initial chunk already matching our requirement */
if (file->chunks_head->len >= FILEMAGIC_MIN_SIZE) {
file->magic = MagicGlobalLookup(file->chunks_head->data, FILEMAGIC_MIN_SIZE);
} else {
uint8_t *buf = SCMalloc(FILEMAGIC_MIN_SIZE);
uint32_t size = 0;
if (likely(buf != NULL)) {
FileData *ffd = file->chunks_head;
for ( ; ffd != NULL; ffd = ffd->next) {
uint32_t copy_len = ffd->len;
if (size + ffd->len > FILEMAGIC_MIN_SIZE)
copy_len = FILEMAGIC_MIN_SIZE - size;
memcpy(buf + size, ffd->data, copy_len);
size += copy_len;
if (size >= FILEMAGIC_MIN_SIZE) {
file->magic = MagicGlobalLookup(buf, size);
break;
}
/* file is done but smaller than FILEMAGIC_MIN_SIZE */
if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) {
file->magic = MagicGlobalLookup(buf, size);
break;
}
}
SCFree(buf);
const uint8_t *data = NULL;
uint32_t data_len = 0;
uint64_t offset = 0;
StreamingBufferGetData(file->sb,
&data, &data_len, &offset);
if (offset == 0) {
if (FileSize(file) >= FILEMAGIC_MIN_SIZE) {
file->magic = MagicGlobalLookup(data, data_len);
} else if (file->state >= FILE_STATE_CLOSED) {
file->magic = MagicGlobalLookup(data, data_len);
}
}
@ -140,43 +121,23 @@ int FilemagicGlobalLookup(File *file)
*/
int FilemagicThreadLookup(magic_t *ctx, File *file)
{
if (ctx == NULL || file == NULL || file->chunks_head == NULL) {
if (ctx == NULL || file == NULL || FileSize(file) == 0) {
SCReturnInt(-1);
}
/* initial chunk already matching our requirement */
if (file->chunks_head->len >= FILEMAGIC_MIN_SIZE) {
file->magic = MagicThreadLookup(ctx, file->chunks_head->data, FILEMAGIC_MIN_SIZE);
} else {
uint8_t *buf = SCMalloc(FILEMAGIC_MIN_SIZE);
uint32_t size = 0;
if (likely(buf != NULL)) {
FileData *ffd = file->chunks_head;
for ( ; ffd != NULL; ffd = ffd->next) {
uint32_t copy_len = ffd->len;
if (size + ffd->len > FILEMAGIC_MIN_SIZE)
copy_len = FILEMAGIC_MIN_SIZE - size;
memcpy(buf + size, ffd->data, copy_len);
size += copy_len;
if (size >= FILEMAGIC_MIN_SIZE) {
file->magic = MagicThreadLookup(ctx, buf, size);
break;
}
/* file is done but smaller than FILEMAGIC_MIN_SIZE */
if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) {
file->magic = MagicThreadLookup(ctx, buf, size);
break;
}
}
SCFree(buf);
const uint8_t *data = NULL;
uint32_t data_len = 0;
uint64_t offset = 0;
StreamingBufferGetData(file->sb,
&data, &data_len, &offset);
if (offset == 0) {
if (FileSize(file) >= FILEMAGIC_MIN_SIZE) {
file->magic = MagicThreadLookup(ctx, data, data_len);
} else if (file->state >= FILE_STATE_CLOSED) {
file->magic = MagicThreadLookup(ctx, data, data_len);
}
}
SCReturnInt(0);
}

14
src/detect-filesize.c
Unescape Escape View File

@ -94,30 +94,32 @@ static int DetectFilesizeMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, F
DetectFilesizeData *fsd = (DetectFilesizeData *)m->ctx;
int ret = 0;
SCLogDebug("file size %"PRIu64", check %"PRIu64, file->size, fsd->size1);
uint64_t file_size = FileSize(file);
SCLogDebug("file size %"PRIu64", check %"PRIu64, file_size, fsd->size1);
if (file->state == FILE_STATE_CLOSED) {
switch (fsd->mode) {
case DETECT_FILESIZE_EQ:
if (file->size == fsd->size1)
if (file_size == fsd->size1)
ret = 1;
break;
case DETECT_FILESIZE_LT:
if (file->size < fsd->size1)
if (file_size < fsd->size1)
ret = 1;
break;
case DETECT_FILESIZE_GT:
if (file->size > fsd->size1)
if (file_size > fsd->size1)
ret = 1;
break;
case DETECT_FILESIZE_RA:
if (file->size > fsd->size1 && file->size < fsd->size2)
if (file_size > fsd->size1 && file_size < fsd->size2)
ret = 1;
break;
}
/* truncated, error: only see if what we have meets the GT condition */
} else if (file->state > FILE_STATE_CLOSED) {
if (fsd->mode == DETECT_FILESIZE_GT && file->size > fsd->size1)
if (fsd->mode == DETECT_FILESIZE_GT && file_size > fsd->size1)
ret = 1;
}
SCReturnInt(ret);

2
src/detect.h
Unescape Escape View File

@ -704,7 +704,7 @@ typedef struct HttpReassembledBody_ {
} HttpReassembledBody;
typedef struct FiledataReassembledBody_ {
uint8_t *buffer;
const uint8_t *buffer;
uint32_t buffer_size; /**< size of the buffer itself */
uint32_t buffer_len; /**< data len in the buffer */
uint64_t offset; /**< data offset */

2
src/log-file.c
Unescape Escape View File

@ -303,7 +303,7 @@ static void LogFileWriteJsonRecord(LogFileLogThread *aft, const Packet *p, const
break;
}
fprintf(fp, "\"stored\": %s, ", ff->flags & FILE_STORED ? "true" : "false");
fprintf(fp, "\"size\": %"PRIu64" ", ff->size);
fprintf(fp, "\"size\": %"PRIu64" ", FileSize(ff));
fprintf(fp, "}\n");
fflush(fp);
SCMutexUnlock(&aft->file_ctx->fp_mutex);

11
src/log-filestore.c
Unescape Escape View File

@ -273,7 +273,7 @@ static void LogFilestoreLogCloseMetaFile(const File *ff)
fprintf(fp, "STATE: UNKNOWN\n");
break;
}
fprintf(fp, "SIZE: %"PRIu64"\n", ff->size);
fprintf(fp, "SIZE: %"PRIu64"\n", FileSize(ff));
fclose(fp);
} else {
@ -281,7 +281,8 @@ static void LogFilestoreLogCloseMetaFile(const File *ff)
}
}
static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, const FileData *ffd, uint8_t flags)
static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p,
const File *ff, const uint8_t *data, uint32_t data_len, uint8_t flags)
{
SCEnter();
LogFilestoreLogThread *aft = (LogFilestoreLogThread *)thread_data;
@ -302,7 +303,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
return 0;
}
SCLogDebug("ff %p, ffd %p", ff, ffd);
SCLogDebug("ff %p, data %p, data_len %u", ff, data, data_len);
snprintf(filename, sizeof(filename), "%s/file.%u",
g_logfile_base_dir, ff->file_id);
@ -319,7 +320,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
return -1;
}
/* we can get called with a NULL ffd when we need to close */
} else if (ffd != NULL) {
} else if (data != NULL) {
file_fd = open(filename, O_APPEND | O_NOFOLLOW | O_WRONLY);
if (file_fd == -1) {
SCLogDebug("failed to open file %s: %s", filename, strerror(errno));
@ -328,7 +329,7 @@ static int LogFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p
}
if (file_fd != -1) {
ssize_t r = write(file_fd, (const void *)ffd->data, (size_t)ffd->len);
ssize_t r = write(file_fd, (const void *)data, (size_t)data_len);
if (r == -1) {
SCLogDebug("write failed: %s", strerror(errno));
}

94
src/output-filedata.c
Unescape Escape View File

@ -92,7 +92,8 @@ int OutputRegisterFiledataLogger(const char *name, FiledataLogger LogFunc, Outpu
SC_ATOMIC_DECLARE(unsigned int, file_id);
static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
Packet *p, const File *ff, const FileData *ffd, uint8_t flags)
Packet *p, const File *ff,
const uint8_t *data, uint32_t data_len, uint8_t flags)
{
OutputFiledataLogger *logger = list;
OutputLoggerThreadStore *store = store_list;
@ -103,7 +104,7 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
SCLogDebug("logger %p", logger);
PACKET_PROFILING_TMM_START(p, logger->module_id);
logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, ffd, flags);
logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags);
PACKET_PROFILING_TMM_END(p, logger->module_id);
file_logged = 1;
@ -172,73 +173,50 @@ static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data, P
/* if we have no data chunks left to log, we should still
* close the logger(s) */
if (ff->chunks_head == NULL && (file_trunc || file_close)) {
CallLoggers(tv, store, p, ff, NULL, OUTPUT_FILEDATA_FLAG_CLOSE);
if (FileSize(ff) == ff->content_stored &&
(file_trunc || file_close)) {
CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE);
ff->flags |= FILE_STORED;
continue;
}
FileData *ffd;
for (ffd = ff->chunks_head; ffd != NULL; ffd = ffd->next) {
uint8_t flags = 0;
int file_logged = 0;
FileData *write_ffd = ffd;
/* store */
SCLogDebug("ffd %p", ffd);
/* special case: on stream end we may inform the
* loggers that the file is truncated. In this
* case we already logged the current ffd, which
* is the last in our list. */
if (ffd->stored == 1) {
if (!(file_close == 1 && ffd->next == NULL)) {
continue;
}
// call writer with NULL ffd, so it can 'close'
// so really a 'close' msg
write_ffd = NULL;
flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
SCLogDebug("OUTPUT_FILEDATA_FLAG_CLOSE set");
}
/* store */
/* if file_id == 0, this is the first store of this file */
if (ff->file_id == 0) {
/* new file */
ff->file_id = SC_ATOMIC_ADD(file_id, 1);
flags |= OUTPUT_FILEDATA_FLAG_OPEN;
} else {
/* existing file */
}
/* if file_id == 0, this is the first store of this file */
if (ff->file_id == 0) {
/* new file */
ff->file_id = SC_ATOMIC_ADD(file_id, 1);
flags |= OUTPUT_FILEDATA_FLAG_OPEN;
} else {
/* existing file */
}
/* if file needs to be closed or truncated, inform
* loggers */
if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
ff->state = FILE_STATE_TRUNCATED;
}
/* if file needs to be closed or truncated, inform
* loggers */
if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
ff->state = FILE_STATE_TRUNCATED;
SCLogDebug("ff->state = FILE_STATE_TRUNCATED set");
}
/* tell the logger we're closing up */
if (ff->state >= FILE_STATE_CLOSED)
flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
/* for the last data chunk we have, also tell the logger
* we're closing up */
if (ffd->next == NULL && ff->state >= FILE_STATE_CLOSED) {
flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
SCLogDebug("OUTPUT_FILEDATA_FLAG_CLOSE set");
}
/* do the actual logging */
const uint8_t *data = NULL;
uint32_t data_len = 0;
SCLogDebug("ff %p ffd %p flags %02x", ff, write_ffd, flags);
/* do the actual logging */
file_logged = CallLoggers(tv, store, p, ff, write_ffd, flags);
StreamingBufferGetDataAtOffset(ff->sb,
&data, &data_len,
ff->content_stored);
if (file_logged) {
ffd->stored = 1;
int file_logged = CallLoggers(tv, store, p, ff, data, data_len, flags);
if (file_logged) {
ff->content_stored += data_len;
/* all done */
if (flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
ff->flags |= FILE_STORED;
break;
}
/* all done */
if (flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
ff->flags |= FILE_STORED;
break;
}
}
}

2
src/output-filedata.h
Unescape Escape View File

@ -34,7 +34,7 @@
/** filedata logger function pointer type */
typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *,
const File *, const FileData *, uint8_t);
const File *, const uint8_t *, uint32_t, uint8_t);
/** packet logger condition function pointer type,
* must return true for packets that should be logged

2
src/output-json-file.c
Unescape Escape View File

@ -151,7 +151,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
if (ff->flags & FILE_STORED) {
json_object_set_new(fjs, "file_id", json_integer(ff->file_id));
}
json_object_set_new(fjs, "size", json_integer(ff->size));
json_object_set_new(fjs, "size", json_integer(FileSize(ff)));
json_object_set_new(fjs, "tx_id", json_integer(ff->txid));
/* originally just 'file', but due to bug 1127 naming it fileinfo */

238
src/util-file.c
Unescape Escape View File

@ -58,7 +58,6 @@ static int g_file_force_tracking = 0;
/* prototypes */
static void FileFree(File *);
static void FileDataFree(FileData *);
void FileForceFilestoreEnable(void)
{
@ -101,61 +100,27 @@ int FileMagicSize(void)
return 512;
}
static int FileAppendFileDataFilePtr(File *ff, FileData *ffd)
{
SCEnter();
if (ff == NULL) {
SCReturnInt(-1);
}
if (ff->chunks_tail == NULL) {
ff->chunks_head = ffd;
ff->chunks_tail = ffd;
ff->content_len_so_far = ffd->len;
} else {
ff->chunks_tail->next = ffd;
ff->chunks_tail = ffd;
ff->content_len_so_far += ffd->len;
}
#ifdef DEBUG
ff->chunks_cnt++;
if (ff->chunks_cnt > ff->chunks_cnt_max)
ff->chunks_cnt_max = ff->chunks_cnt;
#endif
#ifdef HAVE_NSS
if (ff->md5_ctx)
HASH_Update(ff->md5_ctx, ffd->data, ffd->len);
#endif
SCReturnInt(0);
}
static int FileAppendFileData(FileContainer *ffc, FileData *ffd)
/**
* \brief get the size of the file
*
* This doesn't reflect how much of the file we have in memory, just the
* total size tracked so far.
*/
uint64_t FileSize(const File *file)
{
SCEnter();
if (ffc == NULL) {
SCReturnInt(-1);
}
if (FileAppendFileDataFilePtr(ffc->tail, ffd) == -1)
{
SCReturnInt(-1);
if (file != NULL && file->sb != NULL) {
SCLogDebug("returning %"PRIu64,
file->sb->stream_offset + file->sb->buf_offset);
return file->sb->stream_offset + file->sb->buf_offset;
}
SCReturnInt(0);
SCLogDebug("returning 0 (default)");
return 0;
}
static int FilePruneFile(File *file)
{
SCEnter();
SCLogDebug("file %p, file->chunks_cnt %"PRIu64, file, file->chunks_cnt);
if (!(file->flags & FILE_NOMAGIC)) {
/* need magic but haven't set it yet, bail out */
if (file->magic == NULL)
@ -166,37 +131,20 @@ static int FilePruneFile(File *file)
SCLogDebug("file->flags & FILE_NOMAGIC == true");
}
/* okay, we now know we can prune */
FileData *fd = file->chunks_head;
while (fd != NULL) {
SCLogDebug("fd %p", fd);
if (file->flags & FILE_NOSTORE || fd->stored == 1) {
/* keep chunks in memory as long as we still need to
* inspect them or parts of them */
if (file->flags & FILE_USE_DETECT) {
uint64_t right_edge = fd->stream_offset + fd->len;
if (file->content_inspected < right_edge)
break;
}
file->chunks_head = fd->next;
if (file->chunks_tail == fd)
file->chunks_tail = fd->next;
uint64_t left_edge = file->content_stored;
if (file->flags & FILE_NOSTORE) {
left_edge = FileSize(file);
}
if (file->flags & FILE_USE_DETECT) {
left_edge = MIN(left_edge, file->content_inspected);
}
FileDataFree(fd);
if (left_edge) {
StreamingBufferSlideToOffset(file->sb, left_edge);
}
fd = file->chunks_head;
#ifdef DEBUG
file->chunks_cnt--;
SCLogDebug("file->chunks_cnt %"PRIu64, file->chunks_cnt);
#endif
} else if (fd->stored == 0) {
fd = NULL;
SCReturnInt(0);
break;
}
if (left_edge != FileSize(file)) {
SCReturnInt(0);
}
SCLogDebug("file->state %d. Is >= FILE_STATE_CLOSED: %s", file->state, (file->state >= FILE_STATE_CLOSED) ? "yes" : "no");
@ -293,56 +241,6 @@ void FileContainerFree(FileContainer *ffc)
SCFree(ffc);
}
/**
* \internal
*
* \brief allocate a FileData chunk and set it up
*
* \param data data chunk to store in the FileData
* \param data_len lenght of the data
*
* \retval new FileData object
*/
static FileData *FileDataAlloc(const uint8_t *data, uint32_t data_len)
{
FileData *new = SCMalloc(sizeof(FileData));
if (unlikely(new == NULL)) {
return NULL;
}
memset(new, 0, sizeof(FileData));
new->data = SCMalloc(data_len);
if (new->data == NULL) {
SCFree(new);
return NULL;
}
new->len = data_len;
memcpy(new->data, data, data_len);
new->next = NULL;
return new;
}
/**
* \internal
*
* \brief free a FileData object
*
* \param ffd the flow file data object to free
*/
static void FileDataFree(FileData *ffd)
{
if (ffd == NULL)
return;
if (ffd->data != NULL) {
SCFree(ffd->data);
}
SCFree(ffd);
}
/**
* \brief Alloc a new File
*
@ -384,22 +282,14 @@ static void FileFree(File *ff)
if (ff->magic != NULL)
SCFree(ff->magic);
if (ff->chunks_head != NULL) {
FileData *ffd = ff->chunks_head;
while (ffd != NULL) {
FileData *next_ffd = ffd->next;
FileDataFree(ffd);
ffd = next_ffd;
}
if (ff->sb != NULL) {
StreamingBufferFree(ff->sb);
}
#ifdef HAVE_NSS
if (ff->md5_ctx)
HASH_Destroy(ff->md5_ctx);
#endif
SCLogDebug("ff chunks_cnt %"PRIu64", chunks_cnt_max %"PRIu64,
ff->chunks_cnt, ff->chunks_cnt_max);
SCFree(ff);
}
@ -456,7 +346,7 @@ static int FileStoreNoStoreCheck(File *ff)
if (ff->flags & FILE_NOSTORE) {
if (ff->state == FILE_STATE_OPENED &&
ff->size >= (uint64_t)FileMagicSize())
FileSize(ff) >= (uint64_t)FileMagicSize())
{
SCReturnInt(1);
}
@ -465,6 +355,18 @@ static int FileStoreNoStoreCheck(File *ff)
SCReturnInt(0);
}
static int AppendData(File *file, const uint8_t *data, uint32_t data_len)
{
StreamingBufferAppendNoTrack(file->sb, data, data_len);
#ifdef HAVE_NSS
if (file->md5_ctx) {
HASH_Update(file->md5_ctx, data, data_len);
}
#endif
SCReturnInt(0);
}
/**
* \brief Store a chunk of file data in the flow. The open "flowfile"
* will be used.
@ -492,9 +394,6 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
SCReturnInt(-1);
}
ffc->tail->size += data_len;
SCLogDebug("file size is now %"PRIu64, ffc->tail->size);
if (FileStoreNoStoreCheck(ffc->tail) == 1) {
#ifdef HAVE_NSS
/* no storage but forced md5 */
@ -515,23 +414,11 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
SCLogDebug("appending %"PRIu32" bytes", data_len);
FileData *ffd = FileDataAlloc(data, data_len);
if (ffd == NULL) {
if (AppendData(ffc->tail, data, data_len) != 0) {
ffc->tail->state = FILE_STATE_ERROR;
SCReturnInt(-1);
}
if (ffc->tail->chunks_head == NULL)
ffd->stream_offset = 0;
else
ffd->stream_offset = ffc->tail->size;
/* append the data */
if (FileAppendFileData(ffc, ffd) < 0) {
ffc->tail->state = FILE_STATE_ERROR;
FileDataFree(ffd);
SCReturnInt(-1);
}
SCReturnInt(0);
}
@ -539,6 +426,7 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
* \brief Open a new File
*
* \param ffc flow container
* \param sbcfg buffer config
* \param name filename character array
* \param name_len filename len
* \param data initial data
@ -549,7 +437,8 @@ int FileAppendData(FileContainer *ffc, const uint8_t *data, uint32_t data_len)
*
* \note filename is not a string, so it's not nul terminated.
*/
File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
File *FileOpenFile(FileContainer *ffc, const StreamingBufferConfig *sbcfg,
const uint8_t *name, uint16_t name_len,
const uint8_t *data, uint32_t data_len, uint16_t flags)
{
SCEnter();
@ -561,6 +450,13 @@ File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
SCReturnPtr(NULL, "File");
}
ff->sb = StreamingBufferInit(sbcfg);
if (ff->sb == NULL) {
FileFree(ff);
SCReturnPtr(NULL, "File");
}
SCLogDebug("ff->sb %p", ff->sb);
if (flags & FILE_STORE || g_file_force_filestore) {
FileStore(ff);
} else if (flags & FILE_NOSTORE) {
@ -595,22 +491,11 @@ File *FileOpenFile(FileContainer *ffc, const uint8_t *name, uint16_t name_len,
FileContainerAdd(ffc, ff);
if (data != NULL) {
//PrintRawDataFp(stdout, data, data_len);
ff->size += data_len;
SCLogDebug("file size is now %"PRIu64, ff->size);
FileData *ffd = FileDataAlloc(data, data_len);
if (ffd == NULL) {
ff->state = FILE_STATE_ERROR;
SCReturnPtr(NULL, "File");
}
/* append the data */
if (FileAppendFileData(ffc, ffd) < 0) {
if (AppendData(ff, data, data_len) != 0) {
ff->state = FILE_STATE_ERROR;
FileDataFree(ffd);
SCReturnPtr(NULL, "File");
}
SCLogDebug("file size is now %"PRIu64, FileSize(ff));
}
SCReturnPtr(ff, "File");
@ -629,12 +514,7 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data,
SCReturnInt(-1);
}
ff->size += data_len;
SCLogDebug("file size is now %"PRIu64, ff->size);
if (data != NULL) {
//PrintRawDataFp(stdout, data, data_len);
if (ff->flags & FILE_NOSTORE) {
#ifdef HAVE_NSS
/* no storage but md5 */
@ -642,16 +522,8 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data,
HASH_Update(ff->md5_ctx, data, data_len);
#endif
} else {
FileData *ffd = FileDataAlloc(data, data_len);
if (ffd == NULL) {
ff->state = FILE_STATE_ERROR;
SCReturnInt(-1);
}
/* append the data */
if (FileAppendFileDataFilePtr(ff, ffd) < 0) {
if (AppendData(ff, data, data_len) != 0) {
ff->state = FILE_STATE_ERROR;
FileDataFree(ffd);
SCReturnInt(-1);
}
}
@ -858,7 +730,7 @@ void FileDisableStoringForFile(File *ff)
SCLogDebug("not storing this file");
ff->flags |= FILE_NOSTORE;
if (ff->state == FILE_STATE_OPENED && ff->size >= (uint64_t)FileMagicSize()) {
if (ff->state == FILE_STATE_OPENED && FileSize(ff) >= (uint64_t)FileMagicSize()) {
if (g_file_force_md5 == 0 && g_file_force_tracking == 0) {
(void)FileCloseFilePtr(ff, NULL, 0,
(FILE_TRUNCATED|FILE_NOSTORE));

33
src/util-file.h
Unescape Escape View File

@ -29,6 +29,8 @@
#include <sechash.h>
#endif
#include "util-streaming-buffer.h"
#define FILE_TRUNCATED 0x0001
#define FILE_NOMAGIC 0x0002
#define FILE_NOMD5 0x0004
@ -51,38 +53,23 @@ typedef enum FileState_ {
FILE_STATE_MAX
} FileState;
typedef struct FileData_ {
uint8_t *data;
uint32_t len;
uint64_t stream_offset;
int stored; /* true if this chunk has been stored already
* false otherwise */
struct FileData_ *next;
} FileData;
typedef struct File_ {
uint16_t flags;
uint16_t name_len;
int16_t state;
StreamingBuffer *sb;
uint64_t txid; /**< tx this file is part of */
uint32_t file_id;
uint8_t *name;
uint64_t size; /**< size tracked so far */
char *magic;
FileData *chunks_head;
FileData *chunks_tail;
struct File_ *next;
#ifdef HAVE_NSS
HASHContext *md5_ctx;
uint8_t md5[MD5_LENGTH];
#endif
#ifdef DEBUG
uint64_t chunks_cnt;
uint64_t chunks_cnt_max;
#endif
uint64_t content_len_so_far;
uint64_t content_inspected; /**< used in pruning if FILE_USE_DETECT
* flag is set */
uint64_t content_stored;
} File;
typedef struct FileContainer_ {
@ -101,6 +88,7 @@ void FileContainerAdd(FileContainer *, File *);
* \brief Open a new File
*
* \param ffc flow container
* \param sbcfg buffer config
* \param name filename character array
* \param name_len filename len
* \param data initial data
@ -110,9 +98,16 @@ void FileContainerAdd(FileContainer *, File *);
* \retval ff flowfile object
*
* \note filename is not a string, so it's not nul terminated.
*
* If flags contains the FILE_USE_DETECT bit, the pruning code will
* consider not just the content_stored tracker, but also content_inspected.
* It's the responsibility of the API user to make sure this tracker is
* properly updated.
*/
File *FileOpenFile(FileContainer *, const uint8_t *name, uint16_t name_len,
File *FileOpenFile(FileContainer *, const StreamingBufferConfig *,
const uint8_t *name, uint16_t name_len,
const uint8_t *data, uint32_t data_len, uint16_t flags);
/**
* \brief Close a File
*
@ -194,4 +189,6 @@ void FileStoreFileById(FileContainer *fc, uint32_t);
void FileTruncateAllOpenFiles(FileContainer *);
uint64_t FileSize(const File *file);
#endif /* __UTIL_FILE_H__ */

2
src/util-lua-common.c
Unescape Escape View File

@ -624,7 +624,7 @@ static int LuaCallbackFileInfoPushToStackFromFile(lua_State *luastate, const Fil
lua_pushnumber(luastate, file->file_id);
lua_pushnumber(luastate, file->txid);
lua_pushlstring(luastate, (char *)file->name, file->name_len);
lua_pushnumber(luastate, file->size);
lua_pushnumber(luastate, FileSize(file));
lua_pushstring (luastate, file->magic);
lua_pushstring(luastate, md5ptr);
return 6;

Write Preview
Loading…
Cancel
Save