From a0ee6ade3ecd43a20f4d9ebb1331cfc77220b08a Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 21 Apr 2011 12:50:25 +0200 Subject: [PATCH] Improve HTTP multipart parsing, add streaming parsing for files. --- src/Makefile.am | 2 + src/app-layer-htp-body.c | 262 +++++++++++++++++++ src/app-layer-htp-body.h | 36 +++ src/app-layer-htp-file.c | 480 +++++++++++++++++++++++++++++++++++ src/app-layer-htp-file.h | 30 +++ src/app-layer-htp.c | 525 ++++++++++++++++++--------------------- src/app-layer-htp.h | 40 +-- src/detect-fileext.c | 60 ++++- src/detect-filename.c | 54 +++- 9 files changed, 1170 insertions(+), 319 deletions(-) create mode 100644 src/app-layer-htp-body.c create mode 100644 src/app-layer-htp-body.h create mode 100644 src/app-layer-htp-file.c create mode 100644 src/app-layer-htp-file.h diff --git a/src/Makefile.am b/src/Makefile.am index d5f95167a9..0bcc425dd6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -250,6 +250,8 @@ app-layer-detect-proto.c app-layer-detect-proto.h \ app-layer-parser.c app-layer-parser.h \ app-layer-protos.c app-layer-protos.h \ app-layer-htp.c app-layer-htp.h \ +app-layer-htp-body.c app-layer-htp-body.h \ +app-layer-htp-file.c app-layer-htp-file.h \ app-layer-smb.c app-layer-smb.h \ app-layer-smb2.c app-layer-smb2.h \ app-layer-dcerpc.c app-layer-dcerpc.h \ diff --git a/src/app-layer-htp-body.c b/src/app-layer-htp-body.c new file mode 100644 index 0000000000..4ff1a3b227 --- /dev/null +++ b/src/app-layer-htp-body.c @@ -0,0 +1,262 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * \author Gurvinder Singh + * \author Pablo Rincon + * \author Brian Rectanus + * + * This file provides a HTTP protocol support for the engine using HTP library. + */ + +#include "suricata.h" +#include "suricata-common.h" +#include "debug.h" +#include "decode.h" +#include "threads.h" + +#include "util-print.h" +#include "util-pool.h" +#include "util-radix-tree.h" + +#include "stream-tcp-private.h" +#include "stream-tcp-reassemble.h" +#include "stream-tcp.h" +#include "stream.h" + +#include "app-layer-protos.h" +#include "app-layer-parser.h" +#include "app-layer-htp.h" + +#include "util-spm.h" +#include "util-debug.h" +#include "app-layer-htp.h" +#include "app-layer-htp-file.h" +#include "util-time.h" +#include + +#include "util-unittest.h" +#include "util-unittest-helper.h" +#include "flow-util.h" +#include "flow-file.h" + +#include "detect-engine.h" +#include "detect-engine-state.h" +#include "detect-parse.h" + +#include "conf.h" + +#include "util-memcmp.h" + +/** + * \brief Append a chunk of body to the HtpBody struct + * + * \param body pointer to the HtpBody holding the list + * \param data pointer to the data of the chunk + * \param len length of the chunk pointed by data + * + * \retval 0 ok + * \retval -1 error + */ +int HtpBodyAppendChunk(SCHtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32_t len) +{ + SCEnter(); + + HtpBodyChunk *bd = NULL; + + if (len == 0 || data == NULL) { + SCReturnInt(0); + } + + if (body->nchunks == 0) { + /* New chunk */ + bd = (HtpBodyChunk *)SCMalloc(sizeof(HtpBodyChunk)); + if (bd == NULL) + goto error; + + bd->len = len; + bd->stream_offset = 0; + bd->next = NULL; + bd->id = 0; + + bd->data = SCMalloc(len); + if (bd->data == NULL) { + goto error; + } + memcpy(bd->data, data, len); + + body->first = body->last = bd; + body->nchunks++; + + htud->content_len_so_far = len; + } else { + bd = (HtpBodyChunk *)SCMalloc(sizeof(HtpBodyChunk)); + if (bd == NULL) + goto error; + + bd->len = len; + bd->stream_offset = htud->content_len_so_far; + bd->next = NULL; + bd->id = body->nchunks + 1; + + bd->data = SCMalloc(len); + if (bd->data == NULL) { + goto error; + } + memcpy(bd->data, data, len); + + body->last->next = bd; + body->last = bd; + body->nchunks++; + + htud->content_len_so_far += len; + } + SCLogDebug("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"", body, + bd->id, bd->data, (uint32_t)bd->len); + + SCReturnInt(0); + +error: + if (bd != NULL) { + if (bd->data != NULL) { + SCFree(bd->data); + } + SCFree(bd->data); + } + SCReturnInt(-1); +} + +/** + * \brief Print the information and chunks of a Body + * \param body pointer to the HtpBody holding the list + * \retval none + */ +void HtpBodyPrint(HtpBody *body) +{ + if (SCLogDebugEnabled()||1) { + SCEnter(); + + if (body->nchunks == 0) + return; + + HtpBodyChunk *cur = NULL; + SCLogDebug("--- Start body chunks at %p ---", body); + printf("--- Start body chunks at %p ---\n", body); + for (cur = body->first; cur != NULL; cur = cur->next) { + SCLogDebug("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"\n", + body, cur->id, cur->data, (uint32_t)cur->len); + printf("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"\n", + body, cur->id, cur->data, (uint32_t)cur->len); + PrintRawDataFp(stdout, (uint8_t*)cur->data, cur->len); + } + SCLogDebug("--- End body chunks at %p ---", body); + } +} + +/** + * \brief Free the information held in the request body + * \param body pointer to the HtpBody holding the list + * \retval none + */ +void HtpBodyFree(HtpBody *body) +{ + SCEnter(); + + if (body->nchunks == 0) + return; + + SCLogDebug("Removing chunks of Body %p; Last Chunk id: %"PRIu32", data %p," + " len %"PRIu32, body, body->last->id, body->last->data, + (uint32_t)body->last->len); + body->nchunks = 0; + + HtpBodyChunk *cur = NULL; + HtpBodyChunk *prev = NULL; + + prev = body->first; + while (prev != NULL) { + cur = prev->next; + if (prev->data != NULL) + SCFree(prev->data); + SCFree(prev); + prev = cur; + } + body->first = body->last = NULL; + body->operation = HTP_BODY_NONE; +} + +/** + * \brief Free request body chunks that are already fully parsed. + * + * \param htud pointer to the SCHtpTxUserData holding the body + * + * \retval none + */ +void HtpBodyPrune(SCHtpTxUserData *htud) +{ + SCEnter(); + + HtpBody *body = &htud->body; + + if (body->nchunks == 0) { + SCReturn; + } + + if (htud->body_parsed == 0) { + SCReturn; + } + + SCLogDebug("Pruning chunks of Body %p; Last Chunk id: %"PRIu32", data %p," + " len %"PRIu32, body, body->last->id, body->last->data, + (uint32_t)body->last->len); + + HtpBodyChunk *cur = NULL; + + cur = body->first; + + while (cur != NULL) { + HtpBodyChunk *next = cur->next; + + SCLogDebug("cur->stream_offset %"PRIu64" + cur->len %u = %"PRIu64", " + "htud->body_parsed %"PRIu64, cur->stream_offset, cur->len, + cur->stream_offset + cur->len, htud->body_parsed); + + if ((cur->stream_offset + cur->len) >= htud->body_parsed) { + break; + } + + body->first = next; + if (body->last == cur) { + body->last = next; + } + + if (body->nchunks > 0) + body->nchunks--; + + if (cur->data != NULL) { + SCFree(cur->data); + } + SCFree(cur); + + cur = next; + } + + SCReturn; +} diff --git a/src/app-layer-htp-body.h b/src/app-layer-htp-body.h new file mode 100644 index 0000000000..bc450a17b9 --- /dev/null +++ b/src/app-layer-htp-body.h @@ -0,0 +1,36 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * \author Gurvinder Singh + * \author Pablo Rincon + * + * This file provides a HTTP protocol support for the engine using HTP library. + */ + +#ifndef __APP_LAYER_HTP_BODY_H__ +#define __APP_LAYER_HTP_BODY_H__ + +int HtpBodyAppendChunk(SCHtpTxUserData *, HtpBody *, uint8_t *, uint32_t); +void HtpBodyPrint(HtpBody *); +void HtpBodyFree(HtpBody *); +void HtpBodyPrune(SCHtpTxUserData *); + +#endif /* __APP_LAYER_HTP_BODY_H__ */ diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c new file mode 100644 index 0000000000..9a17033a72 --- /dev/null +++ b/src/app-layer-htp-file.c @@ -0,0 +1,480 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * This file provides HTTP protocol file handling support for the engine + * using HTP library. + */ + +#include "suricata.h" +#include "suricata-common.h" +#include "debug.h" +#include "decode.h" +#include "threads.h" + +#include "util-print.h" +#include "util-pool.h" +#include "util-radix-tree.h" + +#include "stream-tcp-private.h" +#include "stream-tcp-reassemble.h" +#include "stream-tcp.h" +#include "stream.h" + +#include "app-layer-protos.h" +#include "app-layer-parser.h" +#include "app-layer-htp.h" + +#include "util-spm.h" +#include "util-debug.h" +#include "app-layer-htp.h" +#include "util-time.h" +#include + +#include "util-unittest.h" +#include "util-unittest-helper.h" +#include "flow-util.h" +#include "flow-file.h" + +#include "detect-engine.h" +#include "detect-engine-state.h" +#include "detect-parse.h" + +#include "conf.h" + +#include "util-memcmp.h" + +#ifdef UNITTESTS +static int HTPFileParserTest01(void) { + int result = 0; + Flow f; + uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n" + "Host: www.server.lan\r\n" + "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n" + "Content-Length: 215\r\n" + "\r\n" + "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n"; + + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + uint8_t httpbuf2[] = "filecontent\r\n" + "-----------------------------277531038314945--"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + + TcpSession ssn; + HtpState *http_state = NULL; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + f.protoctx = (void *)&ssn; + f.src.family = AF_INET; + f.dst.family = AF_INET; + + StreamTcpInitConfig(TRUE); + FlowL7DataPtrInit(&f); + + SCLogDebug("\n>>>> processing chunk 1 <<<<\n"); + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 2 size %u <<<<\n", httplen2); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + http_state = f.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + htp_tx_t *tx = list_get(http_state->connp->conn->transactions, 0); + if (tx == NULL) { + goto end; + } + + if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0) + { + printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method)); + goto end; + } + + result = 1; +end: + FlowL7DataPtrFree(&f); + StreamTcpFreeConfig(TRUE); + if (http_state != NULL) + HTPStateFree(http_state); + return result; +} + +static int HTPFileParserTest02(void) { + int result = 0; + Flow f; + uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n" + "Host: www.server.lan\r\n" + "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n" + "Content-Length: 337\r\n" + "\r\n"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + + uint8_t httpbuf2[] = "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"email\"\r\n" + "\r\n" + "someaddress@somedomain.lan\r\n"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + + uint8_t httpbuf3[] = "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + + uint8_t httpbuf4[] = "filecontent\r\n" + "-----------------------------277531038314945--"; + uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ + + TcpSession ssn; + HtpState *http_state = NULL; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + f.protoctx = (void *)&ssn; + f.src.family = AF_INET; + f.dst.family = AF_INET; + + StreamTcpInitConfig(TRUE); + FlowL7DataPtrInit(&f); + + SCLogDebug("\n>>>> processing chunk 1 <<<<\n"); + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 2 size %u <<<<\n", httplen2); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 3 size %u <<<<\n", httplen3); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf3, httplen3); + if (r != 0) { + printf("toserver chunk 3 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 4 size %u <<<<\n", httplen4); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf4, httplen4); + if (r != 0) { + printf("toserver chunk 4 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + http_state = f.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + htp_tx_t *tx = list_get(http_state->connp->conn->transactions, 0); + if (tx == NULL) { + goto end; + } + + if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0) + { + printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method)); + goto end; + } + + result = 1; +end: + FlowL7DataPtrFree(&f); + StreamTcpFreeConfig(TRUE); + if (http_state != NULL) + HTPStateFree(http_state); + return result; +} + +static int HTPFileParserTest03(void) { + int result = 0; + Flow f; + uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n" + "Host: www.server.lan\r\n" + "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n" + "Content-Length: 337\r\n" + "\r\n"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + + uint8_t httpbuf2[] = "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"email\"\r\n" + "\r\n" + "someaddress@somedomain.lan\r\n"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + + uint8_t httpbuf3[] = "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + + uint8_t httpbuf4[] = "file"; + uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ + + uint8_t httpbuf5[] = "content\r\n"; + uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */ + + uint8_t httpbuf6[] = "-----------------------------277531038314945--"; + uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */ + + TcpSession ssn; + HtpState *http_state = NULL; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + f.protoctx = (void *)&ssn; + f.src.family = AF_INET; + f.dst.family = AF_INET; + + StreamTcpInitConfig(TRUE); + FlowL7DataPtrInit(&f); + + SCLogDebug("\n>>>> processing chunk 1 <<<<\n"); + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 2 size %u <<<<\n", httplen2); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 3 size %u <<<<\n", httplen3); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf3, httplen3); + if (r != 0) { + printf("toserver chunk 3 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 4 size %u <<<<\n", httplen4); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf4, httplen4); + if (r != 0) { + printf("toserver chunk 4 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 5 size %u <<<<\n", httplen5); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf5, httplen5); + if (r != 0) { + printf("toserver chunk 5 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 6 size %u <<<<\n", httplen6); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf6, httplen6); + if (r != 0) { + printf("toserver chunk 6 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + http_state = f.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + htp_tx_t *tx = list_get(http_state->connp->conn->transactions, 0); + if (tx == NULL) { + goto end; + } + + if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0) + { + printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method)); + goto end; + } + + result = 1; +end: + FlowL7DataPtrFree(&f); + StreamTcpFreeConfig(TRUE); + if (http_state != NULL) + HTPStateFree(http_state); + return result; +} + +static int HTPFileParserTest04(void) { + int result = 0; + Flow f; + uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n" + "Host: www.server.lan\r\n" + "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n" + "Content-Length: 373\r\n" + "\r\n"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + + uint8_t httpbuf2[] = "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"email\"\r\n" + "\r\n" + "someaddress@somedomain.lan\r\n"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + + uint8_t httpbuf3[] = "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n"; + uint32_t httplen3 = sizeof(httpbuf3) - 1; /* minus the \0 */ + + uint8_t httpbuf4[] = "file0123456789abcdefghijklmnopqrstuvwxyz"; + uint32_t httplen4 = sizeof(httpbuf4) - 1; /* minus the \0 */ + + uint8_t httpbuf5[] = "content\r\n"; + uint32_t httplen5 = sizeof(httpbuf5) - 1; /* minus the \0 */ + + uint8_t httpbuf6[] = "-----------------------------277531038314945--"; + uint32_t httplen6 = sizeof(httpbuf6) - 1; /* minus the \0 */ + + TcpSession ssn; + HtpState *http_state = NULL; + + memset(&f, 0, sizeof(f)); + memset(&ssn, 0, sizeof(ssn)); + f.protoctx = (void *)&ssn; + f.src.family = AF_INET; + f.dst.family = AF_INET; + + StreamTcpInitConfig(TRUE); + FlowL7DataPtrInit(&f); + + SCLogDebug("\n>>>> processing chunk 1 <<<<\n"); + int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 2 size %u <<<<\n", httplen2); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 3 size %u <<<<\n", httplen3); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf3, httplen3); + if (r != 0) { + printf("toserver chunk 3 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 4 size %u <<<<\n", httplen4); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf4, httplen4); + if (r != 0) { + printf("toserver chunk 4 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 5 size %u <<<<\n", httplen5); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf5, httplen5); + if (r != 0) { + printf("toserver chunk 5 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 6 size %u <<<<\n", httplen6); + r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf6, httplen6); + if (r != 0) { + printf("toserver chunk 6 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + http_state = f.aldata[AlpGetStateIdx(ALPROTO_HTTP)]; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + htp_tx_t *tx = list_get(http_state->connp->conn->transactions, 0); + if (tx == NULL) { + goto end; + } + + if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0) + { + printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method)); + goto end; + } + + result = 1; +end: + FlowL7DataPtrFree(&f); + StreamTcpFreeConfig(TRUE); + if (http_state != NULL) + HTPStateFree(http_state); + return result; +} +#endif /* UNITTESTS */ + +void HTPFileParserRegisterTests(void) { +#ifdef UNITTESTS + UtRegisterTest("HTPFileParserTest01", HTPFileParserTest01, 1); + UtRegisterTest("HTPFileParserTest02", HTPFileParserTest02, 1); + UtRegisterTest("HTPFileParserTest03", HTPFileParserTest03, 1); + UtRegisterTest("HTPFileParserTest04", HTPFileParserTest04, 1); +#endif /* UNITTESTS */ +} diff --git a/src/app-layer-htp-file.h b/src/app-layer-htp-file.h new file mode 100644 index 0000000000..a752057cd7 --- /dev/null +++ b/src/app-layer-htp-file.h @@ -0,0 +1,30 @@ +/* Copyright (C) 2007-2011 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + */ + +#ifndef __APP_LAYER_HTP_FILE_H__ +#define __APP_LAYER_HTP_FILE_H__ + +void HTPFileParserRegisterTests(void); + +#endif /* __APP_LAYER_HTP_FILE_H__ */ diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 655f66842e..68a0d5ab54 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2011 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -48,13 +48,14 @@ #include "app-layer-protos.h" #include "app-layer-parser.h" + #include "app-layer-htp.h" +#include "app-layer-htp-body.h" +#include "app-layer-htp-file.h" #include "util-spm.h" #include "util-debug.h" -#include "app-layer-htp.h" #include "util-time.h" -#include #include "util-unittest.h" #include "util-unittest-helper.h" @@ -543,76 +544,6 @@ static int HTPHandleResponseData(Flow *f, void *htp_state, SCReturnInt(ret); } -/** - * \brief Append a chunk of body to the HtpBody struct - * \param body pointer to the HtpBody holding the list - * \param data pointer to the data of the chunk - * \param len length of the chunk pointed by data - * \retval 0 ok - * \retval -1 error - */ -int HtpBodyAppendChunk(SCHtpTxUserData *htud, HtpBody *body, uint8_t *data, uint32_t len) -{ - SCEnter(); - - HtpBodyChunk *bd = NULL; - - if (len == 0 || data == NULL) { - SCReturnInt(0); - } - - if (body->nchunks == 0) { - /* New chunk */ - bd = (HtpBodyChunk *)SCMalloc(sizeof(HtpBodyChunk)); - if (bd == NULL) - goto error; - - bd->len = len; - bd->data = SCMalloc(len); - if (bd->data == NULL) { - goto error; - } - - memcpy(bd->data, data, len); - htud->content_len_so_far = len; - body->first = body->last = bd; - body->nchunks++; - bd->next = NULL; - bd->id = body->nchunks; - } else { - bd = (HtpBodyChunk *)SCMalloc(sizeof(HtpBodyChunk)); - if (bd == NULL) - goto error; - - bd->len = len; - bd->data = SCMalloc(len); - if (bd->data == NULL) { - goto error; - } - - memcpy(bd->data, data, len); - htud->content_len_so_far += len; - body->last->next = bd; - body->last = bd; - body->nchunks++; - bd->next = NULL; - bd->id = body->nchunks; - } - SCLogDebug("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"", body, - bd->id, bd->data, (uint32_t)bd->len); - - SCReturnInt(0); - -error: - if (bd != NULL) { - if (bd->data != NULL) { - SCFree(bd->data); - } - SCFree(bd); - } - SCReturnInt(-1); -} - /** * \brief get the highest loggable transaction id */ @@ -650,65 +581,6 @@ error: SCReturnInt(-1); } -/** - * \brief Print the information and chunks of a Body - * \param body pointer to the HtpBody holding the list - * \retval none - */ -void HtpBodyPrint(HtpBody *body) -{ - if (SCLogDebugEnabled()||1) { - SCEnter(); - - if (body->nchunks == 0) - return; - - HtpBodyChunk *cur = NULL; - SCLogDebug("--- Start body chunks at %p ---", body); - printf("--- Start body chunks at %p ---\n", body); - for (cur = body->first; cur != NULL; cur = cur->next) { - SCLogDebug("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"\n", - body, cur->id, cur->data, (uint32_t)cur->len); - printf("Body %p; Chunk id: %"PRIu32", data %p, len %"PRIu32"\n", - body, cur->id, cur->data, (uint32_t)cur->len); - PrintRawDataFp(stdout, (uint8_t*)cur->data, cur->len); - } - SCLogDebug("--- End body chunks at %p ---", body); - } -} - -/** - * \brief Free the information held in the request body - * \param body pointer to the HtpBody holding the list - * \retval none - */ -void HtpBodyFree(HtpBody *body) -{ - SCEnter(); - - if (body->nchunks == 0) - return; - - SCLogDebug("Removing chunks of Body %p; Last Chunk id: %"PRIu32", data %p," - " len %"PRIu32"\n", body, body->last->id, body->last->data, - (uint32_t)body->last->len); - body->nchunks = 0; - - HtpBodyChunk *cur = NULL; - HtpBodyChunk *prev = NULL; - - prev = body->first; - while (prev != NULL) { - cur = prev->next; - if (prev->data != NULL) - SCFree(prev->data); - SCFree(prev); - prev = cur; - } - body->first = body->last = NULL; - body->operation = HTP_BODY_NONE; -} - #ifdef HAVE_HTP_URI_NORMALIZE_HOOK /** * \brief Normalize the query part of the URI as if it's part of the URI. @@ -842,8 +714,9 @@ static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len, break; } - if (x >= len) - return 0; + if (x >= len) { + SCReturnInt(0); + } uint8_t *line = data+x; size_t line_len = len-x; @@ -897,14 +770,14 @@ static int HTTPParseContentTypeHeader(uint8_t *name, size_t name_len, #endif *retptr = value; *retlen = value_len; - return 1; + SCReturnInt(1); } } } } } - return 0; + SCReturnInt(0); } static int HTTPStoreFileNameType(Flow *f, uint8_t *filename, size_t filename_len, @@ -999,6 +872,10 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) { SCEnter(); + uint8_t *expected_boundary = NULL; + uint8_t *expected_boundary_end = NULL; + uint8_t *chunks_buffer = NULL; + HtpState *hstate = (HtpState *)d->tx->connp->user_data; if (hstate == NULL) { SCReturnInt(HOOK_ERROR); @@ -1007,7 +884,6 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) SCLogDebug("New response body data available at %p -> %p -> %p, bodylen " "%"PRIu32"", hstate, d, d->data, (uint32_t)d->len); - //PrintRawDataFp(stdout, d->data, d->len); SCHtpTxUserData *htud = (SCHtpTxUserData *) htp_tx_get_user_data(d->tx); if (htud == NULL) { htud = SCMalloc(sizeof(SCHtpTxUserData)); @@ -1018,12 +894,8 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) htud->body.operation = HTP_BODY_NONE; htp_header_t *cl = table_getc(d->tx->request_headers, "content-length"); - if (cl != NULL) { + if (cl != NULL) htud->content_len = htp_parse_content_length(cl->value); - SCLogDebug("content_len %"PRIu32, htud->content_len); - } else { - SCLogDebug("no content_len header"); - } htp_header_t *h = (htp_header_t *)table_getc(d->tx->request_headers, "Content-Type"); @@ -1040,14 +912,18 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) PrintRawDataFp(stdout, boundary, boundary_len); printf("BOUNDARY END: \n"); #endif - htud->boundary = SCMalloc(boundary_len); - if (htud->boundary == NULL) { + if (boundary_len < HTP_BOUNDARY_MAX) { + htud->boundary = SCMalloc(boundary_len); + if (htud->boundary == NULL) { + goto end; + } + htud->boundary_len = (uint8_t)boundary_len; + memcpy(htud->boundary, boundary, boundary_len); + + htud->flags |= HTP_BOUNDARY_SET; + } else { goto end; } - htud->boundary_len = boundary_len; - memcpy(htud->boundary, boundary, boundary_len); - - htud->flags |= HTP_BOUNDARY_SET; } } @@ -1057,7 +933,7 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) htud->body.operation = HTP_BODY_REQUEST; - SCLogDebug("htud->content_len_so_far %u", htud->content_len_so_far); + SCLogDebug("htud->content_len_so_far %"PRIu64, htud->content_len_so_far); SCLogDebug("hstate->request_body_limit %u", hstate->request_body_limit); /* within limits, add the body chunk to the state. */ @@ -1071,10 +947,9 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) len = hstate->request_body_limit - htud->content_len_so_far; BUG_ON(len > (uint32_t)d->len); } - SCLogDebug("len %u", len); - int r = HtpBodyAppendChunk(htud, &htud->body, (uint8_t*)d->data, len); + int r = HtpBodyAppendChunk(htud, &htud->body, (uint8_t *)d->data, len); if (r < 0) { htud->flags |= HTP_BODY_COMPLETE; } else if (hstate->request_body_limit > 0 && @@ -1085,159 +960,243 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) htud->flags |= HTP_BODY_COMPLETE; } - if (htud->flags & HTP_BOUNDARY_SET) { - /* Checkout the boundaries */ - if (!(htud->flags & HTP_BOUNDARY_OPEN)) { - HtpBodyChunk *cur = htud->body.first; - uint8_t *chunks_buffer = NULL; - int32_t chunks_buffer_len = 0; - //TODO: Now that we are concatenating chunks here, free the list - //TODO: of chunks and append only 1 big chunk - while (cur != NULL) { - chunks_buffer_len += cur->len; - if ((chunks_buffer = SCRealloc(chunks_buffer, chunks_buffer_len)) == NULL) { - goto end; - } + if (!(htud->flags & HTP_BOUNDARY_SET)) { + goto end; + } - memcpy(chunks_buffer + chunks_buffer_len - cur->len, cur->data, cur->len); - cur = cur->next; + /* Checkout the boundaries */ + HtpBodyChunk *cur = htud->body.first; + uint32_t chunks_buffer_len = 0; + + for ( ; cur != NULL; cur = cur->next) { + /* skip body chunks entirely before what we parsed already */ + if (cur->stream_offset + cur->len <= htud->body_parsed) + continue; + + if (cur->stream_offset < htud->body_parsed && + cur->stream_offset + cur->len >= htud->body_parsed) { + + uint32_t toff = htud->body_parsed - cur->stream_offset; + uint32_t tlen = (cur->stream_offset + cur->len) - htud->body_parsed; + + chunks_buffer_len += tlen; + if ((chunks_buffer = SCRealloc(chunks_buffer, chunks_buffer_len)) == NULL) { + goto end; } + memcpy(chunks_buffer + chunks_buffer_len - tlen, cur->data + toff, tlen); - if (chunks_buffer != NULL) { - //PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len); + } else { + chunks_buffer_len += cur->len; + if ((chunks_buffer = SCRealloc(chunks_buffer, chunks_buffer_len)) == NULL) { + goto end; + } + memcpy(chunks_buffer + chunks_buffer_len - cur->len, cur->data, cur->len); + } + } - size_t expected_boundary_len = htud->boundary_len + 2; - uint8_t *expected_boundary = (uint8_t *)SCMalloc(expected_boundary_len); - if (expected_boundary == NULL) { - goto end; - } - memset(expected_boundary, '-', expected_boundary_len); - memcpy(expected_boundary + 2, htud->boundary, htud->boundary_len); + if (chunks_buffer == NULL) { + goto end; + } - size_t expected_boundary_end_len = htud->boundary_len + 4; - uint8_t *expected_boundary_end = (uint8_t *)SCMalloc(expected_boundary_end_len); - if (expected_boundary_end == NULL) { - goto end; - } - memset(expected_boundary_end, '-', expected_boundary_end_len); - memcpy(expected_boundary_end + 2, htud->boundary, htud->boundary_len); - - uint8_t *filename = NULL; - size_t filename_len = 0; - uint8_t *filetype = NULL; - size_t filetype_len = 0; - - uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len, - expected_boundary, expected_boundary_len); - uint8_t *header_end = Bs2bmSearch(chunks_buffer, chunks_buffer_len, - (uint8_t *)"\r\n\r\n", 4); - uint8_t *form_end = Bs2bmSearch(chunks_buffer, chunks_buffer_len, - expected_boundary_end, expected_boundary_end_len); - uint8_t *header = NULL; - - while (header_start != NULL && header_end != NULL && - (header_end != form_end) && - header_start < chunks_buffer + chunks_buffer_len && - header_end < chunks_buffer + chunks_buffer_len && header_start < header_end) - { - int header_len = header_end - header_start; - SCLogDebug("header_len %d", header_len); + //#if 0 + printf("CHUNK START: \n"); + PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len); + printf("CHUNK END: \n"); + //#endif - header = header_start + (expected_boundary_len + 2); // + for 0d 0a - header_len -= (expected_boundary_len + 2); -#if 0 - printf("HEADER START: \n"); - PrintRawDataFp(stdout, header, header_len); - printf("HEADER END: \n"); -#endif - while (header_len > 0) { - uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2); - uint8_t *line = header; - size_t line_len; - if (next_line == NULL) { - line_len = header_len; - } else { - line_len = next_line - header; - } -#if 0 - printf("LINE START: \n"); - PrintRawDataFp(stdout, line, line_len); - printf("LINE END: \n"); -#endif - SCLogDebug("line len %"PRIuMAX, (uintmax_t)line_len); - if (line_len >= C_D_HDR_LEN && - SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) - { - uint8_t *value = line + C_D_HDR_LEN; - size_t value_len = line_len - C_D_HDR_LEN; + uint8_t expected_boundary_len = htud->boundary_len + 2; + expected_boundary = (uint8_t *)SCMalloc(expected_boundary_len); + if (expected_boundary == NULL) { + goto end; + } + memset(expected_boundary, '-', expected_boundary_len); + memcpy(expected_boundary + 2, htud->boundary, htud->boundary_len); - /* parse content-disposition */ - int r = HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9, - value, value_len, &filename, &filename_len); - if (r == 1) { -#if 0 - printf("FILENAME START: \n"); - PrintRawDataFp(stdout, filename, filename_len); - printf("FILENAME END: \n"); -#endif - } - } else if (line_len >= C_T_HDR_LEN && - SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) - { - SCLogDebug("content-type line"); - uint8_t *value = line + C_T_HDR_LEN; - size_t value_len = line_len - C_T_HDR_LEN; + uint8_t expected_boundary_end_len = htud->boundary_len + 4; + expected_boundary_end = (uint8_t *)SCMalloc(expected_boundary_end_len); + if (expected_boundary_end == NULL) { + goto end; + } + memset(expected_boundary_end, '-', expected_boundary_end_len); + memcpy(expected_boundary_end + 2, htud->boundary, htud->boundary_len); + + uint8_t *header_start = Bs2bmSearch(chunks_buffer, chunks_buffer_len, + expected_boundary, expected_boundary_len); + uint8_t *header_end = NULL; + if (header_start != NULL) { + header_end = Bs2bmSearch(header_start, chunks_buffer_len - (header_start - chunks_buffer), + (uint8_t *)"\r\n\r\n", 4); + } + uint8_t *form_end = Bs2bmSearch(chunks_buffer, chunks_buffer_len, + expected_boundary_end, expected_boundary_end_len); + + /* if we're in the file storage process, deal with that now */ + if (htud->flags & HTP_FILENAME_SET) { + if (header_start != NULL || form_end != NULL || htud->flags & HTP_BODY_COMPLETE) { + SCLogDebug("reached the end of the file"); + + uint8_t *filedata = chunks_buffer; + uint32_t filedata_len = 0; + + if (header_start < form_end) { + filedata_len = header_start - filedata; + } else if (form_end < header_start) { + filedata_len = form_end - filedata; + } else if (form_end != NULL && form_end == header_start) { + filedata_len = form_end - filedata; + } else if (htud->flags & HTP_BODY_COMPLETE) { + filedata_len = chunks_buffer_len; + } - int r = HTTPParseContentTypeHeader(NULL, 0, - value, value_len, &filetype, &filetype_len); - if (r == 1) { -#if 0 - printf("FILETYPE START: \n"); - PrintRawDataFp(stdout, filetype, filetype_len); - printf("FILETYPE END: \n"); -#endif - } - } + printf("FILEDATA (final chunk) START: \n"); + PrintRawDataFp(stdout, filedata, filedata_len); + printf("FILEDATA (final chunk) END: \n"); - if (next_line == NULL) - break; + htud->flags &=~ HTP_FILENAME_SET; - header_len -= ((next_line + 2) - header); - header = next_line + 2; - } + /* fall through */ + } else { + SCLogDebug("not yet at the end of the file"); - if (filename != NULL) { - HTTPStoreFileNameType(hstate->f, filename, filename_len, - filetype, filetype_len); - } + if (chunks_buffer_len > expected_boundary_end_len) { + uint8_t *filedata = chunks_buffer; + uint32_t filedata_len = chunks_buffer_len - expected_boundary_len; - filename = NULL; - filetype = NULL; + printf("FILEDATA (part) START: \n"); + PrintRawDataFp(stdout, filedata, filedata_len); + printf("FILEDATA (part) END: \n"); - /* Search next boundary entry after the start of body */ - if ( (form_end == NULL && header_end + 4 < chunks_buffer + chunks_buffer_len) || (form_end != NULL && header_end != NULL && header_end + 4 < form_end)) { - uint32_t cursizeread = header_end - chunks_buffer; - header_start = Bs2bmSearch(header_end + 4, chunks_buffer_len - (cursizeread + 4), (uint8_t *) expected_boundary, expected_boundary_len); - header_end = Bs2bmSearch(header_end + 4, chunks_buffer_len - (cursizeread + 4), (uint8_t *) "\r\n\r\n", strlen("\r\n\r\n")); - } - } + htud->body_parsed += filedata_len; + } else { + SCLogDebug("chunk too small to already process in part"); + } + + goto end; + } + } + + while (header_start != NULL && header_end != NULL && + header_end != form_end && + header_start < (chunks_buffer + chunks_buffer_len) && + header_end < (chunks_buffer + chunks_buffer_len) && + header_start < header_end) + { + uint8_t *filename = NULL; + size_t filename_len = 0; + uint8_t *filetype = NULL; + size_t filetype_len = 0; + + uint32_t header_len = header_end - header_start; + SCLogDebug("header_len %u", header_len); + + uint8_t *header = header_start + (expected_boundary_len + 2); // + for 0d 0a + header_len -= (expected_boundary_len + 2); + //#if 0 + printf("HEADER START: \n"); + PrintRawDataFp(stdout, header, header_len); + printf("HEADER END: \n"); + //#endif + while (header_len > 0) { + uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2); + uint8_t *line = header; + uint32_t line_len; + + if (next_line == NULL) { + line_len = header_len; + } else { + line_len = next_line - header; + } + + if (line_len >= C_D_HDR_LEN && + SCMemcmpLowercase(C_D_HDR, line, C_D_HDR_LEN) == 0) { + uint8_t *value = line + C_D_HDR_LEN; + uint32_t value_len = line_len - C_D_HDR_LEN; + + /* parse content-disposition */ + (void)HTTPParseContentDispositionHeader((uint8_t *)"filename=", 9, + value, value_len, &filename, &filename_len); + } else if (line_len >= C_T_HDR_LEN && + SCMemcmpLowercase(C_T_HDR, line, C_T_HDR_LEN) == 0) { + SCLogDebug("content-type line"); + uint8_t *value = line + C_T_HDR_LEN; + uint32_t value_len = line_len - C_T_HDR_LEN; + + (void)HTTPParseContentTypeHeader(NULL, 0, + value, value_len, &filetype, &filetype_len); + } + + if (next_line == NULL) { + SCLogDebug("no next_line"); + break; + } + + header_len -= ((next_line + 2) - header); + header = next_line + 2; + } /* while (header_len > 0) */ + + if (filename != NULL) { + SCLogDebug("we have a filename"); + + HTTPStoreFileNameType(hstate->f, filename, filename_len, + filetype, filetype_len); + htud->flags |= HTP_FILENAME_SET; + + /* everything until the final boundary is the file */ + if (form_end != NULL) { + uint8_t *filedata = header_end + 4; + uint32_t filedata_len = form_end - (header_end + 4 + 2); + SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len); + + //#if 0 + printf("FILEDATA START: \n"); + PrintRawDataFp(stdout, filedata, filedata_len); + printf("FILEDATA END: \n"); + //#endif + } else { + SCLogDebug("more file data to come"); - SCFree(expected_boundary); - SCFree(expected_boundary_end); - SCFree(chunks_buffer); + uint32_t offset = (header_end + 4) - chunks_buffer; + SCLogDebug("offset %u", offset); + htud->body_parsed = offset; } } + + filename = NULL; + filetype = NULL; + + SCLogDebug("header_start %p, header_end %p, form_end %p", + header_start, header_end, form_end); + + /* Search next boundary entry after the start of body */ + uint32_t cursizeread = header_end - chunks_buffer; + header_start = Bs2bmSearch(header_end + 4, + chunks_buffer_len - (cursizeread + 4), + expected_boundary, expected_boundary_len); + if (header_start != NULL) { + header_end = Bs2bmSearch(header_end + 4, + chunks_buffer_len - (cursizeread + 4), + (uint8_t *) "\r\n\r\n", 4); + } } + } + end: - /* set the new chunk flag */ - hstate->flags |= HTP_FLAG_NEW_BODY_SET; + HtpBodyPrune(htud); - //if (SCLogDebugEnabled()) { - //HtpBodyPrint(&htud->body); - //} + if (expected_boundary != NULL) { + SCFree(expected_boundary); + } + if (expected_boundary_end != NULL) { + SCFree(expected_boundary_end); + } + if (chunks_buffer != NULL) { + SCFree(chunks_buffer); } + /* set the new chunk flag */ + hstate->flags |= HTP_FLAG_NEW_BODY_SET; + SCReturnInt(HOOK_OK); } @@ -2819,6 +2778,8 @@ void HTPParserRegisterTests(void) { UtRegisterTest("HTPParserConfigTest01", HTPParserConfigTest01, 1); UtRegisterTest("HTPParserConfigTest02", HTPParserConfigTest02, 1); UtRegisterTest("HTPParserConfigTest03", HTPParserConfigTest03, 1); + + HTPFileParserRegisterTests(); #endif /* UNITTESTS */ } diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index 824d00b48f..26db050f26 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2011 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -38,7 +38,10 @@ #include /* default request body limit */ -#define HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT 4096U +#define HTP_CONFIG_DEFAULT_REQUEST_BODY_LIMIT 4096U + +/** a boundary should be smaller in size */ +#define HTP_BOUNDARY_MAX 200U #define HTP_FLAG_STATE_OPEN 0x01 /**< Flag to indicate that HTTP connection is open */ @@ -72,8 +75,9 @@ enum { typedef struct HtpBodyChunk_ { uint8_t *data; /**< Pointer to the data of the chunk */ uint32_t len; /**< Length of the chunk */ - struct HtpBodyChunk_ *next; /**< Pointer to the next chunk */ uint32_t id; /**< number of chunk of the current body */ + struct HtpBodyChunk_ *next; /**< Pointer to the next chunk */ + uint64_t stream_offset; } HtpBodyChunk; /** Struct used to hold all the chunks of a body on a request */ @@ -83,29 +87,35 @@ typedef struct HtpBody_ { uint32_t nchunks; /**< Number of chunks in the current operation */ uint8_t operation; /**< This flag indicate if it's a request or a response */ + + /* pahole: padding: 3 */ } HtpBody; -#define HTP_BODY_COMPLETE 0x01 /* body is complete or limit is reached, - either way, this is it. */ -#define HTP_CONTENTTYPE_SET 0x02 /* We have the content type */ -#define HTP_BOUNDARY_SET 0x04 /* We have a boundary string */ -#define HTP_BOUNDARY_OPEN 0x08 /* We have a boundary string */ +#define HTP_BODY_COMPLETE 0x01 /**< body is complete or limit is reached, + either way, this is it. */ +#define HTP_CONTENTTYPE_SET 0x02 /**< We have the content type */ +#define HTP_BOUNDARY_SET 0x04 /**< We have a boundary string */ +#define HTP_BOUNDARY_OPEN 0x08 /**< We have a boundary string */ +#define HTP_FILENAME_SET 0x10 /**< filename is registered in the flow */ /** Now the Body Chunks will be stored per transaction, at * the tx user data */ typedef struct SCHtpTxUserData_ { /* Body of the request (if any) */ HtpBody body; + /* Holds the length of the htp request body */ - uint32_t content_len; + uint64_t content_len; /* Holds the length of the htp request body seen so far */ - uint32_t content_len_so_far; - /* Holds the boundary identificator string if any (used on multipart/form-data only) */ + uint64_t content_len_so_far; + + uint64_t body_parsed; + + /** Holds the boundary identificator string if any (used on + * multipart/form-data only) + */ uint8_t *boundary; - uint32_t boundary_len; - /* Holds the content-type */ - uint8_t *contenttype; - uint32_t contenttype_len; + uint8_t boundary_len; uint8_t flags; } SCHtpTxUserData; diff --git a/src/detect-fileext.c b/src/detect-fileext.c index 3c3fe08c54..ef319d59e0 100644 --- a/src/detect-fileext.c +++ b/src/detect-fileext.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2010 Open Information Security Foundation +/* Copyright (C) 2007-2011 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -18,6 +18,7 @@ /** * \file * + * \author Victor Julien * \author Pablo Rincon * */ @@ -52,7 +53,7 @@ /** * \brief Regex for parsing the fileext string */ -#define PARSE_REGEX "^\\s*\"\\s*(.+)\\s*\"\\s*$" +#define PARSE_REGEX "^\\s*\"?\\s*(.+)\\s*\"?\\s*$" static pcre *parse_regex; static pcre_extra *parse_regex_study; @@ -120,8 +121,7 @@ int DetectFileextMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, if (f->files != NULL && f->files->cnt > 0) { FlowFile *file = f->files->start; - for (; file != NULL; file = file->next) - { + for (; file != NULL; file = file->next) { if (file->ext != NULL) { //PrintRawDataFp(stdout, file->ext, file->ext_len); @@ -131,6 +131,7 @@ int DetectFileextMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, Flow *f, { ret = 1; SCLogDebug("File ext found"); + /* Stop searching */ break; } @@ -179,18 +180,23 @@ DetectFileextData *DetectFileextParse (char *str) memset(fileext, 0x00, sizeof(DetectFileextData)); /* Remove quotes if any and copy the filename */ - if (str_ptr[0] == '"') { + if (str_ptr[0] == '\"') { fileext->ext = (uint8_t *)SCStrdup((char*)str_ptr + 1); - fileext->ext[strlen(str_ptr - 1)] = '\0'; } else { fileext->ext = (uint8_t *)SCStrdup((char*)str_ptr); } + if (fileext->ext[strlen((char *)fileext->ext) - 1] == '\"') { + fileext->ext[strlen((char *)fileext->ext) - 1] = '\0'; + } if (fileext->ext == NULL) { goto error; } fileext->len = strlen((char *) fileext->ext); fileext->bm_ctx = BoyerMooreCtxInit(fileext->ext, fileext->len); + if (fileext->bm_ctx == NULL) { + goto error; + } BoyerMooreCtxToNocase(fileext->bm_ctx, fileext->ext, fileext->len); SCLogDebug("will look for fileext %s", fileext->ext); @@ -207,7 +213,7 @@ error: /** * \brief this function is used to add the parsed "id" option - * \brief into the current signature + * into the current signature * * \param de_ctx pointer to the Detection Engine Context * \param s pointer to the Current Signature @@ -260,9 +266,13 @@ error: * \param fileext pointer to DetectFileextData */ void DetectFileextFree(void *ptr) { - DetectFileextData *fileext = (DetectFileextData *)ptr; - BoyerMooreCtxDeInit(fileext->bm_ctx); - SCFree(fileext); + if (ptr != NULL) { + DetectFileextData *fileext = (DetectFileextData *)ptr; + if (fileext->bm_ctx != NULL) { + BoyerMooreCtxDeInit(fileext->bm_ctx); + } + SCFree(fileext); + } } #ifdef UNITTESTS /* UNITTESTS */ @@ -271,6 +281,11 @@ void DetectFileextFree(void *ptr) { * \test DetectFileextTestParse01 */ int DetectFileextTestParse01 (void) { + DetectFileextData *dfd = DetectFileextParse("doc"); + if (dfd != NULL) { + DetectFileextFree(dfd); + return 1; + } return 0; } @@ -278,6 +293,17 @@ int DetectFileextTestParse01 (void) { * \test DetectFileextTestParse02 */ int DetectFileextTestParse02 (void) { + int result = 0; + + DetectFileextData *dfd = DetectFileextParse("\"tar.gz\""); + if (dfd != NULL) { + if (dfd->len == 6 && memcmp(dfd->ext, "tar.gz", 6) == 0) { + result = 1; + } + + DetectFileextFree(dfd); + return result; + } return 0; } @@ -285,11 +311,19 @@ int DetectFileextTestParse02 (void) { * \test DetectFileextTestParse03 */ int DetectFileextTestParse03 (void) { - return 1; -} + int result = 0; + DetectFileextData *dfd = DetectFileextParse("\"pdf\""); + if (dfd != NULL) { + if (dfd->len == 3 && memcmp(dfd->ext, "pdf", 3) == 0) { + result = 1; + } -#include "stream-tcp-reassemble.h" + DetectFileextFree(dfd); + return result; + } + return 0; +} #endif /* UNITTESTS */ diff --git a/src/detect-filename.c b/src/detect-filename.c index 4c3df08269..b976660710 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -52,7 +52,7 @@ /** * \brief Regex for parsing the protoversion string */ -#define PARSE_REGEX "^\\s*\"\\s*(.+)\\s*\"\\s*$" +#define PARSE_REGEX "^\\s*\"?\\s*(.+)\\s*\"?\\s*$" static pcre *parse_regex; static pcre_extra *parse_regex_study; @@ -174,17 +174,25 @@ DetectFilenameData *DetectFilenameParse (char *str) memset(filename, 0x00, sizeof(DetectFilenameData)); - if (str_ptr[0] == '"') { + /* Remove quotes if any and copy the filename */ + if (str_ptr[0] == '\"') { filename->name = (uint8_t *)SCStrdup((char*)str_ptr + 1); - filename->name[strlen(str_ptr - 1)] = '\0'; } else { filename->name = (uint8_t *)SCStrdup((char*)str_ptr); } + if (filename->name[strlen((char *)filename->name) - 1] == '\"') { + filename->name[strlen((char *)filename->name) - 1] = '\0'; + } + if (filename->name == NULL) { goto error; } + filename->len = strlen((char *) filename->name); filename->bm_ctx = BoyerMooreCtxInit(filename->name, filename->len); + if (filename->bm_ctx == NULL) { + goto error; + } BoyerMooreCtxToNocase(filename->bm_ctx, filename->name, filename->len); SCLogDebug("will look for filename %s", filename->name); @@ -254,9 +262,13 @@ error: * \param filename pointer to DetectFilenameData */ void DetectFilenameFree(void *ptr) { - DetectFilenameData *filename = (DetectFilenameData *)ptr; - BoyerMooreCtxDeInit(filename->bm_ctx); - SCFree(filename); + if (ptr != NULL) { + DetectFilenameData *filename = (DetectFilenameData *)ptr; + if (filename->bm_ctx != NULL) { + BoyerMooreCtxDeInit(filename->bm_ctx); + } + SCFree(filename); + } } #ifdef UNITTESTS /* UNITTESTS */ @@ -265,6 +277,11 @@ void DetectFilenameFree(void *ptr) { * \test DetectFilenameTestParse01 */ int DetectFilenameTestParse01 (void) { + DetectFilenameData *dnd = DetectFilenameParse("secret.pdf"); + if (dnd != NULL) { + DetectFilenameFree(dnd); + return 1; + } return 0; } @@ -272,6 +289,17 @@ int DetectFilenameTestParse01 (void) { * \test DetectFilenameTestParse02 */ int DetectFilenameTestParse02 (void) { + int result = 0; + + DetectFilenameData *dnd = DetectFilenameParse("\"backup.tar.gz\""); + if (dnd != NULL) { + if (dnd->len == 13 && memcmp(dnd->name, "backup.tar.gz", 13) == 0) { + result = 1; + } + + DetectFilenameFree(dnd); + return result; + } return 0; } @@ -279,11 +307,19 @@ int DetectFilenameTestParse02 (void) { * \test DetectFilenameTestParse03 */ int DetectFilenameTestParse03 (void) { - return 1; -} + int result = 0; + DetectFilenameData *dnd = DetectFilenameParse("cmd.exe"); + if (dnd != NULL) { + if (dnd->len == 7 && memcmp(dnd->name, "cmd.exe", 7) == 0) { + result = 1; + } -#include "stream-tcp-reassemble.h" + DetectFilenameFree(dnd); + return result; + } + return 0; +} #endif /* UNITTESTS */