From a6e75aff214226e74458cd94812b79fdabb4faa6 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 22 Dec 2011 17:48:48 +0100 Subject: [PATCH] file-extraction: improve handling of complex multipart bodies. --- src/app-layer-htp-file.c | 122 ++++++++++++++++++++++++++++++++++++++- src/app-layer-htp.c | 87 ++++++++++++++++++++++++---- 2 files changed, 197 insertions(+), 12 deletions(-) diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index 70b551923a..a6b5b8b3a1 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -83,6 +83,8 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len, uint8_t flags = 0; FileContainer *files = NULL; + SCLogDebug("data %p data_len %"PRIu32, data, data_len); + if (s == NULL) { SCReturnInt(-1); } @@ -545,6 +547,7 @@ static int HTPFileParserTest03(void) { } if (http_state->files_ts->head->chunks_head->len != 11) { + printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len); goto end; } @@ -760,7 +763,121 @@ static int HTPFileParserTest05(void) { goto end; if (http_state->files_ts->head->chunks_head->len != 11) { - printf("expected 11 but file is %u bytes instead\n", + printf("expected 11 but file is %u bytes instead: ", + http_state->files_ts->head->chunks_head->len); + PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data, + http_state->files_ts->head->chunks_head->len); + goto end; + } + + if (memcmp("filecontent", http_state->files_ts->head->chunks_head->data, + http_state->files_ts->head->chunks_head->len) != 0) { + goto end; + } + + if (http_state->files_ts->tail->chunks_head->len != 11) { + printf("expected 11 but file is %u bytes instead: ", + http_state->files_ts->tail->chunks_head->len); + PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data, + http_state->files_ts->tail->chunks_head->len); + goto end; + } + + if (memcmp("FILECONTENT", http_state->files_ts->tail->chunks_head->data, + http_state->files_ts->tail->chunks_head->len) != 0) { + goto end; + } + result = 1; +end: + StreamTcpFreeConfig(TRUE); + if (http_state != NULL) + HTPStateFree(http_state); + UTHFreeFlow(f); + return result; +} + +/** \test first multipart part contains file but doesn't end in first chunk */ +static int HTPFileParserTest06(void) { + int result = 0; + Flow *f = NULL; + uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n" + "Host: www.server.lan\r\n" + "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n" + "Content-Length: 544\r\n" + "\r\n" + "-----------------------------277531038314945\r\n" + "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n" + "filecontent\r\n" + "-----------------------------27753103831494"; + uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */ + uint8_t httpbuf2[] = "5\r\nContent-Disposition: form-data; name=\"uploadfile_1\"; filename=\"somepicture2.jpg\"\r\n" + "Content-Type: image/jpeg\r\n" + "\r\n" + "FILECONTENT\r\n" + "-----------------------------277531038314945--"; + uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */ + + TcpSession ssn; + HtpState *http_state = NULL; + + memset(&ssn, 0, sizeof(ssn)); + + f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80); + if (f == NULL) + goto end; + f->protoctx = &ssn; + + StreamTcpInitConfig(TRUE); + + SCLogDebug("\n>>>> processing chunk 1 size %u <<<<\n", httplen1); + int r = AppLayerParse(NULL, f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + SCLogDebug("\n>>>> processing chunk 2 size %u <<<<\n", httplen2); + r = AppLayerParse(NULL, f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r); + result = 0; + goto end; + } + + http_state = f->alstate; + if (http_state == NULL) { + printf("no http state: "); + result = 0; + goto end; + } + + htp_tx_t *tx = list_get(http_state->connp->conn->transactions, 0); + if (tx == NULL) { + goto end; + } + + if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0) + { + printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method)); + goto end; + } + + if (http_state->files_ts == NULL || http_state->files_ts->tail == NULL || + http_state->files_ts->tail->state != FILE_STATE_CLOSED) { + goto end; + } + + if (http_state->files_ts->head == http_state->files_ts->tail) + goto end; + + if (http_state->files_ts->head->next != http_state->files_ts->tail) + goto end; + + if (http_state->files_ts->head->chunks_head->len != 11) { + printf("expected 11 but file is %u bytes instead: ", http_state->files_ts->head->chunks_head->len); PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data, http_state->files_ts->head->chunks_head->len); @@ -773,7 +890,7 @@ static int HTPFileParserTest05(void) { } if (http_state->files_ts->tail->chunks_head->len != 11) { - printf("expected 11 but file is %u bytes instead\n", + printf("expected 11 but file is %u bytes instead: ", http_state->files_ts->tail->chunks_head->len); PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data, http_state->files_ts->tail->chunks_head->len); @@ -802,5 +919,6 @@ void HTPFileParserRegisterTests(void) { UtRegisterTest("HTPFileParserTest03", HTPFileParserTest03, 1); UtRegisterTest("HTPFileParserTest04", HTPFileParserTest04, 1); UtRegisterTest("HTPFileParserTest05", HTPFileParserTest05, 1); + UtRegisterTest("HTPFileParserTest06", HTPFileParserTest06, 1); #endif /* UNITTESTS */ } diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index b07729eb7b..23aa06c8fa 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -997,12 +997,20 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud, HtpBodyChunk *cur = htud->request_body.first; for ( ; cur != NULL; cur = cur->next) { + SCLogDebug("chunk %p", cur); + /* skip body chunks entirely before what we parsed already */ - if (cur->stream_offset + cur->len <= htud->request_body.body_parsed) + if (cur->stream_offset + cur->len <= htud->request_body.body_parsed) { + SCLogDebug("skipping chunk"); continue; + } + + SCLogDebug("cur->stream_offset %"PRIu64", cur->len %"PRIu32", body_parsed %"PRIu64, + cur->stream_offset, cur->len, htud->request_body.body_parsed); if (cur->stream_offset < htud->request_body.body_parsed && cur->stream_offset + cur->len >= htud->request_body.body_parsed) { + SCLogDebug("use part"); uint32_t toff = htud->request_body.body_parsed - cur->stream_offset; uint32_t tlen = (cur->stream_offset + cur->len) - htud->request_body.body_parsed; @@ -1015,6 +1023,8 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud, memcpy(buf + buf_len - tlen, cur->data + toff, tlen); } else { + SCLogDebug("use entire chunk"); + buf_len += cur->len; if ((buf = SCRealloc(buf, buf_len)) == NULL) { buf_len = 0; @@ -1195,24 +1205,68 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, goto end; } } + + htud->request_body.body_parsed += (header_end - chunks_buffer); + htud->flags &= ~HTP_FILENAME_SET; } else { - SCLogDebug("more file data to come"); + SCLogDebug("chunk doesn't contain form end"); - uint32_t offset = (header_end + 4) - chunks_buffer; - SCLogDebug("offset %u", offset); - htud->request_body.body_parsed = offset; + filedata = header_end + 4; + filedata_len = chunks_buffer_len - (filedata - chunks_buffer); + SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len); +#ifdef PRINT + printf("FILEDATA START: \n"); + PrintRawDataFp(stdout, filedata, filedata_len); + printf("FILEDATA END: \n"); +#endif + /* form doesn't end in this chunk, but part might. Lets + * see if have another coming up */ + uint8_t *header_next = Bs2bmSearch(filedata, filedata_len, + expected_boundary, expected_boundary_len); + SCLogDebug("header_next %p", header_next); + if (header_next == NULL) { + /* no, but we'll handle the file data when we see the + * form_end */ - result = HTPFileOpen(hstate, filename, filename_len, + SCLogDebug("more file data to come"); + + uint32_t offset = (header_end + 4) - chunks_buffer; + SCLogDebug("offset %u", offset); + htud->request_body.body_parsed += offset; + + result = HTPFileOpen(hstate, filename, filename_len, + NULL, 0, hstate->transaction_cnt, + STREAM_TOSERVER); + if (result == -1) { + goto end; + } else if (result == -2) { + htud->flags |= HTP_DONTSTORE; + } + } else { + filedata_len = header_next - filedata - 2; + SCLogDebug("filedata_len %u", filedata_len); + + result = HTPFileOpen(hstate, filename, filename_len, filedata, filedata_len, hstate->transaction_cnt, STREAM_TOSERVER); - if (result == -1) { - goto end; - } else if (result == -2) { - htud->flags |= HTP_DONTSTORE; + if (result == -1) { + goto end; + } else if (result == -2) { + htud->flags |= HTP_DONTSTORE; + } else { + if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) { + goto end; + } + } + + htud->flags &= ~HTP_FILENAME_SET; + htud->request_body.body_parsed += (header_end - chunks_buffer); } } + } else { + htud->request_body.body_parsed += (header_end - chunks_buffer); } SCLogDebug("header_start %p, header_end %p, form_end %p", @@ -1236,6 +1290,8 @@ end: if (expected_boundary_end != NULL) { SCFree(expected_boundary_end); } + + SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed); return 0; } @@ -1362,6 +1418,12 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) { SCEnter(); +#ifdef PRINT + printf("HTPBODY START: \n"); + PrintRawDataFp(stdout, (uint8_t *)d->data, d->len); + printf("HTPBODY END: \n"); +#endif + HtpState *hstate = (HtpState *)d->tx->connp->user_data; if (hstate == NULL) { SCReturnInt(HOOK_ERROR); @@ -1433,6 +1495,11 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d) if (chunks_buffer == NULL) { goto end; } +#ifdef PRINT + printf("REASSCHUNK START: \n"); + PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len); + printf("REASSCHUNK END: \n"); +#endif HtpRequestBodyHandleMultipart(hstate, htud, chunks_buffer, chunks_buffer_len);