file-extraction: improve handling of complex multipart bodies.

src/app-layer-htp-file.c

@@ -83,6 +83,8 @@ int HTPFileOpen(HtpState *s, uint8_t *filename, uint16_t filename_len,
     uint8_t flags = 0;
     FileContainer *files = NULL;
 
+    SCLogDebug("data %p data_len %"PRIu32, data, data_len);
+
     if (s == NULL) {
         SCReturnInt(-1);
     }
@@ -545,6 +547,7 @@ static int HTPFileParserTest03(void) {
     }
 
     if (http_state->files_ts->head->chunks_head->len != 11) {
+        printf("filedata len not 11 but %u: ", http_state->files_ts->head->chunks_head->len);
         goto end;
     }
 
@@ -760,7 +763,121 @@ static int HTPFileParserTest05(void) {
         goto end;
 
     if (http_state->files_ts->head->chunks_head->len != 11) {
-        printf("expected 11 but file is %u bytes instead\n",
+        printf("expected 11 but file is %u bytes instead: ",
+                http_state->files_ts->head->chunks_head->len);
+        PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
+                http_state->files_ts->head->chunks_head->len);
+        goto end;
+    }
+
+    if (memcmp("filecontent", http_state->files_ts->head->chunks_head->data,
+                http_state->files_ts->head->chunks_head->len) != 0) {
+        goto end;
+    }
+
+    if (http_state->files_ts->tail->chunks_head->len != 11) {
+        printf("expected 11 but file is %u bytes instead: ",
+                http_state->files_ts->tail->chunks_head->len);
+        PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data,
+                http_state->files_ts->tail->chunks_head->len);
+        goto end;
+    }
+
+    if (memcmp("FILECONTENT", http_state->files_ts->tail->chunks_head->data,
+                http_state->files_ts->tail->chunks_head->len) != 0) {
+        goto end;
+    }
+    result = 1;
+end:
+    StreamTcpFreeConfig(TRUE);
+    if (http_state != NULL)
+        HTPStateFree(http_state);
+    UTHFreeFlow(f);
+    return result;
+}
+
+/** \test first multipart part contains file but doesn't end in first chunk */
+static int HTPFileParserTest06(void) {
+    int result = 0;
+    Flow *f = NULL;
+    uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n"
+                         "Host: www.server.lan\r\n"
+                         "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n"
+                         "Content-Length: 544\r\n"
+                         "\r\n"
+                         "-----------------------------277531038314945\r\n"
+                         "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n"
+                         "filecontent\r\n"
+                         "-----------------------------27753103831494";
+    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
+    uint8_t httpbuf2[] = "5\r\nContent-Disposition: form-data; name=\"uploadfile_1\"; filename=\"somepicture2.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n"
+                         "FILECONTENT\r\n"
+        "-----------------------------277531038314945--";
+    uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
+
+    TcpSession ssn;
+    HtpState *http_state = NULL;
+
+    memset(&ssn, 0, sizeof(ssn));
+
+    f = UTHBuildFlow(AF_INET, "1.2.3.4", "1.2.3.5", 1024, 80);
+    if (f == NULL)
+        goto end;
+    f->protoctx = &ssn;
+
+    StreamTcpInitConfig(TRUE);
+
+    SCLogDebug("\n>>>> processing chunk 1 size %u <<<<\n", httplen1);
+    int r = AppLayerParse(NULL, f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
+    if (r != 0) {
+        printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
+        result = 0;
+        goto end;
+    }
+
+    SCLogDebug("\n>>>> processing chunk 2 size %u <<<<\n", httplen2);
+    r = AppLayerParse(NULL, f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf2, httplen2);
+    if (r != 0) {
+        printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r);
+        result = 0;
+        goto end;
+    }
+
+    http_state = f->alstate;
+    if (http_state == NULL) {
+        printf("no http state: ");
+        result = 0;
+        goto end;
+    }
+
+    htp_tx_t *tx = list_get(http_state->connp->conn->transactions, 0);
+    if (tx == NULL) {
+        goto end;
+    }
+
+    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    {
+        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        goto end;
+    }
+
+    if (http_state->files_ts == NULL || http_state->files_ts->tail == NULL ||
+            http_state->files_ts->tail->state != FILE_STATE_CLOSED) {
+        goto end;
+    }
+
+    if (http_state->files_ts->head == http_state->files_ts->tail)
+        goto end;
+
+    if (http_state->files_ts->head->next != http_state->files_ts->tail)
+        goto end;
+
+    if (http_state->files_ts->head->chunks_head->len != 11) {
+        printf("expected 11 but file is %u bytes instead: ",
                 http_state->files_ts->head->chunks_head->len);
         PrintRawDataFp(stdout, http_state->files_ts->head->chunks_head->data,
                 http_state->files_ts->head->chunks_head->len);
@@ -773,7 +890,7 @@ static int HTPFileParserTest05(void) {
     }
 
     if (http_state->files_ts->tail->chunks_head->len != 11) {
-        printf("expected 11 but file is %u bytes instead\n",
+        printf("expected 11 but file is %u bytes instead: ",
                 http_state->files_ts->tail->chunks_head->len);
         PrintRawDataFp(stdout, http_state->files_ts->tail->chunks_head->data,
                 http_state->files_ts->tail->chunks_head->len);
@@ -802,5 +919,6 @@ void HTPFileParserRegisterTests(void) {
     UtRegisterTest("HTPFileParserTest03", HTPFileParserTest03, 1);
     UtRegisterTest("HTPFileParserTest04", HTPFileParserTest04, 1);
     UtRegisterTest("HTPFileParserTest05", HTPFileParserTest05, 1);
+    UtRegisterTest("HTPFileParserTest06", HTPFileParserTest06, 1);
 #endif /* UNITTESTS */
 }

src/app-layer-htp.c

@@ -997,12 +997,20 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud,
     HtpBodyChunk *cur = htud->request_body.first;
 
     for ( ; cur != NULL; cur = cur->next) {
+        SCLogDebug("chunk %p", cur);
+
         /* skip body chunks entirely before what we parsed already */
-        if (cur->stream_offset + cur->len <= htud->request_body.body_parsed)
+        if (cur->stream_offset + cur->len <= htud->request_body.body_parsed) {
+            SCLogDebug("skipping chunk");
             continue;
+        }
+
+        SCLogDebug("cur->stream_offset %"PRIu64", cur->len %"PRIu32", body_parsed %"PRIu64,
+            cur->stream_offset, cur->len, htud->request_body.body_parsed);
 
         if (cur->stream_offset < htud->request_body.body_parsed &&
                 cur->stream_offset + cur->len >= htud->request_body.body_parsed) {
+            SCLogDebug("use part");
 
             uint32_t toff = htud->request_body.body_parsed - cur->stream_offset;
             uint32_t tlen = (cur->stream_offset + cur->len) - htud->request_body.body_parsed;
@@ -1015,6 +1023,8 @@ static void HtpRequestBodyReassemble(HtpTxUserData *htud,
             memcpy(buf + buf_len - tlen, cur->data + toff, tlen);
 
         } else {
+            SCLogDebug("use entire chunk");
+
             buf_len += cur->len;
             if ((buf = SCRealloc(buf, buf_len)) == NULL) {
                 buf_len = 0;
@@ -1195,24 +1205,68 @@ int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
                         goto end;
                     }
                 }
+
+                htud->request_body.body_parsed += (header_end - chunks_buffer);
+                htud->flags &= ~HTP_FILENAME_SET;
             } else {
-                SCLogDebug("more file data to come");
+                SCLogDebug("chunk doesn't contain form end");
 
-                uint32_t offset = (header_end + 4) - chunks_buffer;
+                filedata = header_end + 4;
-                SCLogDebug("offset %u", offset);
+                filedata_len = chunks_buffer_len - (filedata - chunks_buffer);
-                htud->request_body.body_parsed = offset;
+                SCLogDebug("filedata_len %u (chunks_buffer_len %u)", filedata_len, chunks_buffer_len);
 
+#ifdef PRINT
+                printf("FILEDATA START: \n");
+                PrintRawDataFp(stdout, filedata, filedata_len);
+                printf("FILEDATA END: \n");
+#endif
+                /* form doesn't end in this chunk, but part might. Lets
+                 * see if have another coming up */
+                uint8_t *header_next = Bs2bmSearch(filedata, filedata_len,
+                        expected_boundary, expected_boundary_len);
+                SCLogDebug("header_next %p", header_next);
+                if (header_next == NULL) {
+                    /* no, but we'll handle the file data when we see the
+                     * form_end */
 
-                result = HTPFileOpen(hstate, filename, filename_len,
+                    SCLogDebug("more file data to come");
+
+                    uint32_t offset = (header_end + 4) - chunks_buffer;
+                    SCLogDebug("offset %u", offset);
+                    htud->request_body.body_parsed += offset;
+
+                    result = HTPFileOpen(hstate, filename, filename_len,
+                            NULL, 0, hstate->transaction_cnt,
+                            STREAM_TOSERVER);
+                    if (result == -1) {
+                        goto end;
+                    } else if (result == -2) {
+                        htud->flags |= HTP_DONTSTORE;
+                    }
+                } else {
+                    filedata_len = header_next - filedata - 2;
+                    SCLogDebug("filedata_len %u", filedata_len);
+
+                    result = HTPFileOpen(hstate, filename, filename_len,
                             filedata, filedata_len, hstate->transaction_cnt,
                             STREAM_TOSERVER);
-                if (result == -1) {
+                    if (result == -1) {
-                    goto end;
+                        goto end;
-                } else if (result == -2) {
+                    } else if (result == -2) {
-                    htud->flags |= HTP_DONTSTORE;
+                        htud->flags |= HTP_DONTSTORE;
+                    } else {
+                        if (HTPFileClose(hstate, NULL, 0, 0, STREAM_TOSERVER) == -1) {
+                            goto end;
+                        }
+                    }
+
+                    htud->flags &= ~HTP_FILENAME_SET;
+                    htud->request_body.body_parsed += (header_end - chunks_buffer);
                 }
             }
 
+        } else {
+            htud->request_body.body_parsed += (header_end - chunks_buffer);
         }
 
         SCLogDebug("header_start %p, header_end %p, form_end %p",
@@ -1236,6 +1290,8 @@ end:
     if (expected_boundary_end != NULL) {
         SCFree(expected_boundary_end);
     }
+
+    SCLogDebug("htud->request_body.body_parsed %"PRIu64, htud->request_body.body_parsed);
     return 0;
 }
 
@@ -1362,6 +1418,12 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 {
     SCEnter();
 
+#ifdef PRINT
+    printf("HTPBODY START: \n");
+    PrintRawDataFp(stdout, (uint8_t *)d->data, d->len);
+    printf("HTPBODY END: \n");
+#endif
+
     HtpState *hstate = (HtpState *)d->tx->connp->user_data;
     if (hstate == NULL) {
         SCReturnInt(HOOK_ERROR);
@@ -1433,6 +1495,11 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
             if (chunks_buffer == NULL) {
                 goto end;
             }
+#ifdef PRINT
+    printf("REASSCHUNK START: \n");
+    PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
+    printf("REASSCHUNK END: \n");
+#endif
 
             HtpRequestBodyHandleMultipart(hstate, htud, chunks_buffer, chunks_buffer_len);