Improve testing and fix some bugs.

src/app-layer-htp-file.c

@@ -346,6 +346,10 @@ static int HTPFileParserTest02(void) {
         goto end;
     }
 
+    if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) {
+        goto end;
+    }
+
     result = 1;
 end:
     FlowL7DataPtrFree(&f);
@@ -464,6 +468,14 @@ static int HTPFileParserTest03(void) {
         goto end;
     }
 
+    if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) {
+        goto end;
+    }
+
+    if (f.files->head->chunks_head->len != 11) {
+        goto end;
+    }
+
     result = 1;
 end:
     FlowL7DataPtrFree(&f);
@@ -594,6 +606,119 @@ end:
         HTPStateFree(http_state);
     return result;
 }
+
+static int HTPFileParserTest05(void) {
+    int result = 0;
+    Flow f;
+    uint8_t httpbuf1[] = "POST /upload.cgi HTTP/1.1\r\n"
+                         "Host: www.server.lan\r\n"
+                         "Content-Type: multipart/form-data; boundary=---------------------------277531038314945\r\n"
+                         "Content-Length: 544\r\n"
+                         "\r\n"
+                         "-----------------------------277531038314945\r\n"
+                         "Content-Disposition: form-data; name=\"uploadfile_0\"; filename=\"somepicture1.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n"
+                         "filecontent\r\n"
+                         "-----------------------------277531038314945\r\n";
+    uint32_t httplen1 = sizeof(httpbuf1) - 1; /* minus the \0 */
+    uint8_t httpbuf2[] = "Content-Disposition: form-data; name=\"uploadfile_1\"; filename=\"somepicture2.jpg\"\r\n"
+                         "Content-Type: image/jpeg\r\n"
+                         "\r\n"
+                         "FILECONTENT\r\n"
+        "-----------------------------277531038314945--";
+    uint32_t httplen2 = sizeof(httpbuf2) - 1; /* minus the \0 */
+
+    TcpSession ssn;
+    HtpState *http_state = NULL;
+
+    memset(&f, 0, sizeof(f));
+    memset(&ssn, 0, sizeof(ssn));
+    f.protoctx = (void *)&ssn;
+    f.src.family = AF_INET;
+    f.dst.family = AF_INET;
+
+    StreamTcpInitConfig(TRUE);
+    FlowL7DataPtrInit(&f);
+
+    SCLogDebug("\n>>>> processing chunk 1 size %u <<<<\n", httplen1);
+    int r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_START, httpbuf1, httplen1);
+    if (r != 0) {
+        printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r);
+        result = 0;
+        goto end;
+    }
+
+    SCLogDebug("\n>>>> processing chunk 2 size %u <<<<\n", httplen2);
+    r = AppLayerParse(&f, ALPROTO_HTTP, STREAM_TOSERVER|STREAM_EOF, httpbuf2, httplen2);
+    if (r != 0) {
+        printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r);
+        result = 0;
+        goto end;
+    }
+
+    http_state = f.aldata[AlpGetStateIdx(ALPROTO_HTTP)];
+    if (http_state == NULL) {
+        printf("no http state: ");
+        result = 0;
+        goto end;
+    }
+
+    htp_tx_t *tx = list_get(http_state->connp->conn->transactions, 0);
+    if (tx == NULL) {
+        goto end;
+    }
+
+    if (tx->request_method == NULL || memcmp(bstr_tocstr(tx->request_method), "POST", 4) != 0)
+    {
+        printf("expected method POST, got %s \n", bstr_tocstr(tx->request_method));
+        goto end;
+    }
+
+    if (f.files == NULL || f.files->tail == NULL || f.files->tail->state != FLOWFILE_STATE_CLOSED) {
+        goto end;
+    }
+
+    if (f.files->head == f.files->tail)
+        goto end;
+
+    if (f.files->head->next != f.files->tail)
+        goto end;
+
+    if (f.files->head->chunks_head->len != 11) {
+        printf("expected 11 but file is %u bytes instead\n",
+                f.files->head->chunks_head->len);
+        PrintRawDataFp(stdout, f.files->head->chunks_head->data,
+                f.files->head->chunks_head->len);
+        goto end;
+    }
+
+    if (memcmp("filecontent", f.files->head->chunks_head->data,
+                f.files->head->chunks_head->len) != 0) {
+        goto end;
+    }
+
+    if (f.files->tail->chunks_head->len != 11) {
+        printf("expected 11 but file is %u bytes instead\n",
+                f.files->tail->chunks_head->len);
+        PrintRawDataFp(stdout, f.files->tail->chunks_head->data,
+                f.files->tail->chunks_head->len);
+        goto end;
+    }
+
+    if (memcmp("FILECONTENT", f.files->tail->chunks_head->data,
+                f.files->tail->chunks_head->len) != 0) {
+        goto end;
+    }
+    result = 1;
+end:
+    FlowL7DataPtrFree(&f);
+    StreamTcpFreeConfig(TRUE);
+    if (http_state != NULL)
+        HTPStateFree(http_state);
+    return result;
+}
+
 #endif /* UNITTESTS */
 
 void HTPFileParserRegisterTests(void) {
@@ -602,5 +727,6 @@ void HTPFileParserRegisterTests(void) {
     UtRegisterTest("HTPFileParserTest02", HTPFileParserTest02, 1);
     UtRegisterTest("HTPFileParserTest03", HTPFileParserTest03, 1);
     UtRegisterTest("HTPFileParserTest04", HTPFileParserTest04, 1);
+    UtRegisterTest("HTPFileParserTest05", HTPFileParserTest05, 1);
 #endif /* UNITTESTS */
 }

src/app-layer-htp.c

@@ -921,11 +921,11 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
             goto end;
         }
 
-        //#if 0
+#if 0
         printf("CHUNK START: \n");
         PrintRawDataFp(stdout, chunks_buffer, chunks_buffer_len);
         printf("CHUNK END: \n");
-        //#endif
+#endif
 
         uint8_t expected_boundary_len = htud->boundary_len + 2;
         expected_boundary = (uint8_t *)SCMalloc(expected_boundary_len);
@@ -963,20 +963,20 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
                 uint8_t flags = 0;
 
                 if (header_start < form_end) {
-                    filedata_len = header_start - filedata;
+                    filedata_len = header_start - filedata - 2; /* 0d 0a */
                 } else if (form_end < header_start) {
                     filedata_len = form_end - filedata;
                 } else if (form_end != NULL && form_end == header_start) {
-                    filedata_len = form_end - filedata;
+                    filedata_len = form_end - filedata - 2; /* 0d 0a */
                 } else if (htud->flags & HTP_BODY_COMPLETE) {
                     filedata_len = chunks_buffer_len;
                     flags = FLOW_FILE_TRUNCATED;
                 }
-
+#if 0
                 printf("FILEDATA (final chunk) START: \n");
                 PrintRawDataFp(stdout, filedata, filedata_len);
                 printf("FILEDATA (final chunk) END: \n");
-
+#endif
                 if (HTPFileClose(hstate->f, filedata, filedata_len, flags) == -1)
                 {
                     goto end;
@@ -991,11 +991,11 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
                 if (chunks_buffer_len > expected_boundary_end_len) {
                     uint8_t *filedata = chunks_buffer;
                     uint32_t filedata_len = chunks_buffer_len - expected_boundary_len;
-
+#if 0
                     printf("FILEDATA (part) START: \n");
                     PrintRawDataFp(stdout, filedata, filedata_len);
                     printf("FILEDATA (part) END: \n");
-
+#endif
                     if (HTPFileStoreChunk(hstate->f, filedata, filedata_len) == -1) {
                         goto end;
                     }
@@ -1025,11 +1025,11 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
 
             uint8_t *header = header_start + (expected_boundary_len + 2); // + for 0d 0a
             header_len -= (expected_boundary_len + 2);
-            //#if 0
+#if 0
             printf("HEADER START: \n");
             PrintRawDataFp(stdout, header, header_len);
             printf("HEADER END: \n");
-            //#endif
+#endif
             while (header_len > 0) {
                 uint8_t *next_line = Bs2bmSearch(header, header_len, (uint8_t *)"\r\n", 2);
                 uint8_t *line = header;
@@ -1082,23 +1082,32 @@ int HTPCallbackRequestBodyData(htp_tx_data_t *d)
                     filedata_len = form_end - (header_end + 4 + 2);
                     SCLogDebug("filedata_len %"PRIuMAX, (uintmax_t)filedata_len);
 
-                    //#if 0
+#if 0
                     printf("FILEDATA START: \n");
                     PrintRawDataFp(stdout, filedata, filedata_len);
                     printf("FILEDATA END: \n");
-                    //#endif
+#endif
+
+                    if (HTPFileOpen(hstate->f, filename, filename_len,
+                                filedata, filedata_len) == -1) {
+                        goto end;
+                    }
+                    if (HTPFileClose(hstate->f, NULL, 0, 0) == -1) {
+                        goto end;
+                    }
                 } else {
                     SCLogDebug("more file data to come");
 
                     uint32_t offset = (header_end + 4) - chunks_buffer;
                     SCLogDebug("offset %u", offset);
                     htud->body_parsed = offset;
-                }
 
-                if (HTPFileOpen(hstate->f, filename, filename_len,
+                    if (HTPFileOpen(hstate->f, filename, filename_len,
-                        filedata, filedata_len) == -1) {
+                                filedata, filedata_len) == -1) {
-                    goto end;
+                        goto end;
+                    }
                 }
+
             }
 
             filename = NULL;

src/flow-file.c

@@ -225,7 +225,7 @@ FlowFile *FlowFileOpenFile(FlowFileContainer *ffc, uint8_t *name,
 {
     SCEnter();
 
-    PrintRawDataFp(stdout, name, name_len);
+    //PrintRawDataFp(stdout, name, name_len);
 
     FlowFile *ff = FlowFileAlloc(name, name_len);
     if (ff == NULL) {
@@ -235,25 +235,25 @@ FlowFile *FlowFileOpenFile(FlowFileContainer *ffc, uint8_t *name,
     ff->state = FLOWFILE_STATE_OPENED;
     SCLogDebug("flowfile state transitioned to FLOWFILE_STATE_OPENED");
 
+    FlowFileContainerAdd(ffc, ff);
+
     if (data != NULL) {
-        PrintRawDataFp(stdout, data, data_len);
+        //PrintRawDataFp(stdout, data, data_len);
 
         FlowFileData *ffd = FlowFileDataAlloc(data, data_len);
         if (ffd == NULL) {
-            FlowFileFree(ff);
+            ff->state = FLOWFILE_STATE_ERROR;
             SCReturnPtr(NULL, "FlowFile");
         }
 
         /* append the data */
         if (FlowFileAppendFlowFileData(ffc, ffd) < 0) {
-            FlowFileFree(ff);
+            ff->state = FLOWFILE_STATE_ERROR;
             FlowFileDataFree(ffd);
             SCReturnPtr(NULL, "FlowFile");
         }
     }
 
-    FlowFileContainerAdd(ffc, ff);
-
     SCReturnPtr(ff, "FlowFile");
 }
 
@@ -282,15 +282,17 @@ int FlowFileCloseFile(FlowFileContainer *ffc, uint8_t *data,
     }
 
     if (data != NULL) {
-        PrintRawDataFp(stdout, data, data_len);
+        //PrintRawDataFp(stdout, data, data_len);
 
         FlowFileData *ffd = FlowFileDataAlloc(data, data_len);
         if (ffd == NULL) {
+            ffc->tail->state = FLOWFILE_STATE_ERROR;
             SCReturnInt(-1);
         }
 
         /* append the data */
         if (FlowFileAppendFlowFileData(ffc, ffd) < 0) {
+            ffc->tail->state = FLOWFILE_STATE_ERROR;
             FlowFileDataFree(ffd);
             SCReturnInt(-1);
         }
@@ -331,11 +333,13 @@ int FlowFileAppendData(FlowFileContainer *ffc, uint8_t *data, uint32_t data_len)
 
     FlowFileData *ffd = FlowFileDataAlloc(data, data_len);
     if (ffd == NULL) {
+        ffc->tail->state = FLOWFILE_STATE_ERROR;
         SCReturnInt(-1);
     }
 
     /* append the data */
     if (FlowFileAppendFlowFileData(ffc, ffd) < 0) {
+        ffc->tail->state = FLOWFILE_STATE_ERROR;
         FlowFileDataFree(ffd);
         SCReturnInt(-1);
     }

src/flow-file.h

@@ -38,6 +38,7 @@ typedef enum FlowFileState_ {
     FLOWFILE_STATE_TRUNCATED,   /**< flow file is not complete, but
                                      there will be no more data. */
     FLOWFILE_STATE_STORED,      /**< all fully written to disk */
+    FLOWFILE_STATE_ERROR,       /**< file is in an error state */
     FLOWFILE_STATE_MAX
 } FlowFileState;