app-layer: improve EOF handling

On receiving TCP end of stream packets (e.g. RST, but also sometimes FIN
packets), in some cases the AppLayer parser would not be notified. This
could happen in IDS mode, but would especially be an issue in IPS mode.

This patch changes the logic of the AppLayer API to handle this. When no
new data is available, and the stream ends, the AppLayer API now gets
called with a NULL/0 input, but with the EOF flag set.

This allows the AppLayer parser to call it's final routines still in the
context of a real packet.

src/app-layer-htp.c

@@ -739,6 +739,7 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
 
     htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
     /* pass the new data to the htp parser */
+    if (input_len > 0) {
         r = htp_connp_req_data(hstate->connp, &ts, input, input_len);
 
         switch(r) {
@@ -761,6 +762,7 @@ static int HTPHandleRequestData(Flow *f, void *htp_state,
                 hstate->flags &= ~HTP_FLAG_NEW_BODY_SET;
         }
         HTPHandleError(hstate);
+    }
 
     /* if the TCP connection is closed, then close the HTTP connection */
     if (AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF) &&
@@ -815,6 +817,7 @@ static int HTPHandleResponseData(Flow *f, void *htp_state,
     hstate->flags &=~ HTP_FLAG_NEW_BODY_SET;
 
     htp_time_t ts = { f->lastts.tv_sec, f->lastts.tv_usec };
+    if (input_len > 0) {
         r = htp_connp_res_data(hstate->connp, &ts, input, input_len);
         switch(r) {
             case HTP_STREAM_ERROR:
@@ -834,6 +837,7 @@ static int HTPHandleResponseData(Flow *f, void *htp_state,
                 hstate->flags &= ~HTP_FLAG_NEW_BODY_SET;
         }
         HTPHandleError(hstate);
+    }
 
     /* if we the TCP connection is closed, then close the HTTP connection */
     if (AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF) &&

src/app-layer-parser.c

@@ -890,7 +890,7 @@ int AppLayerParserParse(AppLayerParserThreadCtx *alp_tctx, Flow *f, AppProto alp
     }
 
     /* invoke the recursive parser, but only on data. We may get empty msgs on EOF */
-    if (input_len > 0) {
+    if (input_len > 0 || (flags & STREAM_EOF)) {
         /* invoke the parser */
         if (p->Parser[(flags & STREAM_TOSERVER) ? 0 : 1](f, alstate, pstate,
                 input, input_len,