app-layer: notify parsers of gaps if enabled

A parser can now set a flag that will tell the application
layer that it is capable of handling gaps. If enabled, and a
gap occurs, the app-layer needs to be prepared to accept
input that is NULL with a length, where the length is the
number of bytes lost. It is up to the app-layer to
determine if it can sync up with the input data again.

src/app-layer-parser.c

@@ -127,6 +127,9 @@ typedef struct AppLayerParserProtoCtx_
      * STREAM_TOSERVER, STREAM_TOCLIENT */
     uint8_t first_data_dir;
 
+    /* Option flags such as supporting gaps or not. */
+    uint64_t flags;
+
 #ifdef UNITTESTS
     void (*RegisterUnittests)(void);
 #endif
@@ -366,6 +369,16 @@ void AppLayerParserRegisterParserAcceptableDataDirection(uint8_t ipproto, AppPro
     SCReturn;
 }
 
+void AppLayerParserRegisterOptionFlags(uint8_t ipproto, AppProto alproto,
+        uint64_t flags)
+{
+    SCEnter();
+
+    alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].flags |= flags;
+
+    SCReturn;
+}
+
 void AppLayerParserRegisterStateFuncs(uint8_t ipproto, AppProto alproto,
                            void *(*StateAlloc)(void),
                            void (*StateFree)(void *))
@@ -994,14 +1007,15 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow
     if (p->StateAlloc == NULL)
         goto end;
 
-    /* Do this check before calling AppLayerParse */
     if (flags & STREAM_GAP) {
-        SCLogDebug("stream gap detected (missing packets), "
+        if (!(p->flags & APP_LAYER_PARSER_OPT_ACCEPT_GAPS)) {
-                   "this is not yet supported.");
+            SCLogDebug("app-layer parser does not accept gaps");
-
+            if (f->alstate != NULL) {
-        if (f->alstate != NULL)
+                AppLayerParserStreamTruncated(f->proto, alproto, f->alstate,
-            AppLayerParserStreamTruncated(f->proto, alproto, f->alstate, flags);
+                        flags);
-        goto error;
+            }
+            goto error;
+        }
     }
 
     /* Get the parser state (if any) */

src/app-layer-parser.h

@@ -30,11 +30,15 @@
 #include "util-file.h"
 #include "stream-tcp-private.h"
 
+/* Flags for AppLayerParserState. */
 #define APP_LAYER_PARSER_EOF                    0x01
 #define APP_LAYER_PARSER_NO_INSPECTION          0x02
 #define APP_LAYER_PARSER_NO_REASSEMBLY          0x04
 #define APP_LAYER_PARSER_NO_INSPECTION_PAYLOAD  0x08
 
+/* Flags for AppLayerParserProtoCtx. */
+#define APP_LAYER_PARSER_OPT_ACCEPT_GAPS        BIT_U64(0)
+
 int AppLayerParserProtoIsRegistered(uint8_t ipproto, AppProto alproto);
 
 /***** transaction handling *****/
@@ -115,6 +119,8 @@ int AppLayerParserRegisterParser(uint8_t ipproto, AppProto alproto,
 void AppLayerParserRegisterParserAcceptableDataDirection(uint8_t ipproto,
                                               AppProto alproto,
                                               uint8_t direction);
+void AppLayerParserRegisterOptionFlags(uint8_t ipproto, AppProto alproto,
+        uint64_t flags);
 void AppLayerParserRegisterStateFuncs(uint8_t ipproto, AppProto alproto,
                            void *(*StateAlloc)(void),
                            void (*StateFree)(void *));

src/stream-tcp-reassemble.c

@@ -881,7 +881,7 @@ static void GetAppBuffer(TcpStream *stream, const uint8_t **data, uint32_t *data
                     "got data at %"PRIu64". GAP of size %"PRIu64,
                     offset, blk->offset, blk->offset - offset);
             *data = NULL;
-            *data_len = 0;
+            *data_len = blk->offset - offset;
 
         } else if (offset > blk->offset && offset <= (blk->offset + blk->len)) {
             SCLogDebug("get data from offset %"PRIu64". SBB %"PRIu64"/%u",
@@ -964,33 +964,36 @@ static int ReassembleUpdateAppLayer (ThreadVars *tv,
         TcpSession *ssn, TcpStream *stream,
         Packet *p, enum StreamUpdateDir dir)
 {
-    const uint64_t app_progress = STREAM_APP_PROGRESS(stream);
+    uint64_t app_progress = STREAM_APP_PROGRESS(stream);
 
     SCLogDebug("app progress %"PRIu64, app_progress);
     SCLogDebug("last_ack %u, base_seq %u", stream->last_ack, stream->base_seq);
 
     const uint8_t *mydata;
     uint32_t mydata_len;
-    GetAppBuffer(stream, &mydata, &mydata_len, app_progress);
-    SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
 
-    if (mydata == NULL || mydata_len == 0) {
+    while (1) {
-        if (CheckGap(ssn, stream, p)) {
+        GetAppBuffer(stream, &mydata, &mydata_len, app_progress);
-            /* send gap signal */
+        if (mydata == NULL && mydata_len > 0 && CheckGap(ssn, stream, p)) {
-            SCLogDebug("sending GAP to app-layer");
+            SCLogNotice("sending GAP to app-layer (size: %u)", mydata_len);
+
             AppLayerHandleTCPData(tv, ra_ctx, p, p->flow, ssn, stream,
-                    NULL, 0,
+                    NULL, mydata_len,
                     StreamGetAppLayerFlags(ssn, stream, p, dir)|STREAM_GAP);
             AppLayerProfilingStore(ra_ctx->app_tctx, p);
 
-            /* set a GAP flag and make sure not bothering this stream anymore */
-            SCLogDebug("STREAMTCP_STREAM_FLAG_GAP set");
-            stream->flags |= STREAMTCP_STREAM_FLAG_GAP;
-
             StreamTcpSetEvent(p, STREAM_REASSEMBLY_SEQ_GAP);
             StatsIncr(tv, ra_ctx->counter_tcp_reass_gap);
+
+            stream->app_progress_rel += mydata_len;
+            app_progress += mydata_len;
+            continue;
+        } else if (mydata == NULL || mydata_len == 0) {
+            /* Possibly a gap, but no new data. */
+            return 0;
         }
-        return 0;
+        SCLogDebug("%"PRIu64" got %p/%u", p->pcap_cnt, mydata, mydata_len);
+        break;
     }
 
     //PrintRawDataFp(stdout, mydata, mydata_len);
@@ -1069,10 +1072,6 @@ int StreamTcpReassembleAppLayer (ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         SCLogDebug("stream no reassembly flag set or app-layer disabled.");
         SCReturnInt(0);
     }
-    if (stream->flags & STREAMTCP_STREAM_FLAG_GAP) {
-        SCReturnInt(0);
-    }
-
 
     SCLogDebug("stream->seg_list %p", stream->seg_list);
 #ifdef DEBUG