probing parser updated to always accept u32 buflens. Update all probing parser functions to accomodate this change

src/app-layer-detect-proto.c

@@ -515,7 +515,7 @@ end:
  * \brief Call the probing parser if it exists for this src or dst port.
  */
 uint16_t AppLayerDetectGetProtoProbingParser(AlpProtoDetectCtx *ctx, Flow *f,
-                                             uint8_t *buf, uint16_t buflen,
+                                             uint8_t *buf, uint32_t buflen,
                                              uint8_t flags, uint8_t ipproto)
 {
     AppLayerProbingParserElement *pe = NULL;
@@ -562,9 +562,10 @@ uint16_t AppLayerDetectGetProtoProbingParser(AlpProtoDetectCtx *ctx, Flow *f,
         }
 
         int alproto = pe->ProbingParser(buf, buflen);
-        if (alproto > ALPROTO_UNKNOWN)
+        if (alproto != ALPROTO_UNKNOWN && alproto != ALPROTO_FAILED)
             return alproto;
-        if (alproto == -1 || (pe->max_depth != 0 && buflen > pe->max_depth)) {
+        if (alproto == ALPROTO_FAILED ||
+            (pe->max_depth != 0 && buflen > pe->max_depth)) {
             al_proto_masks[0] |= pe->al_proto_mask;
         }
         pe = pe->next;

src/app-layer-detect-proto.h

@@ -90,7 +90,7 @@ uint16_t AppLayerDetectGetProtoPMParser(AlpProtoDetectCtx *,
                                         uint8_t *, uint16_t,
                                         uint8_t, uint8_t);
 uint16_t AppLayerDetectGetProtoProbingParser(AlpProtoDetectCtx *, Flow *,
-                                             uint8_t *, uint16_t,
+                                             uint8_t *, uint32_t,
                                              uint8_t, uint8_t);
 uint16_t AppLayerDetectGetProto(AlpProtoDetectCtx *, AlpProtoDetectThreadCtx *,
                                 Flow *, uint8_t *, uint32_t,

src/app-layer-parser.c

@@ -1402,8 +1402,8 @@ AppLayerCreateAppLayerProbingParserElement(const char *al_proto_name,
                                            uint16_t port,
                                            uint8_t priority,
                                            uint8_t top,
-                                           int16_t (*AppLayerProbingParser)
+                                           uint16_t (*AppLayerProbingParser)
-                                           (uint8_t *input, int32_t input_len))
+                                           (uint8_t *input, uint32_t input_len))
 {
     AppLayerProbingParserElement *pe = SCMalloc(sizeof(AppLayerProbingParserElement));
     if (pe == NULL) {
@@ -1723,8 +1723,8 @@ int AppLayerProbingParserInfoAdd(AlpProtoDetectCtx *ctx,
                                  const char *al_proto_name,
                                  uint16_t ip_proto,
                                  uint16_t al_proto,
-                                 int16_t (*ProbingParser)
+                                 uint16_t (*ProbingParser)
-                                 (uint8_t *input, int32_t input_len))
+                                 (uint8_t *input, uint32_t input_len))
 {
     AppLayerProbingParserInfo *new_ppi = NULL;
 
@@ -1793,8 +1793,8 @@ void AppLayerRegisterProbingParser(AlpProtoDetectCtx *ctx,
                                    uint8_t flags,
                                    uint8_t priority,
                                    uint8_t top,
-                                   int16_t (*ProbingParser)
+                                   uint16_t (*ProbingParser)
-                                   (uint8_t *input, int32_t input_len))
+                                   (uint8_t *input, uint32_t input_len))
 {
     AppLayerProbingParser **probing_parsers = &ctx->probing_parsers;
     AppLayerProbingParserElement *pe = NULL;
@@ -2051,7 +2051,7 @@ end:
     return result;
 }
 
-int16_t ProbingParserDummyForTesting(uint8_t *input, int32_t input_len)
+uint16_t ProbingParserDummyForTesting(uint8_t *input, uint32_t input_len)
 {
     return 0;
 }

src/app-layer-parser.h

@@ -148,7 +148,7 @@ typedef struct AppLayerProbingParserElement_ {
     /* the max length of data after which this parser won't be invoked */
     uint32_t max_depth;
     /* the probing parser function */
-    int16_t (*ProbingParser)(uint8_t *input, int32_t input_len);
+    uint16_t (*ProbingParser)(uint8_t *input, uint32_t input_len);
 
     struct AppLayerProbingParserElement_ *next;
 } AppLayerProbingParserElement;
@@ -172,7 +172,7 @@ typedef struct AppLayerProbingParserInfo_ {
     const char *al_proto_name;
     uint16_t ip_proto;
     uint16_t al_proto;
-    int16_t (*ProbingParser)(uint8_t *input, int32_t input_len);
+    uint16_t (*ProbingParser)(uint8_t *input, uint32_t input_len);
     struct AppLayerProbingParserInfo_ *next;
 } AppLayerProbingParserInfo;
 
@@ -231,7 +231,7 @@ void AppLayerRegisterProbingParser(struct AlpProtoDetectCtx_ *, uint16_t, uint16
                                    const char *, uint16_t,
                                    uint16_t, uint16_t, uint8_t, uint8_t,
                                    uint8_t,
-                                   int16_t (*ProbingParser)(uint8_t *, int32_t));
+                                   uint16_t (*ProbingParser)(uint8_t *, uint32_t));
 void AppLayerRegisterStateFuncs(uint16_t proto, void *(*StateAlloc)(void),
                                 void (*StateFree)(void *));
 void AppLayerRegisterTransactionIdFuncs(uint16_t proto,

src/app-layer-protos.h

@@ -42,6 +42,9 @@ enum {
 #ifdef UNITTESTS
     ALPROTO_TEST,
 #endif /* UNITESTS */
+    /* used by the probing parser when alproto detection fails
+     * permanently for that particular stream */
+    ALPROTO_FAILED,
     /* keep last */
     ALPROTO_MAX,
 };

src/app-layer-smb.c

@@ -1328,9 +1328,10 @@ void SMBUpdateTransactionId(void *state, uint16_t *id) {
 
 #define SMB_PROBING_PARSER_MIN_DEPTH 8
 
-static int16_t SMBProbingParser(uint8_t *input, int32_t input_len)
+static uint16_t SMBProbingParser(uint8_t *input, uint32_t ilen)
 {
     int32_t len;
+    int32_t input_len = ilen;
 
     while (input_len >= SMB_PROBING_PARSER_MIN_DEPTH) {
         switch (input[0]) {
@@ -1353,7 +1354,7 @@ static int16_t SMBProbingParser(uint8_t *input, int32_t input_len)
                 /* -1 indicates a stream where the probing parser would be
                  * unable to find nbss, even if it exists.  This should
                  * prevent the probing parser from beig invoked henceforth */
-                return -1;
+                return ALPROTO_FAILED;
         }
 
         input_len -= 4;