protodetect: use both directions over UDP

As is already done for TCP

Ticket: #2757

src/app-layer.c

@@ -685,8 +685,9 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
 
     /* if we don't know the proto yet and we have received a stream
      * initializer message, we run proto detection.
-     * We receive 2 stream init msgs (one for each direction) but we
-     * only run the proto detection once. */
+     * We receive 2 stream init msgs (one for each direction), we
+     * only run the proto detection for both and emit an event
+     * in the case protocols mismatch. */
     if (alproto == ALPROTO_UNKNOWN && (flags & STREAM_START)) {
         DEBUG_VALIDATE_BUG_ON(FlowChangeProto(f));
         /* run protocol detection */
@@ -785,8 +786,10 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
 int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *f)
 {
     SCEnter();
+    AppProto *alproto;
+    AppProto *alproto_otherdir;
 
-    if (f->alproto == ALPROTO_FAILED) {
+    if (f->alproto_ts == ALPROTO_FAILED && f->alproto_tc == ALPROTO_FAILED) {
         SCReturnInt(0);
     }
 
@@ -794,33 +797,75 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *
     uint8_t flags = 0;
     if (p->flowflags & FLOW_PKT_TOSERVER) {
         flags |= STREAM_TOSERVER;
+        alproto = &f->alproto_ts;
+        alproto_otherdir = &f->alproto_tc;
     } else {
         flags |= STREAM_TOCLIENT;
+        alproto = &f->alproto_tc;
+        alproto_otherdir = &f->alproto_ts;
     }
 
     AppLayerProfilingReset(tctx);
 
     /* if the protocol is still unknown, run detection */
-    if (f->alproto == ALPROTO_UNKNOWN) {
+    if (*alproto == ALPROTO_UNKNOWN) {
         SCLogDebug("Detecting AL proto on udp mesg (len %" PRIu32 ")",
                    p->payload_len);
 
         bool reverse_flow = false;
         PACKET_PROFILING_APP_PD_START(tctx);
-        f->alproto = AppLayerProtoDetectGetProto(tctx->alpd_tctx,
-                                  f, p->payload, p->payload_len,
-                                  IPPROTO_UDP, flags, &reverse_flow);
+        *alproto = AppLayerProtoDetectGetProto(
+                tctx->alpd_tctx, f, p->payload, p->payload_len, IPPROTO_UDP, flags, &reverse_flow);
         PACKET_PROFILING_APP_PD_END(tctx);
 
-        if (f->alproto != ALPROTO_UNKNOWN) {
-            AppLayerIncFlowCounter(tv, f);
-
-            if (p->flowflags & FLOW_PKT_TOSERVER) {
-                f->alproto_ts = f->alproto;
-            } else {
-                f->alproto_tc = f->alproto;
+        switch (*alproto) {
+            case ALPROTO_UNKNOWN:
+                if (*alproto_otherdir != ALPROTO_UNKNOWN) {
+                    // Use recognized side
+                    f->alproto = *alproto_otherdir;
+                    // do not keep ALPROTO_UNKNOWN for this side so as not to loop
+                    *alproto = *alproto_otherdir;
+                    if (*alproto_otherdir == ALPROTO_FAILED) {
+                        SCLogDebug("ALPROTO_UNKNOWN flow %p", f);
+                    }
+                } else {
+                    // First side of protocol is unknown
+                    *alproto = ALPROTO_FAILED;
+                }
+                break;
+            case ALPROTO_FAILED:
+                if (*alproto_otherdir != ALPROTO_UNKNOWN) {
+                    // Use recognized side
+                    f->alproto = *alproto_otherdir;
+                    if (*alproto_otherdir == ALPROTO_FAILED) {
+                        SCLogDebug("ALPROTO_UNKNOWN flow %p", f);
+                    }
+                }
+                // else wait for second side of protocol
+                break;
+            default:
+                if (*alproto_otherdir != ALPROTO_UNKNOWN && *alproto_otherdir != ALPROTO_FAILED) {
+                    if (*alproto_otherdir != *alproto) {
+                        AppLayerDecoderEventsSetEventRaw(
+                                &p->app_layer_events, APPLAYER_MISMATCH_PROTOCOL_BOTH_DIRECTIONS);
+                        // data already sent to parser, we cannot change the protocol to use the one
+                        // of the server
+                    }
+                } else {
+                    f->alproto = *alproto;
+                }
+        }
+        if (*alproto_otherdir == ALPROTO_UNKNOWN) {
+            if (f->alproto == ALPROTO_UNKNOWN) {
+                // so as to increase stat about .app_layer.flow.failed_udp
+                f->alproto = ALPROTO_FAILED;
             }
+            // If the other side is unknown, this is the first packet of the flow
+            AppLayerIncFlowCounter(tv, f);
+        }
 
+        // parse the data if we recognized one protocol
+        if (f->alproto != ALPROTO_UNKNOWN && f->alproto != ALPROTO_FAILED) {
             if (reverse_flow) {
                 SCLogDebug("reversing flow after proto detect told us so");
                 PacketSwap(p);
@@ -832,10 +877,6 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *
             r = AppLayerParserParse(tv, tctx->alp_tctx, f, f->alproto,
                                     flags, p->payload, p->payload_len);
             PACKET_PROFILING_APP_END(tctx, f->alproto);
-        } else {
-            f->alproto = ALPROTO_FAILED;
-            AppLayerIncFlowCounter(tv, f);
-            SCLogDebug("ALPROTO_UNKNOWN flow %p", f);
         }
         PACKET_PROFILING_APP_STORE(tctx, p);
         /* we do only inspection in one direction, so flag both