proto/detect: workaround dns misdetected as dcerpc

The DCERPC UDP detection would misfire on DNS with transaction
ID 0x0400. This would happen as the protocol detection engine
gives preference to pattern based detection over probing parsers for
performance reasons.

This hack/workaround fixes this specific case by still running the
probing parser if DCERPC has been detected on UDP. The probing
parser result will take precedence.

Bug #2736.

src/app-layer-detect-proto.c

@@ -1314,6 +1314,7 @@ AppProto AppLayerProtoDetectGetProto(AppLayerProtoDetectThreadCtx *tctx,
     SCEnter();
 
     AppProto alproto = ALPROTO_UNKNOWN;
+    AppProto pm_alproto = ALPROTO_UNKNOWN;
 
     if (!FLOW_IS_PM_DONE(f, direction)) {
         AppProto pm_results[ALPROTO_MAX];
@@ -1324,7 +1325,15 @@ AppProto AppLayerProtoDetectGetProto(AppLayerProtoDetectThreadCtx *tctx,
                                                    pm_results);
         if (pm_matches > 0) {
             alproto = pm_results[0];
-            goto end;
+
+            /* HACK: if detected protocol is dcerpc/udp, we run PP as well
+             * to avoid misdetecting DNS as DCERPC. */
+            if (!(ipproto == IPPROTO_UDP && alproto == ALPROTO_DCERPC))
+                goto end;
+
+            pm_alproto = alproto;
+
+            /* fall through */
         }
     }
 
@@ -1332,6 +1341,9 @@ AppProto AppLayerProtoDetectGetProto(AppLayerProtoDetectThreadCtx *tctx,
         alproto = AppLayerProtoDetectPPGetProto(f, buf, buflen, ipproto, direction);
 
  end:
+    if (alproto == ALPROTO_UNKNOWN)
+        alproto = pm_alproto;
+
     SCReturnUInt(alproto);
 }