Don't match on IP only rules that use ports if packet is not (proper) TCP, UDP or SCTP. Rules out frags matching as well.

src/detect-engine-iponly.c

@@ -1047,6 +1047,13 @@ void IPOnlyMatchPacket(ThreadVars *tv,
                                 continue;
                             }
                         }
+                    } else {
+                        if (!(s->flags & SIG_FLAG_DP_ANY)) {
+                            continue;
+                        }
+                        if (!(s->flags & SIG_FLAG_SP_ANY)) {
+                            continue;
+                        }
                     }
 
                     if (!IPOnlyMatchCompatSMs(tv, det_ctx, s, p)) {