detect: checks for overflow when comparing signatures priorities

src/detect-engine-sigorder.c

@@ -682,7 +682,12 @@ static int SCSigOrderByIPPairbitsCompare(SCSigSignatureWrapper *sw1,
 static int SCSigOrderByPriorityCompare(SCSigSignatureWrapper *sw1,
                                        SCSigSignatureWrapper *sw2)
 {
-    return sw2->sig->prio - sw1->sig->prio;
+    if (sw1->sig->prio > sw2->sig->prio) {
+        return -1;
+    } else if (sw1->sig->prio < sw2->sig->prio) {
+        return 1;
+    }
+    return 0;
 }
 
 /**