output: do not use tx id 0 when there is no tx

Ticket: 6846

This led to packet rules logging irrelevant app-layer data

src/detect-engine-alert.c

@@ -272,7 +272,7 @@ static inline PacketAlert PacketAlertSet(
     pa.s = (Signature *)s;
     pa.flags = alert_flags;
     /* Set tx_id if the frame has it */
-    pa.tx_id = (tx_id == UINT64_MAX) ? 0 : tx_id;
+    pa.tx_id = tx_id;
     pa.frame_id = (alert_flags & PACKET_ALERT_FLAG_FRAME) ? det_ctx->frame_id : 0;
     return pa;
 }
@@ -317,8 +317,14 @@ static int AlertQueueSortHelper(const void *a, const void *b)
 {
     const PacketAlert *pa0 = a;
     const PacketAlert *pa1 = b;
-    if (pa1->num == pa0->num)
+    if (pa1->num == pa0->num) {
+        if (pa1->tx_id == PACKET_ALERT_NOTX) {
+            return -1;
+        } else if (pa0->tx_id == PACKET_ALERT_NOTX) {
+            return 1;
+        }
         return pa0->tx_id < pa1->tx_id ? 1 : -1;
+    }
     return pa0->num > pa1->num ? 1 : -1;
 }
 

src/detect.c

@@ -807,7 +807,18 @@ static inline void DetectRulePacketRules(
 #endif
         DetectRunPostMatch(tv, det_ctx, p, s);
 
-        AlertQueueAppend(det_ctx, s, p, 0, alert_flags);
+        uint64_t txid = PACKET_ALERT_NOTX;
+        if ((alert_flags & PACKET_ALERT_FLAG_STREAM_MATCH) ||
+                (s->alproto != ALPROTO_UNKNOWN && pflow->proto == IPPROTO_UDP)) {
+            // if there is a stream match (TCP), or
+            // a UDP specific app-layer signature,
+            // try to use the last tx
+            if (pflow->alstate) {
+                txid = AppLayerParserGetTxCnt(pflow, pflow->alstate) - 1;
+                alert_flags |= PACKET_ALERT_FLAG_TX;
+            }
+        }
+        AlertQueueAppend(det_ctx, s, p, txid, alert_flags);
 next:
         DetectVarProcessList(det_ctx, pflow, p);
         DetectReplaceFree(det_ctx);

src/detect.h

@@ -49,6 +49,9 @@
  *  classtype. */
 #define DETECT_DEFAULT_PRIO 3
 
+// tx_id value to use when there is no transaction
+#define PACKET_ALERT_NOTX UINT64_MAX
+
 /* forward declarations for the structures from detect-engine-sigorder.h */
 struct SCSigOrderFunc_;
 struct SCSigSignatureWrapper_;