mqtt: bounds number of messages per tx

Ticket: 8525
(cherry picked from commit 03ba8a71d8)

rules/mqtt-events.rules

@@ -15,3 +15,4 @@ alert mqtt any any -> any any (msg:"SURICATA MQTT missing message ID"; app-layer
 alert mqtt any any -> any any (msg:"SURICATA MQTT unassigned message type (0 or >15)"; app-layer-event:mqtt.unassigned_msg_type; classtype:protocol-command-decode; sid:2229008; rev:1;)
 alert mqtt any any -> any any (msg:"SURICATA MQTT too many transactions"; app-layer-event:mqtt.too_many_transactions; classtype:protocol-command-decode; sid:2229009; rev:1;)
 alert mqtt any any -> any any (msg:"SURICATA MQTT malformed traffic"; app-layer-event:mqtt.malformed_traffic; classtype:protocol-command-decode; sid:2229010; rev:1;)
+alert mqtt any any -> any any (msg:"SURICATA MQTT too many messages"; app-layer-event:mqtt.too_many_messages; classtype:protocol-command-decode; sid:2229011; rev:1;)

rust/src/mqtt/mqtt.rs

@@ -40,6 +40,8 @@ static mut MAX_MSG_LEN: u32 = 1048576;
 
 static mut MQTT_MAX_TX: usize = 1024;
 
+static mut MQTT_MAX_MSGS: usize = 1024;
+
 static mut ALPROTO_MQTT: AppProto = ALPROTO_UNKNOWN;
 
 #[derive(AppLayerFrameType)]
@@ -62,6 +64,7 @@ pub enum MQTTEvent {
     UnassignedMsgType,
     TooManyTransactions,
     MalformedTraffic,
+    TooManyMessages,
 }
 
 #[derive(Debug)]
@@ -313,7 +316,11 @@ impl MQTTState {
             }
             MQTTOperation::PUBREC(ref v) | MQTTOperation::PUBREL(ref v) => {
                 if let Some(tx) = self.get_tx_by_pkt_id(v.message_id as u32) {
-                    tx.msg.push(msg);
+                    if tx.msg.len() >= unsafe { MQTT_MAX_MSGS } {
+                        tx.tx_data.set_event(MQTTEvent::TooManyMessages as u8);
+                    } else {
+                        tx.msg.push(msg);
+                    }
                 } else {
                     let mut tx = self.new_tx(msg, toclient);
                     MQTTState::set_event(&mut tx, MQTTEvent::MissingPublish);
@@ -805,6 +812,13 @@ pub unsafe extern "C" fn rs_mqtt_register_parser(cfg_max_msg_len: u32) {
                 SCLogError!("Invalid value for mqtt.max-tx");
             }
         }
+        if let Some(val) = conf_get("app-layer.protocols.mqtt.max-messages") {
+            if let Ok(v) = val.parse::<usize>() {
+                MQTT_MAX_MSGS = v;
+            } else {
+                SCLogWarning!("Invalid value for mqtt.max-messages");
+            }
+        }
     } else {
         SCLogDebug!("Protocol detector and parser disabled for MQTT.");
     }

suricata.yaml.in

@@ -909,6 +909,8 @@ app-layer:
       # unsubscribe-topic-match-limit: 100
       # Maximum number of live MQTT transactions per flow
       # max-tx: 4096
+      # Maximum number of messages per transaction
+      # max-messages: 1024
     krb5:
       enabled: yes
     bittorrent-dht: