smtp: apply max-tx to live txs

Instead of the cumulative tx count

Ticket: 9038
(cherry picked from commit 04c066977a)

src/app-layer-smtp.c

@@ -547,7 +547,7 @@ static void SMTPSetEvent(SMTPState *s, uint8_t e)
 
 static SMTPTransaction *SMTPTransactionCreate(SMTPState *state)
 {
-    if (state->tx_cnt > smtp_config.max_tx) {
+    if (state->nb_live_tx >= smtp_config.max_tx) {
         return NULL;
     }
     SMTPTransaction *tx = SCCalloc(1, sizeof(*tx));
@@ -1331,6 +1331,7 @@ static int SMTPProcessRequest(
             return -1;
         state->curr_tx = tx;
         TAILQ_INSERT_TAIL(&state->tx_list, tx, next);
+        state->nb_live_tx++;
         tx->tx_id = state->tx_cnt++;
 
         /* keep track of the start of the tx */
@@ -1996,6 +1997,7 @@ static void SMTPStateTransactionFree (void *state, uint64_t tx_id)
         if (tx == smtp_state->curr_tx)
             smtp_state->curr_tx = NULL;
         TAILQ_REMOVE(&smtp_state->tx_list, tx, next);
+        smtp_state->nb_live_tx--;
         SMTPTransactionFree(tx, state);
         break;
     }

src/app-layer-smtp.h

@@ -132,6 +132,7 @@ typedef struct SMTPState_ {
     AppLayerStateData state_data;
     SMTPTransaction *curr_tx;
     TAILQ_HEAD(, SMTPTransaction_) tx_list;  /**< transaction list */
+    uint64_t nb_live_tx;
     uint64_t tx_cnt;
     uint64_t toserver_data_count;
     uint64_t toserver_last_data_stamp;