app-layer: add tx iterator API

Until now, the transaction space is assumed to be terse. Transactions
are handled sequentially so the difference between the lowest and highest
active tx id's is small. For this reason the logic of walking every id
between the 'minimum' and max id made sense. The space might look like:

    [..........TTTT]

Here the looping starts at the first T and loops 4 times.

This assumption isn't a great fit though. A protocol like NFS has 2 types
of transactions. Long running file transfer transactions and short lived
request/reply pairs are causing the id space to be sparse. This leads to
a lot of unnecessary looping in various parts of the engine, but most
prominently: detection, tx house keeping and tx logging.

    [.T..T...TTTT.T]

Here the looping starts at the first T and loops for every spot, even
those where no tx exists anymore.

Cases have been observed where the lowest tx id was 2 and the highest
was 50k. This lead to a lot of unnecessary looping.

This patch add an alternative approach. It allows a protocol to register
an iterator function, that simply returns the next transaction until
all transactions are returned. To do this it uses a bit of state the
caller must keep.

The registration is optional. If no iterator is registered the old
behaviour will be used.

rust/gen-c-headers.py

@@ -88,6 +88,8 @@ type_map = {
     "core::AppLayerDecoderEvents": "AppLayerDecoderEvents",
     "AppLayerDecoderEvents": "AppLayerDecoderEvents",
     "core::AppLayerEventType": "AppLayerEventType",
+    "applayer::AppLayerGetTxIterTuple": "AppLayerGetTxIterTuple",
+    "AppLayerGetTxIterTuple": "AppLayerGetTxIterTuple",
     "AppLayerEventType": "AppLayerEventType",
     "CLuaState": "lua_State",
     "Store": "Store",

rust/src/applayer.rs

@@ -15,6 +15,29 @@
  * 02110-1301, USA.
  */
 
+extern crate libc;
+use std;
+
+#[repr(C)]
+pub struct AppLayerGetTxIterTuple {
+    tx_ptr: *mut libc::c_void,
+    tx_id: u64,
+    has_next: bool,
+}
+
+impl AppLayerGetTxIterTuple {
+    pub fn with_values(tx_ptr: *mut libc::c_void, tx_id: u64, has_next: bool) -> AppLayerGetTxIterTuple {
+        AppLayerGetTxIterTuple {
+            tx_ptr: tx_ptr, tx_id: tx_id, has_next: has_next,
+        }
+    }
+    pub fn not_found() -> AppLayerGetTxIterTuple {
+        AppLayerGetTxIterTuple {
+            tx_ptr: std::ptr::null_mut(), tx_id: 0, has_next: false,
+        }
+    }
+}
+
 /// LoggerFlags tracks which loggers have already been executed.
 #[derive(Debug)]
 pub struct LoggerFlags {

rust/src/nfs/nfs.rs

@@ -28,6 +28,7 @@ use nom;
 use nom::IResult;
 
 use log::*;
+use applayer;
 use applayer::LoggerFlags;
 use core::*;
 use filetracker::*;
@@ -423,6 +424,28 @@ impl NFSState {
         return None;
     }
 
+    // for use with the C API call StateGetTxIterator
+    pub fn get_tx_iterator(&mut self, min_tx_id: u64, state: &mut u64) ->
+        Option<(&NFSTransaction, u64, bool)>
+    {
+        let mut index = *state as usize;
+        let len = self.transactions.len();
+
+        // find tx that is >= min_tx_id
+        while index < len {
+            let tx = &self.transactions[index];
+            if tx.id < min_tx_id + 1 {
+                index += 1;
+                continue;
+            }
+            *state = index as u64 + 1;
+            SCLogDebug!("returning tx_id {} has_next? {} (len {} index {}), tx {:?}",
+                    tx.id - 1, (len - index) > 1, len, index, tx);
+            return Some((tx, tx.id - 1, (len - index) > 1));
+        }
+        return None;
+    }
+
     /// Set an event. The event is set on the most recent transaction.
     pub fn set_event(&mut self, event: NFSEvent) {
         let len = self.transactions.len();
@@ -1850,6 +1873,26 @@ pub extern "C" fn rs_nfs3_state_get_tx(state: &mut NFSState,
     }
 }
 
+// for use with the C API call StateGetTxIterator
+#[no_mangle]
+pub extern "C" fn rs_nfs_state_get_tx_iterator(
+                                      state: &mut NFSState,
+                                      min_tx_id: libc::uint64_t,
+                                      istate: &mut libc::uint64_t)
+                                      -> applayer::AppLayerGetTxIterTuple
+{
+    match state.get_tx_iterator(min_tx_id, istate) {
+        Some((tx, out_tx_id, has_next)) => {
+            let c_tx = unsafe { transmute(tx) };
+            let ires = applayer::AppLayerGetTxIterTuple::with_values(c_tx, out_tx_id, has_next);
+            return ires;
+        }
+        None => {
+            return applayer::AppLayerGetTxIterTuple::not_found();
+        }
+    }
+}
+
 #[no_mangle]
 pub extern "C" fn rs_nfs3_state_tx_free(state: &mut NFSState,
                                        tx_id: libc::uint64_t)

src/app-layer-nfs-tcp.c

@@ -194,6 +194,14 @@ static void *NFSTCPGetTx(void *state, uint64_t tx_id)
     return rs_nfs3_state_get_tx(state, tx_id);
 }
 
+static AppLayerGetTxIterTuple RustNFSTCPGetTxIterator(
+        const uint8_t ipproto, const AppProto alproto,
+        void *alstate, uint64_t min_tx_id, uint64_t max_tx_id,
+        AppLayerGetTxIterState *istate)
+{
+    return rs_nfs_state_get_tx_iterator(alstate, min_tx_id, (uint64_t *)istate);
+}
+
 static void NFSTCPSetTxLogged(void *state, void *vtx, LoggerId logged)
 {
     rs_nfs3_tx_set_logged(state, vtx, logged);
@@ -348,6 +356,8 @@ void RegisterNFSTCPParsers(void)
             ALPROTO_NFS, NFSTCPGetStateProgress);
         AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_NFS,
             NFSTCPGetTx);
+        AppLayerParserRegisterGetTxIterator(IPPROTO_TCP, ALPROTO_NFS,
+                RustNFSTCPGetTxIterator);
 
         AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_NFS, NFSTCPGetFiles);
 

src/app-layer-nfs-udp.c

@@ -187,6 +187,14 @@ static void *NFSGetTx(void *state, uint64_t tx_id)
     return rs_nfs3_state_get_tx(state, tx_id);
 }
 
+static AppLayerGetTxIterTuple RustNFSGetTxIterator(
+        const uint8_t ipproto, const AppProto alproto,
+        void *alstate, uint64_t min_tx_id, uint64_t max_tx_id,
+        AppLayerGetTxIterState *istate)
+{
+    return rs_nfs_state_get_tx_iterator(alstate, min_tx_id, (uint64_t *)istate);
+}
+
 static void NFSSetTxLogged(void *state, void *vtx, LoggerId logged)
 {
     rs_nfs3_tx_set_logged(state, vtx, logged);
@@ -341,6 +349,8 @@ void RegisterNFSUDPParsers(void)
             ALPROTO_NFS, NFSGetStateProgress);
         AppLayerParserRegisterGetTx(IPPROTO_UDP, ALPROTO_NFS,
             NFSGetTx);
+        AppLayerParserRegisterGetTxIterator(IPPROTO_UDP, ALPROTO_NFS,
+                RustNFSGetTxIterator);
 
         AppLayerParserRegisterGetFilesFunc(IPPROTO_UDP, ALPROTO_NFS, NFSGetFiles);
 

src/app-layer-parser.c

@@ -105,6 +105,7 @@ typedef struct AppLayerParserProtoCtx_
     int (*StateGetProgress)(void *alstate, uint8_t direction);
     uint64_t (*StateGetTxCnt)(void *alstate);
     void *(*StateGetTx)(void *alstate, uint64_t tx_id);
+    AppLayerGetTxIteratorFunc StateGetTxIterator;
     int (*StateGetProgressCompletionStatus)(uint8_t direction);
     int (*StateGetEventInfo)(const char *event_name,
                              int *event_id, AppLayerEventType *event_type);
@@ -518,6 +519,14 @@ void AppLayerParserRegisterGetTx(uint8_t ipproto, AppProto alproto,
     SCReturn;
 }
 
+void AppLayerParserRegisterGetTxIterator(uint8_t ipproto, AppProto alproto,
+                      AppLayerGetTxIteratorFunc Func)
+{
+    SCEnter();
+    alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetTxIterator = Func;
+    SCReturn;
+}
+
 void AppLayerParserRegisterGetStateProgressCompletionStatus(AppProto alproto,
     int (*StateGetProgressCompletionStatus)(uint8_t direction))
 {
@@ -609,6 +618,49 @@ void AppLayerParserDestroyProtocolParserLocalStorage(uint8_t ipproto, AppProto a
     SCReturn;
 }
 
+/** \brief default tx iterator
+ *
+ *  Used if the app layer parser doesn't register its own iterator.
+ *  Simply walks the tx_id space until it finds a tx. Uses 'state' to
+ *  keep track of where it left off.
+ *
+ *  \retval txptr or NULL if no more txs in list
+ */
+static AppLayerGetTxIterTuple AppLayerDefaultGetTxIterator(
+        const uint8_t ipproto, const AppProto alproto,
+        void *alstate, uint64_t min_tx_id, uint64_t max_tx_id,
+        AppLayerGetTxIterState *state)
+{
+    uint64_t ustate = *(uint64_t *)state;
+    uint64_t tx_id = MAX(min_tx_id, ustate);
+    for ( ; tx_id < max_tx_id; tx_id++) {
+        void *tx_ptr = AppLayerParserGetTx(ipproto, alproto, alstate, tx_id);
+        if (tx_ptr != NULL) {
+            ustate = tx_id + 1;
+            *state = *(AppLayerGetTxIterState *)&ustate;
+            AppLayerGetTxIterTuple tuple = {
+                .tx_ptr = tx_ptr,
+                .tx_id = tx_id,
+                .has_next = (tx_id + 1 < max_tx_id),
+            };
+            SCLogDebug("tulpe: %p/%"PRIu64"/%s", tuple.tx_ptr, tuple.tx_id,
+                    tuple.has_next ? "true" : "false");
+            return tuple;
+        }
+    }
+
+    AppLayerGetTxIterTuple no_tuple = { NULL, 0, false };
+    return no_tuple;
+}
+
+AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto,
+        const AppProto alproto)
+{
+    AppLayerGetTxIteratorFunc Func =
+        alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetTxIterator;
+    return Func ? Func : AppLayerDefaultGetTxIterator;
+}
+
 void AppLayerParserSetTxLogged(uint8_t ipproto, AppProto alproto,
                                void *alstate, void *tx, LoggerId logger)
 {
@@ -677,28 +729,37 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p
     const uint8_t ipproto = f->proto;
     const AppProto alproto = f->alproto;
 
+    AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
+    AppLayerGetTxIterState state;
+    memset(&state, 0, sizeof(state));
+
     SCLogDebug("called: %s, tag_txs_as_inspected %s",direction==0?"toserver":"toclient",
             tag_txs_as_inspected?"true":"false");
 
     /* mark all txs as inspected if the applayer progress is
      * at the 'end state'. */
-    for (; idx < total_txs; idx++) {
-        void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx);
-        if (tx == NULL)
-            continue;
+    while (1) {
+        AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, idx, total_txs, &state);
+        if (ires.tx_ptr == NULL)
+            break;
+
+        void *tx = ires.tx_ptr;
+        idx = ires.tx_id;
+
         int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
-        if (state_progress >= state_done_progress) {
-            if (tag_txs_as_inspected) {
-                uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
-                if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
-                    detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
-                    AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
-                    SCLogDebug("%p/%"PRIu64" in-order tx is done for direction %s. Flag %016"PRIx64,
-                            tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
-                }
+        if (state_progress < state_done_progress)
+            break;
+
+        if (tag_txs_as_inspected) {
+            uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
+            if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
+                detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
+                AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
+                SCLogDebug("%p/%"PRIu64" in-order tx is done for direction %s. Flag %016"PRIx64,
+                        tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
             }
-            continue;
-        } else
+        }
+        if (!ires.has_next)
             break;
     }
     pstate->inspect_id[direction] = idx;
@@ -707,29 +768,39 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p
     /* if necessary we flag all txs that are complete as 'inspected'
      * also move inspect_id forward. */
     if (tag_txs_as_inspected) {
-        for (; idx < total_txs; idx++) {
-            bool check_inspect_id = false;
-            void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx);
-            if (tx == NULL) {
-                check_inspect_id = true;
-            } else {
-                int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
-                if (state_progress >= state_done_progress) {
-                    uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
-                    if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
-                        detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
-                        AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
-                        SCLogDebug("%p/%"PRIu64" out of order tx is done for direction %s. Flag %016"PRIx64,
-                                tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
-                        check_inspect_id = true;
-                    }
-                }
+        /* continue at idx */
+        while (1) {
+            AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, idx, total_txs, &state);
+            if (ires.tx_ptr == NULL)
+                break;
+
+            void *tx = ires.tx_ptr;
+            /* if we got a higher id than the minimum we requested, we
+             * skipped a bunch of 'null-txs'. Lets see if we can up the
+             * inspect tracker */
+            if (ires.tx_id > idx && pstate->inspect_id[direction] == idx) {
+                pstate->inspect_id[direction] = ires.tx_id;
             }
-            if (check_inspect_id) {
-                SCLogDebug("%p/%"PRIu64" out of order tx. Update inspect_id? %"PRIu64, tx, idx, pstate->inspect_id[direction]);
+            idx = ires.tx_id;
+
+            const int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
+            if (state_progress < state_done_progress)
+                break;
+
+            uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
+            if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
+                detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
+                AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
+                SCLogDebug("%p/%"PRIu64" out of order tx is done for direction %s. Flag %016"PRIx64,
+                        tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
+
+                SCLogDebug("%p/%"PRIu64" out of order tx. Update inspect_id? %"PRIu64,
+                        tx, idx, pstate->inspect_id[direction]);
                 if (pstate->inspect_id[direction]+1 == idx)
                     pstate->inspect_id[direction] = idx;
             }
+            if (!ires.has_next)
+                break;
         }
     }
 
@@ -808,31 +879,38 @@ void AppLayerParserTransactionsCleanup(Flow *f)
     const int tx_end_state_ts = AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOSERVER);
     const int tx_end_state_tc = AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOCLIENT);
 
-    SCLogDebug("checking %"PRIu64" txs from offset %"PRIu64, total_txs, min);
-    for (uint64_t i = min ; i < total_txs; i++) {
-        void * const tx = AppLayerParserGetTx(ipproto, alproto, alstate, i);
-        if (tx == NULL) {
-            SCLogDebug("%p/%"PRIu64" skipping: no tx", tx, i);
-            goto wrap_up;
-        }
+    AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
+    AppLayerGetTxIterState state;
+    memset(&state, 0, sizeof(state));
+    uint64_t i = min;
+    uint64_t new_min = min;
+
+    while (1) {
+        AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, i, total_txs, &state);
+        if (ires.tx_ptr == NULL)
+            break;
+
+        void *tx = ires.tx_ptr;
+        i = ires.tx_id;
+
         SCLogDebug("%p/%"PRIu64" checking", tx, i);
 
         const int tx_progress_tc = AppLayerParserGetStateProgress(ipproto, alproto, tx, STREAM_TOCLIENT);
         if (tx_progress_tc < tx_end_state_tc) {
             SCLogDebug("%p/%"PRIu64" skipping: tc parser not done", tx, i);
-            continue;
+            goto next;
         }
         const int tx_progress_ts = AppLayerParserGetStateProgress(ipproto, alproto, tx, STREAM_TOSERVER);
         if (tx_progress_ts < tx_end_state_ts) {
             SCLogDebug("%p/%"PRIu64" skipping: ts parser not done", tx, i);
-            continue;
+            goto next;
         }
         if (f->sgh_toserver != NULL) {
             uint64_t detect_flags_ts = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, STREAM_TOSERVER);
             if (!(detect_flags_ts & APP_LAYER_TX_INSPECTED_FLAG)) {
                 SCLogDebug("%p/%"PRIu64" skipping: TS inspect not done: ts:%"PRIx64,
                         tx, i, detect_flags_ts);
-                continue;
+                goto next;
             }
         }
         if (f->sgh_toclient != NULL) {
@@ -840,7 +918,7 @@ void AppLayerParserTransactionsCleanup(Flow *f)
             if (!(detect_flags_tc & APP_LAYER_TX_INSPECTED_FLAG)) {
                 SCLogDebug("%p/%"PRIu64" skipping: TC inspect not done: tc:%"PRIx64,
                         tx, i, detect_flags_tc);
-                continue;
+                goto next;
             }
         }
         if (logger_expectation != 0) {
@@ -848,25 +926,32 @@ void AppLayerParserTransactionsCleanup(Flow *f)
             if (tx_logged != logger_expectation) {
                 SCLogDebug("%p/%"PRIu64" skipping: logging not done: want:%"PRIx32", have:%"PRIx32,
                         tx, i, logger_expectation, tx_logged);
-                continue;
+                goto next;
             }
         }
 
         /* if we are here, the tx can be freed. */
         p->StateTransactionFree(alstate, i);
         SCLogDebug("%p/%"PRIu64" freed", tx, i);
-    wrap_up:
-        /* see if this tx is actually in order. If so, we need to bring all
-         * trackers up to date. */
-        SCLogDebug("%p/%"PRIu64" update f->alparser->min_id? %"PRIu64, tx, i, alparser->min_id);
-        if (i == alparser->min_id) {
-            uint64_t next_id = i + 1;
-            alparser->min_id = next_id;
-            alparser->inspect_id[0] = MAX(alparser->inspect_id[0], next_id);
-            alparser->inspect_id[1] = MAX(alparser->inspect_id[1], next_id);
-            alparser->log_id = MAX(alparser->log_id, next_id);
-            SCLogDebug("%p/%"PRIu64" updated f->alparser->min_id %"PRIu64, tx, i, alparser->min_id);
-        }
+
+        /* if this tx was the minimum, up the minimum */
+        if (i == new_min)
+            new_min = i + 1;
+
+next:
+        if (!ires.has_next)
+            break;
+    }
+
+    /* see if we need to bring all trackers up to date. */
+    SCLogDebug("update f->alparser->min_id? %"PRIu64, alparser->min_id);
+    if (new_min > alparser->min_id) {
+        const uint64_t next_id = new_min;
+        alparser->min_id = next_id;
+        alparser->inspect_id[0] = MAX(alparser->inspect_id[0], next_id);
+        alparser->inspect_id[1] = MAX(alparser->inspect_id[1], next_id);
+        alparser->log_id = MAX(alparser->log_id, next_id);
+        SCLogDebug("updated f->alparser->min_id %"PRIu64, alparser->min_id);
     }
 }
 

src/app-layer-parser.h

@@ -92,6 +92,25 @@ typedef int (*AppLayerParserFPtr)(Flow *f, void *protocol_state,
         uint8_t *buf, uint32_t buf_len,
         void *local_storage);
 
+typedef struct AppLayerGetTxIterTuple {
+    void *tx_ptr;
+    uint64_t tx_id;
+    bool has_next;
+} AppLayerGetTxIterTuple;
+
+typedef struct AppLayerGetTxIterState {
+    union {
+        void *ptr;
+        uint64_t u64;
+    } un;
+} AppLayerGetTxIterState;
+
+/** \brief tx iterator prototype */
+typedef AppLayerGetTxIterTuple (*AppLayerGetTxIteratorFunc)
+       (const uint8_t ipproto, const AppProto alproto,
+        void *alstate, uint64_t min_tx_id, uint64_t max_tx_id,
+        AppLayerGetTxIterState *state);
+
 /***** Parser related registration *****/
 
 /**
@@ -135,6 +154,8 @@ void AppLayerParserRegisterGetTxCnt(uint8_t ipproto, AppProto alproto,
                          uint64_t (*StateGetTxCnt)(void *alstate));
 void AppLayerParserRegisterGetTx(uint8_t ipproto, AppProto alproto,
                       void *(StateGetTx)(void *alstate, uint64_t tx_id));
+void AppLayerParserRegisterGetTxIterator(uint8_t ipproto, AppProto alproto,
+                      AppLayerGetTxIteratorFunc Func);
 void AppLayerParserRegisterGetStateProgressCompletionStatus(AppProto alproto,
     int (*StateGetStateProgressCompletionStatus)(uint8_t direction));
 void AppLayerParserRegisterGetEventInfo(uint8_t ipproto, AppProto alproto,
@@ -156,6 +177,9 @@ void AppLayerParserRegisterDetectFlagsFuncs(uint8_t ipproto, AppProto alproto,
 
 /***** Get and transaction functions *****/
 
+AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto,
+         const AppProto alproto);
+
 void *AppLayerParserGetProtocolParserLocalStorage(uint8_t ipproto, AppProto alproto);
 void AppLayerParserDestroyProtocolParserLocalStorage(uint8_t ipproto, AppProto alproto,
                                           void *local_data);

src/detect-nfs-procedure.c

@@ -34,6 +34,8 @@
 #include "detect-pcre.h"
 #include "detect-nfs-procedure.h"
 
+#include "app-layer-parser.h"
+
 #include "flow.h"
 #include "flow-util.h"
 #include "flow-var.h"

src/detect-nfs-version.c

@@ -34,6 +34,8 @@
 #include "detect-pcre.h"
 #include "detect-nfs-version.h"
 
+#include "app-layer-parser.h"
+
 #include "flow.h"
 #include "flow-util.h"
 #include "flow-var.h"

src/detect.c

@@ -1302,15 +1302,10 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
  *  \brief get a DetectTransaction object
  *  \retval struct filled with relevant info or all nulls/0s
  */
-static DetectTransaction GetTx(const uint8_t ipproto, const AppProto alproto,
-        void *alstate, const uint64_t tx_id, const int tx_end_state,
+static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alproto,
+        void *alstate, const uint64_t tx_id, void *tx_ptr, const int tx_end_state,
         const uint8_t flow_flags)
 {
-    void *tx_ptr = AppLayerParserGetTx(ipproto, alproto, alstate, tx_id);
-    if (tx_ptr == NULL) {
-        DetectTransaction no_tx = { NULL, 0, NULL, 0, 0, 0, 0, 0, };
-        return no_tx;
-    }
     const uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx_ptr, flow_flags);
     if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) {
         SCLogDebug("%"PRIu64" tx already fully inspected for %s. Flags %016"PRIx64,
@@ -1356,13 +1351,21 @@ static void DetectRunTx(ThreadVars *tv,
     uint64_t tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags);
     const int tx_end_state = AppLayerParserGetStateProgressCompletionStatus(alproto, flow_flags);
 
-    for ( ; tx_id < total_txs; tx_id++) {
-        DetectTransaction tx = GetTx(ipproto, alproto,
-                alstate, tx_id, tx_end_state, flow_flags);
+    AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
+    AppLayerGetTxIterState state;
+    memset(&state, 0, sizeof(state));
+
+    while (1) {
+        AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
+        if (ires.tx_ptr == NULL)
+            break;
+
+        DetectTransaction tx = GetDetectTx(ipproto, alproto,
+                alstate, ires.tx_id, ires.tx_ptr, tx_end_state, flow_flags);
         if (tx.tx_ptr == NULL) {
             SCLogDebug("%p/%"PRIu64" no transaction to inspect",
                     tx.tx_ptr, tx_id);
-            continue;
+            goto next;
         }
 
         uint32_t array_idx = 0;
@@ -1571,6 +1574,9 @@ static void DetectRunTx(ThreadVars *tv,
             AppLayerParserSetTxDetectFlags(ipproto, alproto, tx.tx_ptr,
                     flow_flags, new_detect_flags);
         }
+next:
+        if (!ires.has_next)
+            break;
     }
 }
 

src/output-tx.c

@@ -140,6 +140,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
         return TM_ECODE_OK;
 
     Flow * const f = p->flow;
+    const uint8_t ipproto = f->proto;
     const AppProto alproto = f->alproto;
 
     if (AppLayerParserProtocolIsTxAware(p->proto, alproto) == 0)
@@ -164,20 +165,23 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
     int logged = 0;
     int gap = 0;
 
-    for (; tx_id < total_txs; tx_id++)
-    {
-        void *tx = AppLayerParserGetTx(p->proto, alproto, alstate, tx_id);
-        if (tx == NULL) {
-            SCLogDebug("tx is NULL not logging");
-            continue;
-        }
+    AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
+    AppLayerGetTxIterState state;
+    memset(&state, 0, sizeof(state));
+
+    while (1) {
+        AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
+        if (ires.tx_ptr == NULL)
+            break;
+        void * const tx = ires.tx_ptr;
+        tx_id = ires.tx_id;
 
         LoggerId tx_logged = AppLayerParserGetTxLogged(f, alstate, tx);
         const LoggerId tx_logged_old = tx_logged;
         SCLogDebug("logger: expect %08x, have %08x", logger_expectation, tx_logged);
         if (tx_logged == logger_expectation) {
             /* tx already fully logged */
-            continue;
+            goto next_tx;
         }
 
         int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto,
@@ -211,17 +215,17 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
                         int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
                         if (r == FALSE) {
                             SCLogDebug("conditions not met, not logging");
-                            goto next;
+                            goto next_logger;
                         }
                     } else {
                         if (tx_progress_tc < logger->tc_log_progress) {
                             SCLogDebug("progress not far enough, not logging");
-                            goto next;
+                            goto next_logger;
                         }
 
                         if (tx_progress_ts < logger->ts_log_progress) {
                             SCLogDebug("progress not far enough, not logging");
-                            goto next;
+                            goto next_logger;
                         }
                     }
                 }
@@ -235,7 +239,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
                 tx_logged |= (1<<logger->logger_id);
             }
 
-next:
+next_logger:
             logger = logger->next;
             store = store->next;
 #ifdef DEBUG_VALIDATION
@@ -263,6 +267,9 @@ next:
         } else {
             gap = 1;
         }
+next_tx:
+        if (!ires.has_next)
+            break;
     }
 
     /* Update the the last ID that has been logged with all