Fix luajit compilation failure introduced by the transaction update.

Fix coverity lock issues reported by transaction update as well.

src/app-layer-parser.c

@@ -1154,7 +1154,9 @@ void AppLayerTransactionUpdateInspectId(Flow *f, uint8_t flags)
 
     ((AppLayerParserStateStore *)f->alparser)->inspect_id[direction] = idx;
     if (tx_updated_by > 0) {
+        SCMutexLock(&f->de_state_m);
         DetectEngineStateReset(f->de_state, flags);
+        SCMutexUnlock(&f->de_state_m);
     }
 
     return;

src/detect-engine-state.c

@@ -416,6 +416,7 @@ void DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
 
     DetectEngineStateDirection *dir_state = &f->de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1];
     DeStateStore *store = dir_state->head;
+    void *inspect_tx = NULL;
     uint64_t inspect_tx_id = 0;
     uint64_t total_txs = 0;
     uint8_t alproto_supports_txs = 0;
@@ -423,8 +424,10 @@ void DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
     DeStateResetFileInspection(f, alproto, alstate, flags);
 
     if (AppLayerAlprotoSupportsTxs(alproto)) {
+        FLOWLOCK_RDLOCK(f);
         inspect_tx_id = AppLayerTransactionGetInspectId(f, flags);
         total_txs = AppLayerGetTxCnt(alproto, alstate);
+        FLOWLOCK_UNLOCK(f);
         alproto_supports_txs = 1;
     }
 
@@ -505,11 +508,9 @@ void DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx,
                 }
 
                 engine = app_inspection_engine[alproto][(flags & STREAM_TOSERVER) ? 0 : 1];
-                void *inspect_tx = AppLayerGetTx(alproto, alstate, inspect_tx_id);
-                if (inspect_tx == NULL) {
-                    FLOWLOCK_UNLOCK(f);
-                    goto end;
-                }
+                inspect_tx = AppLayerGetTx(alproto, alstate, inspect_tx_id);
+                if (inspect_tx == NULL)
+                    continue;
                 while (engine != NULL) {
                     if (!(item->flags & engine->inspect_flags) &&
                         s->sm_lists[engine->sm_list] != NULL)

src/detect-luajit.c

@@ -48,6 +48,7 @@
 #include "util-unittest-helper.h"
 
 #include "app-layer.h"
+#include "app-layer-parser.h"
 
 #include "stream-tcp.h"
 
@@ -380,14 +381,11 @@ static int DetectLuajitMatch (ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
         FLOWLOCK_RDLOCK(p->flow);
         HtpState *htp_state = p->flow->alstate;
         if (htp_state != NULL && htp_state->connp != NULL && htp_state->connp->conn != NULL) {
-            uint64_t idx = AppLayerTransactionGetInspectId(p->flow, flags);
-            if (idx != -1) {
             htp_tx_t *tx = NULL;
-
-                uint64_t total_txs= AppLayerGetNoOfTxs(ALPROTO_HTTP, htp_state);
-                for ( ; idx < size; idx++)
-                {
-                    tx = AppLayerGetTx(http_state, idx);
+            uint64_t idx = AppLayerTransactionGetInspectId(p->flow, 0);
+            uint64_t total_txs= AppLayerGetTxCnt(ALPROTO_HTTP, htp_state);
+            for ( ; idx < total_txs; idx++) {
+                tx = AppLayerGetTx(ALPROTO_HTTP, htp_state, idx);
                 if (tx == NULL)
                     continue;
 
@@ -401,7 +399,6 @@ static int DetectLuajitMatch (ThreadVars *tv, DetectEngineThreadCtx *det_ctx,
                 }
             }
         }
-        }
         FLOWLOCK_UNLOCK(p->flow);
     }