detect/lua: fix per-thread mem limit disabling

An edge case existed where a lua detect script might fail to reenable
the mem limit after temporarily clearing it to allow for set up. This
would then be persistent for the lifetime of that thread.

Make sure to test these cases before the reset.

Ticket: #8805.

src/detect-lua.c

@@ -311,6 +311,13 @@ static int DetectLuaMatch (DetectEngineThreadCtx *det_ctx,
     else if (p->flowflags & FLOW_PKT_TOCLIENT)
         flags = STREAM_TOCLIENT;
 
+    /* bail early if we're not going to run inspection, avoid running the
+     * reset/restore logic at all. */
+    if ((tlua->flags & FLAG_DATATYPE_PAYLOAD) && p->payload_len == 0)
+        SCReturnInt(0);
+    if ((tlua->flags & FLAG_DATATYPE_PACKET) && GET_PKT_LEN(p) == 0)
+        SCReturnInt(0);
+
     /* disable bytes limit temporarily to allow the setup of buffer and other data the script will
      * use. */
     const uint64_t cfg_limit = SCLuaSbResetBytesLimit(tlua->luastate);
@@ -319,11 +326,6 @@ static int DetectLuaMatch (DetectEngineThreadCtx *det_ctx,
 
     LuaExtensionsMatchSetup(tlua->luastate, lua, det_ctx, p->flow, p, s, flags);
 
-    if ((tlua->flags & FLAG_DATATYPE_PAYLOAD) && p->payload_len == 0)
-        SCReturnInt(0);
-    if ((tlua->flags & FLAG_DATATYPE_PACKET) && GET_PKT_LEN(p) == 0)
-        SCReturnInt(0);
-
     lua_getglobal(tlua->luastate, "match");
     lua_newtable(tlua->luastate); /* stack at -1 */