lua: NULL-check update/finalize hasher functions

The commit guards Suricata from crashing when the hasher
is incorrectly accessed after it is finalized.

Ticket: 8814

src/util-lua-hashlib.c

@@ -72,6 +72,9 @@ static int LuaHashLibSha256Update(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "sha256 hasher already finalized");
+    }
     size_t data_len;
     const char *data = luaL_checklstring(L, 2, &data_len);
     SCSha256Update(*hasher, (const uint8_t *)data, (uint32_t)data_len);
@@ -84,6 +87,9 @@ static int LuaHashLibSha256Finalize(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "sha256 hasher already finalized");
+    }
 
     uint8_t hash[SC_SHA256_LEN];
     SCSha256Finalize(*hasher, hash, sizeof(hash));
@@ -102,6 +108,9 @@ static int LuaHashLibSha256FinalizeToHex(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "sha256 hasher already finalized");
+    }
 
     char hash[SC_SHA256_HEX_LEN + 1];
     if (!SCSha256FinalizeToHex(*hasher, hash, sizeof(hash))) {
@@ -174,6 +183,9 @@ static int LuaHashLibSha1Update(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "sha1 hasher already finalized");
+    }
 
     size_t data_len;
     const char *data = luaL_checklstring(L, 2, &data_len);
@@ -187,6 +199,9 @@ static int LuaHashLibSha1Finalize(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "sha1 hasher already finalized");
+    }
 
     uint8_t hash[SC_SHA1_LEN];
     SCSha1Finalize(*hasher, hash, sizeof(hash));
@@ -205,6 +220,9 @@ static int LuaHashLibSha1FinalizeToHex(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "sha1 hasher already finalized");
+    }
 
     char hash[SC_SHA1_HEX_LEN + 1];
     if (!SCSha1FinalizeToHex(*hasher, hash, sizeof(hash))) {
@@ -276,6 +294,9 @@ static int LuaHashLibMd5Update(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "md5 hasher already finalized");
+    }
 
     size_t data_len;
     const char *data = luaL_checklstring(L, 2, &data_len);
@@ -289,6 +310,9 @@ static int LuaHashLibMd5Finalize(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "md5 hasher already finalized");
+    }
 
     uint8_t hash[SC_MD5_LEN];
     SCMd5Finalize(*hasher, hash, sizeof(hash));
@@ -307,6 +331,9 @@ static int LuaHashLibMd5FinalizeToHex(lua_State *L)
     if (hasher == NULL) {
         return luaL_error(L, "null userdata");
     }
+    if (*hasher == NULL) {
+        return luaL_error(L, "md5 hasher already finalized");
+    }
 
     char hash[SC_MD5_HEX_LEN + 1];
     if (!SCMd5FinalizeToHex(*hasher, hash, sizeof(hash))) {