From 3da7dad514fff3469d14f8eb9993062cabaee523 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 17 Nov 2016 08:54:44 +0100 Subject: [PATCH] lua: luajit improvements Luajit has a strange memory requirement, it's 'states' need to be in the first 2G of the process' memory. This patch improves the pool approach by moving it to the front of the start up. A new config option 'luajit.states' is added to control how many states are preallocated. It defaults to 128. Add a warning when more states are used then preallocated. This may fail if flow/stream/detect engines use a lot of memory. Add hint at exit that gives the max states in use if it's higher than the default. --- src/Makefile.am | 1 + src/detect-lua.c | 114 +---------------------------- src/detect-lua.h | 8 --- src/detect.c | 9 +-- src/output-lua.c | 12 ++-- src/runmode-unittests.c | 9 +++ src/suricata-common.h | 6 ++ src/suricata.c | 13 +++- src/util-lua.c | 22 ++++++ src/util-lua.h | 5 ++ src/util-luajit.c | 155 ++++++++++++++++++++++++++++++++++++++++ src/util-luajit.h | 36 ++++++++++ 12 files changed, 256 insertions(+), 134 deletions(-) create mode 100644 src/util-luajit.c create mode 100644 src/util-luajit.h diff --git a/src/Makefile.am b/src/Makefile.am index 708ecb7a62..48f5dd0ee0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -378,6 +378,7 @@ util-ip.h util-ip.c \ util-logopenfile.h util-logopenfile.c \ util-logopenfile-tile.h util-logopenfile-tile.c \ util-lua.c util-lua.h \ +util-luajit.c util-luajit.h \ util-lua-common.c util-lua-common.h \ util-lua-dnp3.c util-lua-dnp3.h \ util-lua-dnp3-objects.c util-lua-dnp3-objects.h \ diff --git a/src/detect-lua.c b/src/detect-lua.c index d77ee4edd5..a1ce743285 100644 --- a/src/detect-lua.c +++ b/src/detect-lua.c @@ -85,26 +85,6 @@ void DetectLuaRegister(void) #else /* HAVE_LUA */ -#ifdef HAVE_LUAJIT -#include "util-pool.h" - -/** \brief lua_State pool - * - * Lua requires states to be alloc'd in memory <2GB. For this reason we - * prealloc the states early during engine startup so we have a better chance - * of getting the states. We protect the pool with a lock as the detect - * threads access it during their init and cleanup. - * - * Pool size is automagically determined based on number of keyword occurences, - * cpus/cores and rule reloads being enabled or not. - * - * Alternatively, the "detect-engine.luajit-states" var can be set. - */ -static Pool *luajit_states = NULL; -static pthread_mutex_t luajit_states_lock = SCMUTEX_INITIALIZER; - -#endif /* HAVE_LUAJIT */ - #include "util-lua.h" static int DetectLuaMatch (ThreadVars *, DetectEngineThreadCtx *, @@ -170,94 +150,6 @@ void DetectLuaRegister(void) #define DATATYPE_DNP3 (1<<20) -#ifdef HAVE_LUAJIT -static void *LuaStatePoolAlloc(void) -{ - return luaL_newstate(); -} - -static void LuaStatePoolFree(void *d) -{ - lua_State *s = (lua_State *)d; - if (s != NULL) - lua_close(s); -} - -/** \brief Populate lua states pool - * - * \param num keyword instances - * \param reloads bool indicating we have rule reloads enabled - */ -int DetectLuajitSetupStatesPool(int num, int reloads) -{ - int retval = 0; - pthread_mutex_lock(&luajit_states_lock); - - if (luajit_states == NULL) { - intmax_t cnt = 0; - ConfNode *denode = NULL; - ConfNode *decnf = ConfGetNode("detect-engine"); - if (decnf != NULL) { - TAILQ_FOREACH(denode, &decnf->head, next) { - if (strcmp(denode->val, "luajit-states") == 0) { - ConfGetChildValueInt(denode, "luajit-states", &cnt); - } - } - } - - if (cnt == 0) { - int cpus = UtilCpuGetNumProcessorsOnline(); - if (cpus == 0) { - cpus = 10; - } - cnt = num * cpus; - cnt *= 3; /* assume 3 threads per core */ - - /* alloc a bunch extra so reload can add new rules/instances */ - if (reloads) - cnt *= 5; - } - - luajit_states = PoolInit(0, cnt, 0, LuaStatePoolAlloc, NULL, NULL, NULL, LuaStatePoolFree); - if (luajit_states == NULL) { - SCLogError(SC_ERR_LUA_ERROR, "luastate pool init failed, lua/luajit keywords won't work"); - retval = -1; - } - } - - pthread_mutex_unlock(&luajit_states_lock); - return retval; -} -#endif /* HAVE_LUAJIT */ - -static lua_State *DetectLuaGetState(void) -{ - - lua_State *s = NULL; -#ifdef HAVE_LUAJIT - pthread_mutex_lock(&luajit_states_lock); - if (luajit_states != NULL) - s = (lua_State *)PoolGet(luajit_states); - pthread_mutex_unlock(&luajit_states_lock); -#else - s = luaL_newstate(); -#endif - return s; -} - -static void DetectLuaReturnState(lua_State *s) -{ - if (s != NULL) { -#ifdef HAVE_LUAJIT - pthread_mutex_lock(&luajit_states_lock); - PoolReturn(luajit_states, (void *)s); - pthread_mutex_unlock(&luajit_states_lock); -#else - lua_close(s); -#endif - } -} - /** \brief dump stack from lua state to screen */ void LuaDumpStack(lua_State *state) { @@ -691,7 +583,7 @@ static void *DetectLuaThreadInit(void *data) t->alproto = luajit->alproto; t->flags = luajit->flags; - t->luastate = DetectLuaGetState(); + t->luastate = LuaGetState(); if (t->luastate == NULL) { SCLogError(SC_ERR_LUA_ERROR, "luastate pool depleted"); goto error; @@ -737,7 +629,7 @@ static void *DetectLuaThreadInit(void *data) error: if (t->luastate != NULL) - DetectLuaReturnState(t->luastate); + LuaReturnState(t->luastate); SCFree(t); return NULL; } @@ -747,7 +639,7 @@ static void DetectLuaThreadFree(void *ctx) if (ctx != NULL) { DetectLuaThreadData *t = (DetectLuaThreadData *)ctx; if (t->luastate != NULL) - DetectLuaReturnState(t->luastate); + LuaReturnState(t->luastate); SCFree(t); } } diff --git a/src/detect-lua.h b/src/detect-lua.h index f1e86c2f2e..d1259b3e66 100644 --- a/src/detect-lua.h +++ b/src/detect-lua.h @@ -26,10 +26,6 @@ #ifdef HAVE_LUA -#include -#include -#include - typedef struct DetectLuaThreadData { lua_State *luastate; uint32_t flags; @@ -63,10 +59,6 @@ int DetectLuaMatchBuffer(DetectEngineThreadCtx *det_ctx, Signature *s, SigMatch uint8_t *buffer, uint32_t buffer_len, uint32_t offset, Flow *f); -#ifdef HAVE_LUAJIT -int DetectLuajitSetupStatesPool(int num, int reloads); -#endif /* HAVE_LUAJIT */ - void DetectLuaPostSetup(Signature *s); #endif /* __DETECT_FILELUAJIT_H__ */ diff --git a/src/detect.c b/src/detect.c index 27047c5c0a..e75e393dd7 100644 --- a/src/detect.c +++ b/src/detect.c @@ -200,6 +200,7 @@ #include "stream-tcp.h" #include "stream-tcp-inline.h" +#include "util-lua.h" #include "util-var-name.h" #include "util-classification-config.h" #include "util-print.h" @@ -3366,14 +3367,6 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) "preprocessing rules..."); } -#ifdef HAVE_LUAJIT - /* run this before the mpm states are initialized */ - if (DetectLuajitSetupStatesPool(de_ctx->detect_luajit_instances, TRUE) != 0) { - if (de_ctx->failure_fatal) - return -1; - } -#endif - de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx); de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *)); de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size); diff --git a/src/output-lua.c b/src/output-lua.c index b71b0e3736..d9a6dc497a 100644 --- a/src/output-lua.c +++ b/src/output-lua.c @@ -521,7 +521,7 @@ typedef struct LogLuaScriptOptions_ { static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) { int status; - lua_State *luastate = luaL_newstate(); + lua_State *luastate = LuaGetState(); if (luastate == NULL) goto error; luaL_openlibs(luastate); @@ -649,11 +649,11 @@ static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) { /* pop the table */ lua_pop(luastate, 1); - lua_close(luastate); + LuaReturnState(luastate); return 0; error: if (luastate) - lua_close(luastate); + LuaReturnState(luastate); return -1; } @@ -665,7 +665,7 @@ error: */ static lua_State *LuaScriptSetup(const char *filename) { - lua_State *luastate = luaL_newstate(); + lua_State *luastate = LuaGetState(); if (luastate == NULL) { SCLogError(SC_ERR_LUA_ERROR, "luaL_newstate failed"); goto error; @@ -720,7 +720,7 @@ static lua_State *LuaScriptSetup(const char *filename) return luastate; error: if (luastate) - lua_close(luastate); + LuaReturnState(luastate); return NULL; } @@ -941,7 +941,7 @@ static void OutputLuaLogDoDeinit(LogLuaCtx *lua_ctx) SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'deinit' function: %s", lua_tostring(luastate, -1)); return; } - lua_close(luastate); + LuaReturnState(luastate); } /** \internal diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index 7c97d28629..4f03f18486 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -121,6 +121,7 @@ #include "detect-engine-siggroup.h" #include "util-streaming-buffer.h" +#include "util-lua.h" #endif /* UNITTESTS */ @@ -143,6 +144,11 @@ void RunUnittests(int list_unittests, char *regex_arg) GlobalInits(); TimeInit(); SupportFastPatternForSigMatchTypes(); +#ifdef HAVE_LUAJIT + if (LuajitSetupStatesPool() != 0) { + exit(EXIT_FAILURE); + } +#endif default_packet_size = DEFAULT_PACKET_SIZE; #ifdef __SC_CUDA_SUPPORT__ @@ -309,6 +315,9 @@ void RunUnittests(int list_unittests, char *regex_arg) } } +#ifdef HAVE_LUAJIT + LuajitFreeStatesPool(); +#endif #ifdef DBG_MEM_ALLOC SCLogInfo("Total memory used (without SCFree()): %"PRIdMAX, (intmax_t)global_mem); #endif diff --git a/src/suricata-common.h b/src/suricata-common.h index a2c9414a16..5a40156b16 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -376,6 +376,12 @@ typedef enum { #include "util-path.h" #include "util-conf.h" +#ifdef HAVE_LUA +#include +#include +#include +#endif + #ifndef HAVE_STRLCAT size_t strlcat(char *, const char *src, size_t siz); #endif diff --git a/src/suricata.c b/src/suricata.c index ac22824280..292ffd708e 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -170,6 +170,8 @@ #include "util-storage.h" #include "host-storage.h" +#include "util-lua.h" + /* * we put this here, because we only use it here in main. */ @@ -2221,6 +2223,13 @@ static int PostConfLoadedSetup(SCInstance *suri) { char *hostmode = NULL; + /* do this as early as possible #1577 #1955 */ +#ifdef HAVE_LUAJIT + if (LuajitSetupStatesPool() != 0) { + SCReturnInt(TM_ECODE_FAILED); + } +#endif + /* load the pattern matchers */ MpmTableSetup(); #ifdef __SC_CUDA_SUPPORT__ @@ -2782,7 +2791,9 @@ int main(int argc, char **argv) CudaHandlerFreeProfiles(); #endif ConfDeInit(); - +#ifdef HAVE_LUAJIT + LuajitFreeStatesPool(); +#endif SCLogDeInitLogModule(); DetectParseFreeRegexes(); exit(engine_retval); diff --git a/src/util-lua.c b/src/util-lua.c index 7f1d44348c..bc0c20c887 100644 --- a/src/util-lua.c +++ b/src/util-lua.c @@ -56,6 +56,28 @@ #include "util-lua.h" +lua_State *LuaGetState(void) +{ + lua_State *s = NULL; +#ifdef HAVE_LUAJIT + s = LuajitGetState(); +#else + s = luaL_newstate(); +#endif + return s; +} + +void LuaReturnState(lua_State *s) +{ + if (s != NULL) { +#ifdef HAVE_LUAJIT + LuajitReturnState(s); +#else + lua_close(s); +#endif + } +} + /* key for tv (threadvars) pointer */ const char lua_ext_key_tv[] = "suricata:lua:tv:ptr"; /* key for tx pointer */ diff --git a/src/util-lua.h b/src/util-lua.h index 30431e74e8..d6a9838cd8 100644 --- a/src/util-lua.h +++ b/src/util-lua.h @@ -26,12 +26,17 @@ #ifdef HAVE_LUA +#include "util-luajit.h" + typedef struct LuaStreamingBuffer_ { const uint8_t *data; uint32_t data_len; uint8_t flags; } LuaStreamingBuffer; +lua_State *LuaGetState(void); +void LuaReturnState(lua_State *s); + /* gets */ /** \brief get tv pointer from the lua state */ diff --git a/src/util-luajit.c b/src/util-luajit.c new file mode 100644 index 0000000000..cb11dc79af --- /dev/null +++ b/src/util-luajit.c @@ -0,0 +1,155 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + */ + +#include "suricata-common.h" + +#ifdef HAVE_LUAJIT +#include "conf.h" +#include "util-pool.h" +#include "util-lua.h" + +/** \brief lua_State pool + * + * Lua requires states to be alloc'd in memory <2GB. For this reason we + * prealloc the states early during engine startup so we have a better chance + * of getting the states. We protect the pool with a lock as the detect + * threads access it during their init and cleanup. + * + * Pool size is automagically determined based on number of keyword occurences, + * cpus/cores and rule reloads being enabled or not. + * + * Alternatively, the "detect-engine.luajit-states" var can be set. + */ +static Pool *luajit_states = NULL; +static pthread_mutex_t luajit_states_lock = SCMUTEX_INITIALIZER; +static int luajit_states_cnt = 0; +static int luajit_states_cnt_max = 0; +static int luajit_states_size = 0; +#define LUAJIT_DEFAULT_STATES 128 + +static void *LuaStatePoolAlloc(void) +{ + return luaL_newstate(); +} + +static void LuaStatePoolFree(void *d) +{ + lua_State *s = (lua_State *)d; + if (s != NULL) + lua_close(s); +} + +/** \brief Populate lua states pool + * + * \param num keyword instances + * \param reloads bool indicating we have rule reloads enabled + */ +int LuajitSetupStatesPool(void) +{ + int retval = 0; + pthread_mutex_lock(&luajit_states_lock); + + if (luajit_states == NULL) { + intmax_t cnt = 0; + if (ConfGetInt("luajit.states", &cnt) != 1) { + ConfNode *denode = NULL; + ConfNode *decnf = ConfGetNode("detect-engine"); + if (decnf != NULL) { + TAILQ_FOREACH(denode, &decnf->head, next) { + if (strcmp(denode->val, "luajit-states") == 0) { + ConfGetChildValueInt(denode, "luajit-states", &cnt); + } + } + } + } + if (cnt == 0) { + cnt = LUAJIT_DEFAULT_STATES; + } + luajit_states_size = cnt; + + luajit_states = PoolInit(0, cnt, 0, LuaStatePoolAlloc, NULL, NULL, NULL, LuaStatePoolFree); + if (luajit_states == NULL) { + SCLogError(SC_ERR_LUA_ERROR, "luastate pool init failed, lua/luajit keywords won't work"); + retval = -1; + } + + if (retval == 0) { + SCLogConfig("luajit states preallocated: %d", luajit_states_size); + } + } + + pthread_mutex_unlock(&luajit_states_lock); + return retval; +} + +void LuajitFreeStatesPool(void) +{ + pthread_mutex_lock(&luajit_states_lock); + if (luajit_states_cnt_max > luajit_states_size) { + SCLogNotice("luajit states used %d is bigger than pool size %d. Set " + "luajit.states to %d to avoid memory issues. " + "See #1577 and #1955.", luajit_states_cnt_max, luajit_states_size, + luajit_states_cnt_max); + } + PoolFree(luajit_states); + luajit_states = NULL; + luajit_states_size = 0; + luajit_states_cnt = 0; + pthread_mutex_unlock(&luajit_states_lock); +} + +lua_State *LuajitGetState(void) +{ + lua_State *s = NULL; + pthread_mutex_lock(&luajit_states_lock); + if (luajit_states != NULL) { + s = (lua_State *)PoolGet(luajit_states); + if (s != NULL) { + if (luajit_states_cnt == luajit_states_size) { + SCLogWarning(SC_WARN_LUA_SCRIPT, "luajit states pool size %d " + "reached. Increase luajit.states config option. " + "See #1577 and #1955", luajit_states_size); + } + + luajit_states_cnt++; + if (luajit_states_cnt > luajit_states_cnt_max) + luajit_states_cnt_max = luajit_states_cnt; + } + } + pthread_mutex_unlock(&luajit_states_lock); + return s; +} + +void LuajitReturnState(lua_State *s) +{ + if (s != NULL) { + pthread_mutex_lock(&luajit_states_lock); + PoolReturn(luajit_states, (void *)s); + BUG_ON(luajit_states_cnt <= 0); + luajit_states_cnt--; + pthread_mutex_unlock(&luajit_states_lock); + } +} + +#endif /* HAVE_LUAJIT */ diff --git a/src/util-luajit.h b/src/util-luajit.h new file mode 100644 index 0000000000..5aa810b1ea --- /dev/null +++ b/src/util-luajit.h @@ -0,0 +1,36 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __UTIL_LUAJIT_H__ +#define __UTIL_LUAJIT_H__ + +#ifdef HAVE_LUAJIT + +int LuajitSetupStatesPool(void); +void LuajitFreeStatesPool(void); +lua_State *LuajitGetState(void); +void LuajitReturnState(lua_State *s); + +#endif /* HAVE_LUAJIT */ + +#endif /* __UTIL_LUAJIT_H__ */