diff --git a/src/Makefile.am b/src/Makefile.am index 708ecb7a62..48f5dd0ee0 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -378,6 +378,7 @@ util-ip.h util-ip.c \ util-logopenfile.h util-logopenfile.c \ util-logopenfile-tile.h util-logopenfile-tile.c \ util-lua.c util-lua.h \ +util-luajit.c util-luajit.h \ util-lua-common.c util-lua-common.h \ util-lua-dnp3.c util-lua-dnp3.h \ util-lua-dnp3-objects.c util-lua-dnp3-objects.h \ diff --git a/src/detect-lua.c b/src/detect-lua.c index d77ee4edd5..a1ce743285 100644 --- a/src/detect-lua.c +++ b/src/detect-lua.c @@ -85,26 +85,6 @@ void DetectLuaRegister(void) #else /* HAVE_LUA */ -#ifdef HAVE_LUAJIT -#include "util-pool.h" - -/** \brief lua_State pool - * - * Lua requires states to be alloc'd in memory <2GB. For this reason we - * prealloc the states early during engine startup so we have a better chance - * of getting the states. We protect the pool with a lock as the detect - * threads access it during their init and cleanup. - * - * Pool size is automagically determined based on number of keyword occurences, - * cpus/cores and rule reloads being enabled or not. - * - * Alternatively, the "detect-engine.luajit-states" var can be set. - */ -static Pool *luajit_states = NULL; -static pthread_mutex_t luajit_states_lock = SCMUTEX_INITIALIZER; - -#endif /* HAVE_LUAJIT */ - #include "util-lua.h" static int DetectLuaMatch (ThreadVars *, DetectEngineThreadCtx *, @@ -170,94 +150,6 @@ void DetectLuaRegister(void) #define DATATYPE_DNP3 (1<<20) -#ifdef HAVE_LUAJIT -static void *LuaStatePoolAlloc(void) -{ - return luaL_newstate(); -} - -static void LuaStatePoolFree(void *d) -{ - lua_State *s = (lua_State *)d; - if (s != NULL) - lua_close(s); -} - -/** \brief Populate lua states pool - * - * \param num keyword instances - * \param reloads bool indicating we have rule reloads enabled - */ -int DetectLuajitSetupStatesPool(int num, int reloads) -{ - int retval = 0; - pthread_mutex_lock(&luajit_states_lock); - - if (luajit_states == NULL) { - intmax_t cnt = 0; - ConfNode *denode = NULL; - ConfNode *decnf = ConfGetNode("detect-engine"); - if (decnf != NULL) { - TAILQ_FOREACH(denode, &decnf->head, next) { - if (strcmp(denode->val, "luajit-states") == 0) { - ConfGetChildValueInt(denode, "luajit-states", &cnt); - } - } - } - - if (cnt == 0) { - int cpus = UtilCpuGetNumProcessorsOnline(); - if (cpus == 0) { - cpus = 10; - } - cnt = num * cpus; - cnt *= 3; /* assume 3 threads per core */ - - /* alloc a bunch extra so reload can add new rules/instances */ - if (reloads) - cnt *= 5; - } - - luajit_states = PoolInit(0, cnt, 0, LuaStatePoolAlloc, NULL, NULL, NULL, LuaStatePoolFree); - if (luajit_states == NULL) { - SCLogError(SC_ERR_LUA_ERROR, "luastate pool init failed, lua/luajit keywords won't work"); - retval = -1; - } - } - - pthread_mutex_unlock(&luajit_states_lock); - return retval; -} -#endif /* HAVE_LUAJIT */ - -static lua_State *DetectLuaGetState(void) -{ - - lua_State *s = NULL; -#ifdef HAVE_LUAJIT - pthread_mutex_lock(&luajit_states_lock); - if (luajit_states != NULL) - s = (lua_State *)PoolGet(luajit_states); - pthread_mutex_unlock(&luajit_states_lock); -#else - s = luaL_newstate(); -#endif - return s; -} - -static void DetectLuaReturnState(lua_State *s) -{ - if (s != NULL) { -#ifdef HAVE_LUAJIT - pthread_mutex_lock(&luajit_states_lock); - PoolReturn(luajit_states, (void *)s); - pthread_mutex_unlock(&luajit_states_lock); -#else - lua_close(s); -#endif - } -} - /** \brief dump stack from lua state to screen */ void LuaDumpStack(lua_State *state) { @@ -691,7 +583,7 @@ static void *DetectLuaThreadInit(void *data) t->alproto = luajit->alproto; t->flags = luajit->flags; - t->luastate = DetectLuaGetState(); + t->luastate = LuaGetState(); if (t->luastate == NULL) { SCLogError(SC_ERR_LUA_ERROR, "luastate pool depleted"); goto error; @@ -737,7 +629,7 @@ static void *DetectLuaThreadInit(void *data) error: if (t->luastate != NULL) - DetectLuaReturnState(t->luastate); + LuaReturnState(t->luastate); SCFree(t); return NULL; } @@ -747,7 +639,7 @@ static void DetectLuaThreadFree(void *ctx) if (ctx != NULL) { DetectLuaThreadData *t = (DetectLuaThreadData *)ctx; if (t->luastate != NULL) - DetectLuaReturnState(t->luastate); + LuaReturnState(t->luastate); SCFree(t); } } diff --git a/src/detect-lua.h b/src/detect-lua.h index f1e86c2f2e..d1259b3e66 100644 --- a/src/detect-lua.h +++ b/src/detect-lua.h @@ -26,10 +26,6 @@ #ifdef HAVE_LUA -#include -#include -#include - typedef struct DetectLuaThreadData { lua_State *luastate; uint32_t flags; @@ -63,10 +59,6 @@ int DetectLuaMatchBuffer(DetectEngineThreadCtx *det_ctx, Signature *s, SigMatch uint8_t *buffer, uint32_t buffer_len, uint32_t offset, Flow *f); -#ifdef HAVE_LUAJIT -int DetectLuajitSetupStatesPool(int num, int reloads); -#endif /* HAVE_LUAJIT */ - void DetectLuaPostSetup(Signature *s); #endif /* __DETECT_FILELUAJIT_H__ */ diff --git a/src/detect.c b/src/detect.c index 27047c5c0a..e75e393dd7 100644 --- a/src/detect.c +++ b/src/detect.c @@ -200,6 +200,7 @@ #include "stream-tcp.h" #include "stream-tcp-inline.h" +#include "util-lua.h" #include "util-var-name.h" #include "util-classification-config.h" #include "util-print.h" @@ -3366,14 +3367,6 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) "preprocessing rules..."); } -#ifdef HAVE_LUAJIT - /* run this before the mpm states are initialized */ - if (DetectLuajitSetupStatesPool(de_ctx->detect_luajit_instances, TRUE) != 0) { - if (de_ctx->failure_fatal) - return -1; - } -#endif - de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx); de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *)); de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size); diff --git a/src/output-lua.c b/src/output-lua.c index b71b0e3736..d9a6dc497a 100644 --- a/src/output-lua.c +++ b/src/output-lua.c @@ -521,7 +521,7 @@ typedef struct LogLuaScriptOptions_ { static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) { int status; - lua_State *luastate = luaL_newstate(); + lua_State *luastate = LuaGetState(); if (luastate == NULL) goto error; luaL_openlibs(luastate); @@ -649,11 +649,11 @@ static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) { /* pop the table */ lua_pop(luastate, 1); - lua_close(luastate); + LuaReturnState(luastate); return 0; error: if (luastate) - lua_close(luastate); + LuaReturnState(luastate); return -1; } @@ -665,7 +665,7 @@ error: */ static lua_State *LuaScriptSetup(const char *filename) { - lua_State *luastate = luaL_newstate(); + lua_State *luastate = LuaGetState(); if (luastate == NULL) { SCLogError(SC_ERR_LUA_ERROR, "luaL_newstate failed"); goto error; @@ -720,7 +720,7 @@ static lua_State *LuaScriptSetup(const char *filename) return luastate; error: if (luastate) - lua_close(luastate); + LuaReturnState(luastate); return NULL; } @@ -941,7 +941,7 @@ static void OutputLuaLogDoDeinit(LogLuaCtx *lua_ctx) SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'deinit' function: %s", lua_tostring(luastate, -1)); return; } - lua_close(luastate); + LuaReturnState(luastate); } /** \internal diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index 7c97d28629..4f03f18486 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -121,6 +121,7 @@ #include "detect-engine-siggroup.h" #include "util-streaming-buffer.h" +#include "util-lua.h" #endif /* UNITTESTS */ @@ -143,6 +144,11 @@ void RunUnittests(int list_unittests, char *regex_arg) GlobalInits(); TimeInit(); SupportFastPatternForSigMatchTypes(); +#ifdef HAVE_LUAJIT + if (LuajitSetupStatesPool() != 0) { + exit(EXIT_FAILURE); + } +#endif default_packet_size = DEFAULT_PACKET_SIZE; #ifdef __SC_CUDA_SUPPORT__ @@ -309,6 +315,9 @@ void RunUnittests(int list_unittests, char *regex_arg) } } +#ifdef HAVE_LUAJIT + LuajitFreeStatesPool(); +#endif #ifdef DBG_MEM_ALLOC SCLogInfo("Total memory used (without SCFree()): %"PRIdMAX, (intmax_t)global_mem); #endif diff --git a/src/suricata-common.h b/src/suricata-common.h index a2c9414a16..5a40156b16 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -376,6 +376,12 @@ typedef enum { #include "util-path.h" #include "util-conf.h" +#ifdef HAVE_LUA +#include +#include +#include +#endif + #ifndef HAVE_STRLCAT size_t strlcat(char *, const char *src, size_t siz); #endif diff --git a/src/suricata.c b/src/suricata.c index ac22824280..292ffd708e 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -170,6 +170,8 @@ #include "util-storage.h" #include "host-storage.h" +#include "util-lua.h" + /* * we put this here, because we only use it here in main. */ @@ -2221,6 +2223,13 @@ static int PostConfLoadedSetup(SCInstance *suri) { char *hostmode = NULL; + /* do this as early as possible #1577 #1955 */ +#ifdef HAVE_LUAJIT + if (LuajitSetupStatesPool() != 0) { + SCReturnInt(TM_ECODE_FAILED); + } +#endif + /* load the pattern matchers */ MpmTableSetup(); #ifdef __SC_CUDA_SUPPORT__ @@ -2782,7 +2791,9 @@ int main(int argc, char **argv) CudaHandlerFreeProfiles(); #endif ConfDeInit(); - +#ifdef HAVE_LUAJIT + LuajitFreeStatesPool(); +#endif SCLogDeInitLogModule(); DetectParseFreeRegexes(); exit(engine_retval); diff --git a/src/util-lua.c b/src/util-lua.c index 7f1d44348c..bc0c20c887 100644 --- a/src/util-lua.c +++ b/src/util-lua.c @@ -56,6 +56,28 @@ #include "util-lua.h" +lua_State *LuaGetState(void) +{ + lua_State *s = NULL; +#ifdef HAVE_LUAJIT + s = LuajitGetState(); +#else + s = luaL_newstate(); +#endif + return s; +} + +void LuaReturnState(lua_State *s) +{ + if (s != NULL) { +#ifdef HAVE_LUAJIT + LuajitReturnState(s); +#else + lua_close(s); +#endif + } +} + /* key for tv (threadvars) pointer */ const char lua_ext_key_tv[] = "suricata:lua:tv:ptr"; /* key for tx pointer */ diff --git a/src/util-lua.h b/src/util-lua.h index 30431e74e8..d6a9838cd8 100644 --- a/src/util-lua.h +++ b/src/util-lua.h @@ -26,12 +26,17 @@ #ifdef HAVE_LUA +#include "util-luajit.h" + typedef struct LuaStreamingBuffer_ { const uint8_t *data; uint32_t data_len; uint8_t flags; } LuaStreamingBuffer; +lua_State *LuaGetState(void); +void LuaReturnState(lua_State *s); + /* gets */ /** \brief get tv pointer from the lua state */ diff --git a/src/util-luajit.c b/src/util-luajit.c new file mode 100644 index 0000000000..cb11dc79af --- /dev/null +++ b/src/util-luajit.c @@ -0,0 +1,155 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + */ + +#include "suricata-common.h" + +#ifdef HAVE_LUAJIT +#include "conf.h" +#include "util-pool.h" +#include "util-lua.h" + +/** \brief lua_State pool + * + * Lua requires states to be alloc'd in memory <2GB. For this reason we + * prealloc the states early during engine startup so we have a better chance + * of getting the states. We protect the pool with a lock as the detect + * threads access it during their init and cleanup. + * + * Pool size is automagically determined based on number of keyword occurences, + * cpus/cores and rule reloads being enabled or not. + * + * Alternatively, the "detect-engine.luajit-states" var can be set. + */ +static Pool *luajit_states = NULL; +static pthread_mutex_t luajit_states_lock = SCMUTEX_INITIALIZER; +static int luajit_states_cnt = 0; +static int luajit_states_cnt_max = 0; +static int luajit_states_size = 0; +#define LUAJIT_DEFAULT_STATES 128 + +static void *LuaStatePoolAlloc(void) +{ + return luaL_newstate(); +} + +static void LuaStatePoolFree(void *d) +{ + lua_State *s = (lua_State *)d; + if (s != NULL) + lua_close(s); +} + +/** \brief Populate lua states pool + * + * \param num keyword instances + * \param reloads bool indicating we have rule reloads enabled + */ +int LuajitSetupStatesPool(void) +{ + int retval = 0; + pthread_mutex_lock(&luajit_states_lock); + + if (luajit_states == NULL) { + intmax_t cnt = 0; + if (ConfGetInt("luajit.states", &cnt) != 1) { + ConfNode *denode = NULL; + ConfNode *decnf = ConfGetNode("detect-engine"); + if (decnf != NULL) { + TAILQ_FOREACH(denode, &decnf->head, next) { + if (strcmp(denode->val, "luajit-states") == 0) { + ConfGetChildValueInt(denode, "luajit-states", &cnt); + } + } + } + } + if (cnt == 0) { + cnt = LUAJIT_DEFAULT_STATES; + } + luajit_states_size = cnt; + + luajit_states = PoolInit(0, cnt, 0, LuaStatePoolAlloc, NULL, NULL, NULL, LuaStatePoolFree); + if (luajit_states == NULL) { + SCLogError(SC_ERR_LUA_ERROR, "luastate pool init failed, lua/luajit keywords won't work"); + retval = -1; + } + + if (retval == 0) { + SCLogConfig("luajit states preallocated: %d", luajit_states_size); + } + } + + pthread_mutex_unlock(&luajit_states_lock); + return retval; +} + +void LuajitFreeStatesPool(void) +{ + pthread_mutex_lock(&luajit_states_lock); + if (luajit_states_cnt_max > luajit_states_size) { + SCLogNotice("luajit states used %d is bigger than pool size %d. Set " + "luajit.states to %d to avoid memory issues. " + "See #1577 and #1955.", luajit_states_cnt_max, luajit_states_size, + luajit_states_cnt_max); + } + PoolFree(luajit_states); + luajit_states = NULL; + luajit_states_size = 0; + luajit_states_cnt = 0; + pthread_mutex_unlock(&luajit_states_lock); +} + +lua_State *LuajitGetState(void) +{ + lua_State *s = NULL; + pthread_mutex_lock(&luajit_states_lock); + if (luajit_states != NULL) { + s = (lua_State *)PoolGet(luajit_states); + if (s != NULL) { + if (luajit_states_cnt == luajit_states_size) { + SCLogWarning(SC_WARN_LUA_SCRIPT, "luajit states pool size %d " + "reached. Increase luajit.states config option. " + "See #1577 and #1955", luajit_states_size); + } + + luajit_states_cnt++; + if (luajit_states_cnt > luajit_states_cnt_max) + luajit_states_cnt_max = luajit_states_cnt; + } + } + pthread_mutex_unlock(&luajit_states_lock); + return s; +} + +void LuajitReturnState(lua_State *s) +{ + if (s != NULL) { + pthread_mutex_lock(&luajit_states_lock); + PoolReturn(luajit_states, (void *)s); + BUG_ON(luajit_states_cnt <= 0); + luajit_states_cnt--; + pthread_mutex_unlock(&luajit_states_lock); + } +} + +#endif /* HAVE_LUAJIT */ diff --git a/src/util-luajit.h b/src/util-luajit.h new file mode 100644 index 0000000000..5aa810b1ea --- /dev/null +++ b/src/util-luajit.h @@ -0,0 +1,36 @@ +/* Copyright (C) 2007-2016 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __UTIL_LUAJIT_H__ +#define __UTIL_LUAJIT_H__ + +#ifdef HAVE_LUAJIT + +int LuajitSetupStatesPool(void); +void LuajitFreeStatesPool(void); +lua_State *LuajitGetState(void); +void LuajitReturnState(lua_State *s); + +#endif /* HAVE_LUAJIT */ + +#endif /* __UTIL_LUAJIT_H__ */