lua: luajit improvements

Luajit has a strange memory requirement, it's 'states' need to be in the
first 2G of the process' memory.

This patch improves the pool approach by moving it to the front of the
start up.

A new config option 'luajit.states' is added to control how many states
are preallocated. It defaults to 128.

Add a warning when more states are used then preallocated. This may fail
if flow/stream/detect engines use a lot of memory. Add hint at exit that
gives the max states in use if it's higher than the default.
pull/2414/head
Victor Julien 9 years ago
parent 0792f80909
commit 3da7dad514

@ -378,6 +378,7 @@ util-ip.h util-ip.c \
util-logopenfile.h util-logopenfile.c \
util-logopenfile-tile.h util-logopenfile-tile.c \
util-lua.c util-lua.h \
util-luajit.c util-luajit.h \
util-lua-common.c util-lua-common.h \
util-lua-dnp3.c util-lua-dnp3.h \
util-lua-dnp3-objects.c util-lua-dnp3-objects.h \

@ -85,26 +85,6 @@ void DetectLuaRegister(void)
#else /* HAVE_LUA */
#ifdef HAVE_LUAJIT
#include "util-pool.h"
/** \brief lua_State pool
*
* Lua requires states to be alloc'd in memory <2GB. For this reason we
* prealloc the states early during engine startup so we have a better chance
* of getting the states. We protect the pool with a lock as the detect
* threads access it during their init and cleanup.
*
* Pool size is automagically determined based on number of keyword occurences,
* cpus/cores and rule reloads being enabled or not.
*
* Alternatively, the "detect-engine.luajit-states" var can be set.
*/
static Pool *luajit_states = NULL;
static pthread_mutex_t luajit_states_lock = SCMUTEX_INITIALIZER;
#endif /* HAVE_LUAJIT */
#include "util-lua.h"
static int DetectLuaMatch (ThreadVars *, DetectEngineThreadCtx *,
@ -170,94 +150,6 @@ void DetectLuaRegister(void)
#define DATATYPE_DNP3 (1<<20)
#ifdef HAVE_LUAJIT
static void *LuaStatePoolAlloc(void)
{
return luaL_newstate();
}
static void LuaStatePoolFree(void *d)
{
lua_State *s = (lua_State *)d;
if (s != NULL)
lua_close(s);
}
/** \brief Populate lua states pool
*
* \param num keyword instances
* \param reloads bool indicating we have rule reloads enabled
*/
int DetectLuajitSetupStatesPool(int num, int reloads)
{
int retval = 0;
pthread_mutex_lock(&luajit_states_lock);
if (luajit_states == NULL) {
intmax_t cnt = 0;
ConfNode *denode = NULL;
ConfNode *decnf = ConfGetNode("detect-engine");
if (decnf != NULL) {
TAILQ_FOREACH(denode, &decnf->head, next) {
if (strcmp(denode->val, "luajit-states") == 0) {
ConfGetChildValueInt(denode, "luajit-states", &cnt);
}
}
}
if (cnt == 0) {
int cpus = UtilCpuGetNumProcessorsOnline();
if (cpus == 0) {
cpus = 10;
}
cnt = num * cpus;
cnt *= 3; /* assume 3 threads per core */
/* alloc a bunch extra so reload can add new rules/instances */
if (reloads)
cnt *= 5;
}
luajit_states = PoolInit(0, cnt, 0, LuaStatePoolAlloc, NULL, NULL, NULL, LuaStatePoolFree);
if (luajit_states == NULL) {
SCLogError(SC_ERR_LUA_ERROR, "luastate pool init failed, lua/luajit keywords won't work");
retval = -1;
}
}
pthread_mutex_unlock(&luajit_states_lock);
return retval;
}
#endif /* HAVE_LUAJIT */
static lua_State *DetectLuaGetState(void)
{
lua_State *s = NULL;
#ifdef HAVE_LUAJIT
pthread_mutex_lock(&luajit_states_lock);
if (luajit_states != NULL)
s = (lua_State *)PoolGet(luajit_states);
pthread_mutex_unlock(&luajit_states_lock);
#else
s = luaL_newstate();
#endif
return s;
}
static void DetectLuaReturnState(lua_State *s)
{
if (s != NULL) {
#ifdef HAVE_LUAJIT
pthread_mutex_lock(&luajit_states_lock);
PoolReturn(luajit_states, (void *)s);
pthread_mutex_unlock(&luajit_states_lock);
#else
lua_close(s);
#endif
}
}
/** \brief dump stack from lua state to screen */
void LuaDumpStack(lua_State *state)
{
@ -691,7 +583,7 @@ static void *DetectLuaThreadInit(void *data)
t->alproto = luajit->alproto;
t->flags = luajit->flags;
t->luastate = DetectLuaGetState();
t->luastate = LuaGetState();
if (t->luastate == NULL) {
SCLogError(SC_ERR_LUA_ERROR, "luastate pool depleted");
goto error;
@ -737,7 +629,7 @@ static void *DetectLuaThreadInit(void *data)
error:
if (t->luastate != NULL)
DetectLuaReturnState(t->luastate);
LuaReturnState(t->luastate);
SCFree(t);
return NULL;
}
@ -747,7 +639,7 @@ static void DetectLuaThreadFree(void *ctx)
if (ctx != NULL) {
DetectLuaThreadData *t = (DetectLuaThreadData *)ctx;
if (t->luastate != NULL)
DetectLuaReturnState(t->luastate);
LuaReturnState(t->luastate);
SCFree(t);
}
}

@ -26,10 +26,6 @@
#ifdef HAVE_LUA
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
typedef struct DetectLuaThreadData {
lua_State *luastate;
uint32_t flags;
@ -63,10 +59,6 @@ int DetectLuaMatchBuffer(DetectEngineThreadCtx *det_ctx, Signature *s, SigMatch
uint8_t *buffer, uint32_t buffer_len, uint32_t offset,
Flow *f);
#ifdef HAVE_LUAJIT
int DetectLuajitSetupStatesPool(int num, int reloads);
#endif /* HAVE_LUAJIT */
void DetectLuaPostSetup(Signature *s);
#endif /* __DETECT_FILELUAJIT_H__ */

@ -200,6 +200,7 @@
#include "stream-tcp.h"
#include "stream-tcp-inline.h"
#include "util-lua.h"
#include "util-var-name.h"
#include "util-classification-config.h"
#include "util-print.h"
@ -3366,14 +3367,6 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
"preprocessing rules...");
}
#ifdef HAVE_LUAJIT
/* run this before the mpm states are initialized */
if (DetectLuajitSetupStatesPool(de_ctx->detect_luajit_instances, TRUE) != 0) {
if (de_ctx->failure_fatal)
return -1;
}
#endif
de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx);
de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *));
de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size);

@ -521,7 +521,7 @@ typedef struct LogLuaScriptOptions_ {
static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) {
int status;
lua_State *luastate = luaL_newstate();
lua_State *luastate = LuaGetState();
if (luastate == NULL)
goto error;
luaL_openlibs(luastate);
@ -649,11 +649,11 @@ static int LuaScriptInit(const char *filename, LogLuaScriptOptions *options) {
/* pop the table */
lua_pop(luastate, 1);
lua_close(luastate);
LuaReturnState(luastate);
return 0;
error:
if (luastate)
lua_close(luastate);
LuaReturnState(luastate);
return -1;
}
@ -665,7 +665,7 @@ error:
*/
static lua_State *LuaScriptSetup(const char *filename)
{
lua_State *luastate = luaL_newstate();
lua_State *luastate = LuaGetState();
if (luastate == NULL) {
SCLogError(SC_ERR_LUA_ERROR, "luaL_newstate failed");
goto error;
@ -720,7 +720,7 @@ static lua_State *LuaScriptSetup(const char *filename)
return luastate;
error:
if (luastate)
lua_close(luastate);
LuaReturnState(luastate);
return NULL;
}
@ -941,7 +941,7 @@ static void OutputLuaLogDoDeinit(LogLuaCtx *lua_ctx)
SCLogError(SC_ERR_LUA_ERROR, "couldn't run script 'deinit' function: %s", lua_tostring(luastate, -1));
return;
}
lua_close(luastate);
LuaReturnState(luastate);
}
/** \internal

@ -121,6 +121,7 @@
#include "detect-engine-siggroup.h"
#include "util-streaming-buffer.h"
#include "util-lua.h"
#endif /* UNITTESTS */
@ -143,6 +144,11 @@ void RunUnittests(int list_unittests, char *regex_arg)
GlobalInits();
TimeInit();
SupportFastPatternForSigMatchTypes();
#ifdef HAVE_LUAJIT
if (LuajitSetupStatesPool() != 0) {
exit(EXIT_FAILURE);
}
#endif
default_packet_size = DEFAULT_PACKET_SIZE;
#ifdef __SC_CUDA_SUPPORT__
@ -309,6 +315,9 @@ void RunUnittests(int list_unittests, char *regex_arg)
}
}
#ifdef HAVE_LUAJIT
LuajitFreeStatesPool();
#endif
#ifdef DBG_MEM_ALLOC
SCLogInfo("Total memory used (without SCFree()): %"PRIdMAX, (intmax_t)global_mem);
#endif

@ -376,6 +376,12 @@ typedef enum {
#include "util-path.h"
#include "util-conf.h"
#ifdef HAVE_LUA
#include <lua.h>
#include <lualib.h>
#include <lauxlib.h>
#endif
#ifndef HAVE_STRLCAT
size_t strlcat(char *, const char *src, size_t siz);
#endif

@ -170,6 +170,8 @@
#include "util-storage.h"
#include "host-storage.h"
#include "util-lua.h"
/*
* we put this here, because we only use it here in main.
*/
@ -2221,6 +2223,13 @@ static int PostConfLoadedSetup(SCInstance *suri)
{
char *hostmode = NULL;
/* do this as early as possible #1577 #1955 */
#ifdef HAVE_LUAJIT
if (LuajitSetupStatesPool() != 0) {
SCReturnInt(TM_ECODE_FAILED);
}
#endif
/* load the pattern matchers */
MpmTableSetup();
#ifdef __SC_CUDA_SUPPORT__
@ -2782,7 +2791,9 @@ int main(int argc, char **argv)
CudaHandlerFreeProfiles();
#endif
ConfDeInit();
#ifdef HAVE_LUAJIT
LuajitFreeStatesPool();
#endif
SCLogDeInitLogModule();
DetectParseFreeRegexes();
exit(engine_retval);

@ -56,6 +56,28 @@
#include "util-lua.h"
lua_State *LuaGetState(void)
{
lua_State *s = NULL;
#ifdef HAVE_LUAJIT
s = LuajitGetState();
#else
s = luaL_newstate();
#endif
return s;
}
void LuaReturnState(lua_State *s)
{
if (s != NULL) {
#ifdef HAVE_LUAJIT
LuajitReturnState(s);
#else
lua_close(s);
#endif
}
}
/* key for tv (threadvars) pointer */
const char lua_ext_key_tv[] = "suricata:lua:tv:ptr";
/* key for tx pointer */

@ -26,12 +26,17 @@
#ifdef HAVE_LUA
#include "util-luajit.h"
typedef struct LuaStreamingBuffer_ {
const uint8_t *data;
uint32_t data_len;
uint8_t flags;
} LuaStreamingBuffer;
lua_State *LuaGetState(void);
void LuaReturnState(lua_State *s);
/* gets */
/** \brief get tv pointer from the lua state */

@ -0,0 +1,155 @@
/* Copyright (C) 2007-2016 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*
*/
#include "suricata-common.h"
#ifdef HAVE_LUAJIT
#include "conf.h"
#include "util-pool.h"
#include "util-lua.h"
/** \brief lua_State pool
*
* Lua requires states to be alloc'd in memory <2GB. For this reason we
* prealloc the states early during engine startup so we have a better chance
* of getting the states. We protect the pool with a lock as the detect
* threads access it during their init and cleanup.
*
* Pool size is automagically determined based on number of keyword occurences,
* cpus/cores and rule reloads being enabled or not.
*
* Alternatively, the "detect-engine.luajit-states" var can be set.
*/
static Pool *luajit_states = NULL;
static pthread_mutex_t luajit_states_lock = SCMUTEX_INITIALIZER;
static int luajit_states_cnt = 0;
static int luajit_states_cnt_max = 0;
static int luajit_states_size = 0;
#define LUAJIT_DEFAULT_STATES 128
static void *LuaStatePoolAlloc(void)
{
return luaL_newstate();
}
static void LuaStatePoolFree(void *d)
{
lua_State *s = (lua_State *)d;
if (s != NULL)
lua_close(s);
}
/** \brief Populate lua states pool
*
* \param num keyword instances
* \param reloads bool indicating we have rule reloads enabled
*/
int LuajitSetupStatesPool(void)
{
int retval = 0;
pthread_mutex_lock(&luajit_states_lock);
if (luajit_states == NULL) {
intmax_t cnt = 0;
if (ConfGetInt("luajit.states", &cnt) != 1) {
ConfNode *denode = NULL;
ConfNode *decnf = ConfGetNode("detect-engine");
if (decnf != NULL) {
TAILQ_FOREACH(denode, &decnf->head, next) {
if (strcmp(denode->val, "luajit-states") == 0) {
ConfGetChildValueInt(denode, "luajit-states", &cnt);
}
}
}
}
if (cnt == 0) {
cnt = LUAJIT_DEFAULT_STATES;
}
luajit_states_size = cnt;
luajit_states = PoolInit(0, cnt, 0, LuaStatePoolAlloc, NULL, NULL, NULL, LuaStatePoolFree);
if (luajit_states == NULL) {
SCLogError(SC_ERR_LUA_ERROR, "luastate pool init failed, lua/luajit keywords won't work");
retval = -1;
}
if (retval == 0) {
SCLogConfig("luajit states preallocated: %d", luajit_states_size);
}
}
pthread_mutex_unlock(&luajit_states_lock);
return retval;
}
void LuajitFreeStatesPool(void)
{
pthread_mutex_lock(&luajit_states_lock);
if (luajit_states_cnt_max > luajit_states_size) {
SCLogNotice("luajit states used %d is bigger than pool size %d. Set "
"luajit.states to %d to avoid memory issues. "
"See #1577 and #1955.", luajit_states_cnt_max, luajit_states_size,
luajit_states_cnt_max);
}
PoolFree(luajit_states);
luajit_states = NULL;
luajit_states_size = 0;
luajit_states_cnt = 0;
pthread_mutex_unlock(&luajit_states_lock);
}
lua_State *LuajitGetState(void)
{
lua_State *s = NULL;
pthread_mutex_lock(&luajit_states_lock);
if (luajit_states != NULL) {
s = (lua_State *)PoolGet(luajit_states);
if (s != NULL) {
if (luajit_states_cnt == luajit_states_size) {
SCLogWarning(SC_WARN_LUA_SCRIPT, "luajit states pool size %d "
"reached. Increase luajit.states config option. "
"See #1577 and #1955", luajit_states_size);
}
luajit_states_cnt++;
if (luajit_states_cnt > luajit_states_cnt_max)
luajit_states_cnt_max = luajit_states_cnt;
}
}
pthread_mutex_unlock(&luajit_states_lock);
return s;
}
void LuajitReturnState(lua_State *s)
{
if (s != NULL) {
pthread_mutex_lock(&luajit_states_lock);
PoolReturn(luajit_states, (void *)s);
BUG_ON(luajit_states_cnt <= 0);
luajit_states_cnt--;
pthread_mutex_unlock(&luajit_states_lock);
}
}
#endif /* HAVE_LUAJIT */

@ -0,0 +1,36 @@
/* Copyright (C) 2007-2016 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author Victor Julien <victor@inliniac.net>
*/
#ifndef __UTIL_LUAJIT_H__
#define __UTIL_LUAJIT_H__
#ifdef HAVE_LUAJIT
int LuajitSetupStatesPool(void);
void LuajitFreeStatesPool(void);
lua_State *LuajitGetState(void);
void LuajitReturnState(lua_State *s);
#endif /* HAVE_LUAJIT */
#endif /* __UTIL_LUAJIT_H__ */
Loading…
Cancel
Save