From 6e18ed0489c7d0026210b323fdb740ec900e07c2 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Mon, 15 Apr 2013 19:59:28 +0200 Subject: [PATCH] luajit flowvar support This patch adds flowvar support to luajit. It does so by exposing two special C functions to the luajit scripts: ScFlowvarGet and ScFlowvarSet. --- src/Makefile.am | 1 + src/detect-engine-content-inspection.c | 9 +- src/detect-engine-content-inspection.h | 6 + src/detect-engine-state.c | 4 + src/detect-flowvar.c | 32 +- src/detect-flowvar.h | 4 +- src/detect-luajit-extensions.c | 305 ++++++++++++++ src/detect-luajit-extensions.h | 35 ++ src/detect-luajit.c | 547 ++++++++++++++++++++++++- src/detect-luajit.h | 10 +- src/detect-pcre.c | 33 +- src/detect.c | 2 +- src/detect.h | 6 + src/flow-var.c | 3 +- 14 files changed, 959 insertions(+), 38 deletions(-) create mode 100644 src/detect-luajit-extensions.c create mode 100644 src/detect-luajit-extensions.h diff --git a/src/Makefile.am b/src/Makefile.am index bb4fe1a369..6b50657397 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -147,6 +147,7 @@ detect-isdataat.c detect-isdataat.h \ detect-itype.c detect-itype.h \ detect-l3proto.c detect-l3proto.h \ detect-luajit.c detect-luajit.h \ +detect-luajit-extensions.c detect-luajit-extensions.h \ detect-mark.c detect-mark.h \ detect-metadata.c detect-metadata.h \ detect-msg.c detect-msg.h \ diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index e19d052337..ab44f2ee3a 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -498,7 +498,14 @@ int DetectEngineContentInspection(DetectEngineCtx *de_ctx, DetectEngineThreadCtx #ifdef HAVE_LUAJIT } else if (sm->type == DETECT_LUAJIT) { - if (DetectLuajitMatchBuffer(det_ctx, s, sm, buffer, buffer_len, det_ctx->buffer_offset) != 1) { + /* for flowvar gets and sets we need to know the flow's lock status */ + int need_flow_lock = 0; + if (inspection_mode <= DETECT_ENGINE_CONTENT_INSPECTION_MODE_STREAM) + need_flow_lock = 1; + + if (DetectLuajitMatchBuffer(det_ctx, s, sm, buffer, buffer_len, + det_ctx->buffer_offset, f, need_flow_lock) != 1) + { SCReturnInt(0); } goto match; diff --git a/src/detect-engine-content-inspection.h b/src/detect-engine-content-inspection.h index 19f0126778..a627a8d2f6 100644 --- a/src/detect-engine-content-inspection.h +++ b/src/detect-engine-content-inspection.h @@ -24,9 +24,15 @@ #ifndef __DETECT_ENGINE_CONTENT_INSPECTION_H__ #define __DETECT_ENGINE_CONTENT_INSPECTION_H__ +/** \warning make sure to add new entries to the proper position + * wrt flow lock status + */ enum { + /* called with flow unlocked */ DETECT_ENGINE_CONTENT_INSPECTION_MODE_PAYLOAD = 0, DETECT_ENGINE_CONTENT_INSPECTION_MODE_STREAM, + + /* called with flow locked */ DETECT_ENGINE_CONTENT_INSPECTION_MODE_DCE, DETECT_ENGINE_CONTENT_INSPECTION_MODE_URI, DETECT_ENGINE_CONTENT_INSPECTION_MODE_HRUD, diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c index 21817055b8..ae1875a8f2 100644 --- a/src/detect-engine-state.c +++ b/src/detect-engine-state.c @@ -57,6 +57,8 @@ #include "detect-engine-state.h" #include "detect-engine-dcepayload.h" +#include "detect-flowvar.h" + #include "stream-tcp.h" #include "stream-tcp-private.h" #include "stream-tcp-reassemble.h" @@ -615,6 +617,8 @@ void DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, p->action |= s->action; } } + + DetectFlowvarProcessList(det_ctx, f); } } diff --git a/src/detect-flowvar.c b/src/detect-flowvar.c index c0bcd7f83d..1f5bdc1aff 100644 --- a/src/detect-flowvar.c +++ b/src/detect-flowvar.c @@ -34,9 +34,11 @@ #include "flow.h" #include "flow-var.h" #include "detect-flowvar.h" + #include "util-spm.h" #include "util-var-name.h" #include "util-debug.h" +#include "util-print.h" #define PARSE_REGEX "(.*),(.*)" static pcre *parse_regex; @@ -209,7 +211,9 @@ error: /** \brief Store flowvar in det_ctx so we can exec it post-match */ -int DetectFlowvarStoreMatch(DetectEngineThreadCtx *det_ctx, uint16_t idx, uint8_t *buffer, uint16_t len) { +int DetectFlowvarStoreMatch(DetectEngineThreadCtx *det_ctx, uint16_t idx, + uint8_t *buffer, uint16_t len, int type) +{ DetectFlowvarList *fs = det_ctx->flowvarlist; /* first check if we have had a previous match for this idx */ @@ -234,6 +238,7 @@ int DetectFlowvarStoreMatch(DetectEngineThreadCtx *det_ctx, uint16_t idx, uint8_ } fs->len = len; + fs->type = type; fs->buffer = buffer; return 0; } @@ -286,6 +291,9 @@ static int DetectFlowvarPostMatch(ThreadVars *tv, DetectEngineThreadCtx *det_ctx fs = det_ctx->flowvarlist; while (fs != NULL) { if (fd->idx == fs->idx) { + SCLogDebug("adding to the flow %u:", fs->idx); + //PrintRawDataFp(stdout, fs->buffer, fs->len); + FlowVarAddStr(p->flow, fs->idx, fs->buffer, fs->len); /* memory at fs->buffer is now the responsibility of * the flowvar code. */ @@ -307,14 +315,30 @@ static int DetectFlowvarPostMatch(ThreadVars *tv, DetectEngineThreadCtx *det_ctx return 1; } -/** \brief Clean flowvar candidate list in det_ctx */ -void DetectFlowvarCleanupList(DetectEngineThreadCtx *det_ctx) { +/** \brief Handle flowvar candidate list in det_ctx: + * - clean up the list + * - enforce storage for type ALWAYS (luajit) */ +void DetectFlowvarProcessList(DetectEngineThreadCtx *det_ctx, Flow *f) { DetectFlowvarList *fs, *next; + + SCLogDebug("det_ctx->flowvarlist %p", det_ctx->flowvarlist); + if (det_ctx->flowvarlist != NULL) { fs = det_ctx->flowvarlist; while (fs != NULL) { next = fs->next; - SCFree(fs->buffer); + + if (fs->type == DETECT_FLOWVAR_TYPE_ALWAYS) { + BUG_ON(f == NULL); + SCLogDebug("adding to the flow %u:", fs->idx); + //PrintRawDataFp(stdout, fs->buffer, fs->len); + + FlowVarAddStr(f, fs->idx, fs->buffer, fs->len); + /* memory at fs->buffer is now the responsibility of + * the flowvar code. */ + } else { + SCFree(fs->buffer); + } SCFree(fs); fs = next; } diff --git a/src/detect-flowvar.h b/src/detect-flowvar.h index 96d0f5aa64..a2975f0521 100644 --- a/src/detect-flowvar.h +++ b/src/detect-flowvar.h @@ -36,8 +36,8 @@ typedef struct DetectFlowvarData_ { void DetectFlowvarRegister (void); int DetectFlowvarPostMatchSetup(Signature *s, uint16_t idx); -int DetectFlowvarStoreMatch(DetectEngineThreadCtx *, uint16_t, uint8_t *, uint16_t); -void DetectFlowvarCleanupList(DetectEngineThreadCtx *det_ctx); +int DetectFlowvarStoreMatch(DetectEngineThreadCtx *, uint16_t, uint8_t *, uint16_t, int); +void DetectFlowvarProcessList(DetectEngineThreadCtx *det_ctx, Flow *); #endif /* __DETECT_FLOWVAR_H__ */ diff --git a/src/detect-luajit-extensions.c b/src/detect-luajit-extensions.c new file mode 100644 index 0000000000..bc2db10a74 --- /dev/null +++ b/src/detect-luajit-extensions.c @@ -0,0 +1,305 @@ +/* Copyright (C) 2007-2013 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + * + * Functions to expose to the lua scripts. + */ + +#include "suricata-common.h" +#include "conf.h" + +#include "threads.h" +#include "debug.h" +#include "decode.h" + +#include "detect.h" +#include "detect-parse.h" +#include "detect-flowvar.h" + +#include "detect-engine.h" +#include "detect-engine-mpm.h" +#include "detect-engine-state.h" + +#include "flow.h" +#include "flow-var.h" +#include "flow-util.h" + +#include "util-debug.h" +#include "util-spm-bm.h" +#include "util-print.h" + +#include "util-unittest.h" +#include "util-unittest-helper.h" + +#include "app-layer.h" + +#include "stream-tcp.h" + +#include "detect-luajit.h" + +#include "queue.h" +#include "util-cpu.h" + +#ifdef HAVE_LUAJIT + +static const char luaext_key_ld[] = "suricata:luajitdata"; +static const char luaext_key_det_ctx[] = "suricata:det_ctx"; +static const char luaext_key_flow[] = "suricata:flow"; +static const char luaext_key_need_flow_lock[] = "suricata:need_flow_lock"; + +static int LuajitGetFlowvar(lua_State *luastate) { + uint16_t idx; + int id; + Flow *f; + FlowVar *fv; + DetectLuajitData *ld; + int need_flow_lock = 0; + + /* need luajit data for id -> idx conversion */ + lua_pushlightuserdata(luastate, (void *)&luaext_key_ld); + lua_gettable(luastate, LUA_REGISTRYINDEX); + ld = lua_touserdata(luastate, -1); + SCLogDebug("ld %p", ld); + if (ld == NULL) { + lua_pushnil(luastate); + lua_pushstring(luastate, "internal error: no ld"); + return 2; + } + + /* need flow */ + lua_pushlightuserdata(luastate, (void *)&luaext_key_flow); + lua_gettable(luastate, LUA_REGISTRYINDEX); + f = lua_touserdata(luastate, -1); + SCLogDebug("f %p", f); + if (f == NULL) { + lua_pushnil(luastate); + lua_pushstring(luastate, "no flow"); + return 2; + } + + /* need flow lock hint */ + lua_pushlightuserdata(luastate, (void *)&luaext_key_need_flow_lock); + lua_gettable(luastate, LUA_REGISTRYINDEX); + need_flow_lock = lua_toboolean(luastate, -1); + + /* need flowvar idx */ + if (!lua_isnumber(luastate, 1)) { + lua_pushnil(luastate); + lua_pushstring(luastate, "1st arg not a number"); + return 2; + } + id = lua_tonumber(luastate, 1); + if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWVARS) { + lua_pushnil(luastate); + lua_pushstring(luastate, "flowvar id out of range"); + return 2; + } + idx = ld->flowvar[id]; + if (idx == 0) { + lua_pushnil(luastate); + lua_pushstring(luastate, "flowvar id uninitialized"); + return 2; + } + + /* lookup var */ + if (need_flow_lock) + FLOWLOCK_RDLOCK(f); + + fv = FlowVarGet(f, idx); + if (fv == NULL) { + if (need_flow_lock) + FLOWLOCK_UNLOCK(f); + + lua_pushnil(luastate); + lua_pushstring(luastate, "no flow var"); + return 2; + } + + //SCLogInfo("returning:"); + //PrintRawDataFp(stdout,fv->data.fv_str.value,fv->data.fv_str.value_len); + + /* we're using a buffer sized at a multiple of 4 as lua_pushlstring generates + * invalid read errors in valgrind otherwise. Adding in a nul to be sure. + * + * Buffer size = len + 1 (for nul) + whatever makes it a multiple of 4 */ + size_t buflen = fv->data.fv_str.value_len + 1 + ((fv->data.fv_str.value_len + 1) % 4); + uint8_t buf[buflen]; + memset(buf, 0x00, buflen); + memcpy(buf, fv->data.fv_str.value, fv->data.fv_str.value_len); + buf[fv->data.fv_str.value_len] = '\0'; + + if (need_flow_lock) + FLOWLOCK_UNLOCK(f); + + /* return value through luastate, as a luastring */ + lua_pushlstring(luastate, (char *)buf, buflen); + + return 1; + +} + +int LuajitSetFlowvar(lua_State *luastate) { + uint16_t idx; + int id; + Flow *f; + const char *str; + int len; + uint8_t *buffer; + DetectEngineThreadCtx *det_ctx; + DetectLuajitData *ld; + + /* need luajit data for id -> idx conversion */ + lua_pushlightuserdata(luastate, (void *)&luaext_key_ld); + lua_gettable(luastate, LUA_REGISTRYINDEX); + ld = lua_touserdata(luastate, -1); + SCLogDebug("ld %p", ld); + if (ld == NULL) { + lua_pushnil(luastate); + lua_pushstring(luastate, "internal error: no ld"); + return 2; + } + + /* need det_ctx */ + lua_pushlightuserdata(luastate, (void *)&luaext_key_det_ctx); + lua_gettable(luastate, LUA_REGISTRYINDEX); + det_ctx = lua_touserdata(luastate, -1); + SCLogDebug("det_ctx %p", det_ctx); + if (det_ctx == NULL) { + lua_pushnil(luastate); + lua_pushstring(luastate, "internal error: no det_ctx"); + return 2; + } + + /* need flow */ + lua_pushlightuserdata(luastate, (void *)&luaext_key_flow); + lua_gettable(luastate, LUA_REGISTRYINDEX); + f = lua_touserdata(luastate, -1); + SCLogDebug("f %p", f); + if (f == NULL) { + lua_pushnil(luastate); + lua_pushstring(luastate, "no flow"); + return 2; + } + + /* need flowvar idx */ + if (!lua_isnumber(luastate, 1)) { + lua_pushnil(luastate); + lua_pushstring(luastate, "1st arg not a number"); + return 2; + } + id = lua_tonumber(luastate, 1); + if (id < 0 || id >= DETECT_LUAJIT_MAX_FLOWVARS) { + lua_pushnil(luastate); + lua_pushstring(luastate, "flowvar id out of range"); + return 2; + } + + if (!lua_isstring(luastate, 2)) { + lua_pushnil(luastate); + lua_pushstring(luastate, "2nd arg not a string"); + return 2; + } + str = lua_tostring(luastate, 2); + if (str == NULL) { + lua_pushnil(luastate); + lua_pushstring(luastate, "null string"); + return 2; + } + + if (!lua_isnumber(luastate, 3)) { + lua_pushnil(luastate); + lua_pushstring(luastate, "3rd arg not a number"); + return 2; + } + len = lua_tonumber(luastate, 3); + if (len < 0 || len > 0xffff) { + lua_pushnil(luastate); + lua_pushstring(luastate, "len out of range: max 64k"); + return 2; + } + + idx = ld->flowvar[id]; + if (idx == 0) { + lua_pushnil(luastate); + lua_pushstring(luastate, "flowvar id uninitialized"); + return 2; + } + + buffer = SCMalloc(len+1); + if (buffer == NULL) { + lua_pushnil(luastate); + lua_pushstring(luastate, "out of memory"); + return 2; + } + memcpy(buffer, str, len); + buffer[len] = '\0'; + + if (DetectFlowvarStoreMatch(det_ctx, idx, buffer, len, + DETECT_FLOWVAR_TYPE_ALWAYS) < 0) { + SCLogInfo("store failed"); + SCFree(buffer); + + lua_pushnil(luastate); + lua_pushstring(luastate, "store failed"); + return 2; + } + //SCLogInfo("stored:"); + //PrintRawDataFp(stdout,buffer,len); + return 0; +} + +void LuajitExtensionsMatchSetup(lua_State *lua_state, DetectLuajitData *ld, DetectEngineThreadCtx *det_ctx, Flow *f, int need_flow_lock) { + SCLogDebug("det_ctx %p, f %p", det_ctx, f); + + /* luajit keyword data */ + lua_pushlightuserdata(lua_state, (void *)&luaext_key_ld); + lua_pushlightuserdata(lua_state, (void *)ld); + lua_settable(lua_state, LUA_REGISTRYINDEX); + + /* detection engine thread ctx */ + lua_pushlightuserdata(lua_state, (void *)&luaext_key_det_ctx); + lua_pushlightuserdata(lua_state, (void *)det_ctx); + lua_settable(lua_state, LUA_REGISTRYINDEX); + + /* flow */ + lua_pushlightuserdata(lua_state, (void *)&luaext_key_flow); + lua_pushlightuserdata(lua_state, (void *)f); + lua_settable(lua_state, LUA_REGISTRYINDEX); + + /* flow lock status hint */ + lua_pushlightuserdata(lua_state, (void *)&luaext_key_need_flow_lock); + lua_pushboolean(lua_state, need_flow_lock); + lua_settable(lua_state, LUA_REGISTRYINDEX); +} + +/** + * \brief Register Suricata Lua functions + */ +int LuajitRegisterExtensions(lua_State *lua_state) { + lua_pushcfunction(lua_state, LuajitGetFlowvar); + lua_setglobal(lua_state, "ScFlowvarGet"); + + lua_pushcfunction(lua_state, LuajitSetFlowvar); + lua_setglobal(lua_state, "ScFlowvarSet"); + return 0; +} + +#endif /* HAVE_LUAJIT */ diff --git a/src/detect-luajit-extensions.h b/src/detect-luajit-extensions.h new file mode 100644 index 0000000000..87f3c056ad --- /dev/null +++ b/src/detect-luajit-extensions.h @@ -0,0 +1,35 @@ +/* Copyright (C) 2007-2013 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Victor Julien + */ + +#ifndef __DETECT_LUAJIT_EXT_H__ +#define __DETECT_LUAJIT_EXT_H__ + +#ifdef HAVE_LUAJIT +int LuajitRegisterExtensions(lua_State *); + +void LuajitExtensionsMatchSetup(lua_State *lua_state, + DetectLuajitData *, DetectEngineThreadCtx *det_ctx, + Flow *f, int need_flow_lock); + +#endif /* HAVE_LUAJIT */ +#endif diff --git a/src/detect-luajit.c b/src/detect-luajit.c index 866a4a6da8..26c1e6e3ad 100644 --- a/src/detect-luajit.c +++ b/src/detect-luajit.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2012 Open Information Security Foundation +/* Copyright (C) 2007-2013 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -53,9 +53,11 @@ #include "stream-tcp.h" #include "detect-luajit.h" +#include "detect-luajit-extensions.h" #include "queue.h" #include "util-cpu.h" +#include "util-var-name.h" #ifndef HAVE_LUAJIT @@ -241,7 +243,10 @@ void LuaDumpStack(lua_State *state) { } } -int DetectLuajitMatchBuffer(DetectEngineThreadCtx *det_ctx, Signature *s, SigMatch *sm, uint8_t *buffer, uint32_t buffer_len, uint32_t offset) { +int DetectLuajitMatchBuffer(DetectEngineThreadCtx *det_ctx, Signature *s, SigMatch *sm, + uint8_t *buffer, uint32_t buffer_len, uint32_t offset, + Flow *f, int need_flow_lock) +{ SCEnter(); int ret = 0; @@ -256,6 +261,10 @@ int DetectLuajitMatchBuffer(DetectEngineThreadCtx *det_ctx, Signature *s, SigMat if (tluajit == NULL) SCReturnInt(0); + /* setup extension data for use in lua c functions */ + LuajitExtensionsMatchSetup(tluajit->luastate, luajit, det_ctx, f, need_flow_lock); + + /* prepare data to pass to script */ lua_getglobal(tluajit->luastate, "match"); lua_newtable(tluajit->luastate); /* stack at -1 */ @@ -348,6 +357,9 @@ static int DetectLuajitMatch (ThreadVars *tv, DetectEngineThreadCtx *det_ctx, if (tluajit == NULL) SCReturnInt(0); + /* setup extension data for use in lua c functions */ + LuajitExtensionsMatchSetup(tluajit->luastate, luajit, det_ctx, p->flow, /* flow not locked */0); + if ((tluajit->flags & DATATYPE_PAYLOAD) && p->payload_len == 0) SCReturnInt(0); if ((tluajit->flags & DATATYPE_PACKET) && GET_PKT_LEN(p) == 0) @@ -456,7 +468,14 @@ static int DetectLuajitMatch (ThreadVars *tv, DetectEngineThreadCtx *det_ctx, SCReturnInt(ret); } +#ifdef UNITTESTS +/* if this ptr is set the luajit setup functions will use this buffer as the + * lua script instead of calling luaL_loadfile on the filename supplied. */ +static const char *ut_script = NULL; +#endif + static void *DetectLuajitThreadInit(void *data) { + int status; DetectLuajitData *luajit = (DetectLuajitData *)data; BUG_ON(luajit == NULL); @@ -478,11 +497,26 @@ static void *DetectLuajitThreadInit(void *data) { luaL_openlibs(t->luastate); - int status = luaL_loadfile(t->luastate, luajit->filename); - if (status) { - SCLogError(SC_ERR_LUAJIT_ERROR, "couldn't load file: %s", lua_tostring(t->luastate, -1)); - goto error; + LuajitRegisterExtensions(t->luastate); + + /* hackish, needed to allow unittests to pass buffers as scripts instead of files */ +#ifdef UNITTESTS + if (ut_script != NULL) { + status = luaL_loadbuffer(t->luastate, ut_script, strlen(ut_script), "unittest"); + if (status) { + SCLogError(SC_ERR_LUAJIT_ERROR, "couldn't load file: %s", lua_tostring(t->luastate, -1)); + goto error; + } + } else { +#endif + status = luaL_loadfile(t->luastate, luajit->filename); + if (status) { + SCLogError(SC_ERR_LUAJIT_ERROR, "couldn't load file: %s", lua_tostring(t->luastate, -1)); + goto error; + } +#ifdef UNITTESTS } +#endif /* prime the script (or something) */ if (lua_pcall(t->luastate, 0, 0, 0) != 0) { @@ -546,17 +580,32 @@ error: return NULL; } -static int DetectLuaSetupPrime(DetectLuajitData *ld) { +static int DetectLuaSetupPrime(DetectEngineCtx *de_ctx, DetectLuajitData *ld) { + int status; + lua_State *luastate = luaL_newstate(); if (luastate == NULL) goto error; luaL_openlibs(luastate); - int status = luaL_loadfile(luastate, ld->filename); - if (status) { - SCLogError(SC_ERR_LUAJIT_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1)); - goto error; + /* hackish, needed to allow unittests to pass buffers as scripts instead of files */ +#ifdef UNITTESTS + if (ut_script != NULL) { + status = luaL_loadbuffer(luastate, ut_script, strlen(ut_script), "unittest"); + if (status) { + SCLogError(SC_ERR_LUAJIT_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1)); + goto error; + } + } else { +#endif + status = luaL_loadfile(luastate, ld->filename); + if (status) { + SCLogError(SC_ERR_LUAJIT_ERROR, "couldn't load file: %s", lua_tostring(luastate, -1)); + goto error; + } +#ifdef UNITTESTS } +#endif /* prime the script (or something) */ if (lua_pcall(luastate, 0, 0, 0) != 0) { @@ -598,10 +647,38 @@ static int DetectLuaSetupPrime(DetectLuajitData *ld) { lua_pushnil(luastate); const char *k, *v; while (lua_next(luastate, -2)) { + k = lua_tostring(luastate, -2); + if (k == NULL) + continue; + + /* handle flowvar separately as it has a table as value */ + if (strcmp(k, "flowvar") == 0) { + if (lua_istable(luastate, -1)) { + lua_pushnil(luastate); + while (lua_next(luastate, -2) != 0) { + /* value at -1, key is at -2 which we ignore */ + const char *value = lua_tostring(luastate, -1); + SCLogDebug("value %s", value); + /* removes 'value'; keeps 'key' for next iteration */ + lua_pop(luastate, 1); + + if (ld->flowvars == DETECT_LUAJIT_MAX_FLOWVARS) { + SCLogError(SC_ERR_LUAJIT_ERROR, "too many flowvars registered"); + goto error; + } + + uint16_t idx = VariableNameGetIdx(de_ctx, (char *)value, DETECT_FLOWVAR); + ld->flowvar[ld->flowvars++] = idx; + SCLogDebug("script uses flowvar %u with script id %u", idx, ld->flowvars - 1); + } + } + lua_pop(luastate, 1); + continue; + } + v = lua_tostring(luastate, -1); lua_pop(luastate, 1); - k = lua_tostring(luastate, -1); - if (!k || !v) + if (v == NULL) continue; SCLogDebug("k='%s', v='%s'", k, v); @@ -704,7 +781,7 @@ static int DetectLuajitSetup (DetectEngineCtx *de_ctx, Signature *s, char *str) if (luajit == NULL) goto error; - if (DetectLuaSetupPrime(luajit) == -1) { + if (DetectLuaSetupPrime(de_ctx, luajit) == -1) { goto error; } @@ -783,14 +860,454 @@ static void DetectLuajitFree(void *ptr) { } #ifdef UNITTESTS +/** \test http buffer */ static int LuajitMatchTest01(void) { - return 1; + const char script[] = + "function init (args)\n" + " local needs = {}\n" + " needs[\"http.request_headers\"] = tostring(true)\n" + " needs[\"flowvar\"] = {\"cnt\"}\n" + " return needs\n" + "end\n" + "\n" + "function match(args)\n" + " a = ScFlowvarGet(0)\n" + " if a then\n" + " a = tostring(tonumber(a)+1)\n" + " print (a)\n" + " ScFlowvarSet(0, a, #a)\n" + " else\n" + " a = tostring(1)\n" + " print (a)\n" + " ScFlowvarSet(0, a, #a)\n" + " end\n" + " \n" + " print (\"pre check: \" .. (a))\n" + " if tonumber(a) == 2 then\n" + " print \"match\"\n" + " return 1\n" + " end\n" + " return 0\n" + "end\n" + "return 0\n"; + char sig[] = "alert http any any -> any any (flow:to_server; luajit:unittest; sid:1;)"; + int result = 0; + uint8_t httpbuf1[] = + "POST / HTTP/1.1\r\n" + "Host: www.emergingthreats.net\r\n" + "User-Agent: Mozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.9.0.13) Gecko/2009080315 Ubuntu/8.10 (intrepid) Firefox/3.0.13\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9;q=0.8\r\n"; + uint8_t httpbuf2[] = + "Accept-Language: es-es,es;q=0.8,en-us;q=0.5,en;q=0.3\r\n" + "Accept-Encoding: gzip,deflate\r\n" + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + "Date: Tue, 22 Sep 2009 19:24:48 GMT\r\n" + "Server: Apache\r\n" + "Content-Length: 500\r\n" + "\r\n" + "flow = &f; + p1->flowflags |= FLOW_PKT_TOSERVER; + p1->flowflags |= FLOW_PKT_ESTABLISHED; + p1->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + p2->flow = &f; + p2->flowflags |= FLOW_PKT_TOSERVER; + p2->flowflags |= FLOW_PKT_ESTABLISHED; + p2->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + + StreamTcpInitConfig(TRUE); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + if (de_ctx == NULL) { + goto end; + } + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, sig); + if (s == NULL) { + printf("sig parse failed: "); + goto end; + } + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + int r = AppLayerParse(NULL, &f, ALPROTO_HTTP, STREAM_TOSERVER, httpbuf1, httplen1); + if (r != 0) { + printf("toserver chunk 1 returned %" PRId32 ", expected 0: ", r); + goto end; + } + HtpState *http_state = f.alstate; + if (http_state == NULL) { + printf("no http state: "); + goto end; + } + + /* do detect for p1 */ + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + + if ((PacketAlertCheck(p1, 1))) { + printf("sid 1 didn't match on p1 but should have: "); + goto end; + } + + r = AppLayerParse(NULL, &f, ALPROTO_HTTP, STREAM_TOSERVER, httpbuf2, httplen2); + if (r != 0) { + printf("toserver chunk 2 returned %" PRId32 ", expected 0: ", r); + goto end; + } + /* do detect for p2 */ + SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); + + if (!(PacketAlertCheck(p2, 1))) { + printf("sid 1 didn't match on p2 but should have: "); + goto end; + } + + FlowVar *fv = FlowVarGet(&f, 1); + if (fv == NULL) { + printf("no flowvar: "); + goto end; + } + + if (fv->data.fv_str.value_len != 1) { + printf("%u != %u: ", fv->data.fv_str.value_len, 1); + goto end; + } + + if (memcmp(fv->data.fv_str.value, "2", 1) != 0) { + PrintRawDataFp(stdout, fv->data.fv_str.value, fv->data.fv_str.value_len); + + printf("buffer mismatch: "); + goto end; + } + + result = 1; +end: + if (de_ctx != NULL) + DetectEngineCtxFree(de_ctx); + + StreamTcpFreeConfig(TRUE); + FLOW_DESTROY(&f); + UTHFreePackets(&p1, 1); + UTHFreePackets(&p2, 1); + return result; +} + +/** \test payload buffer */ +static int LuajitMatchTest02(void) { + const char script[] = + "function init (args)\n" + " local needs = {}\n" + " needs[\"payload\"] = tostring(true)\n" + " needs[\"flowvar\"] = {\"cnt\"}\n" + " return needs\n" + "end\n" + "\n" + "function match(args)\n" + " a = ScFlowvarGet(0)\n" + " if a then\n" + " a = tostring(tonumber(a)+1)\n" + " print (a)\n" + " ScFlowvarSet(0, a, #a)\n" + " else\n" + " a = tostring(1)\n" + " print (a)\n" + " ScFlowvarSet(0, a, #a)\n" + " end\n" + " \n" + " print (\"pre check: \" .. (a))\n" + " if tonumber(a) == 2 then\n" + " print \"match\"\n" + " return 1\n" + " end\n" + " return 0\n" + "end\n" + "return 0\n"; + char sig[] = "alert tcp any any -> any any (flow:to_server; luajit:unittest; sid:1;)"; + int result = 0; + uint8_t httpbuf1[] = + "POST / HTTP/1.1\r\n" + "Host: www.emergingthreats.net\r\n" + "User-Agent: Mozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.9.0.13) Gecko/2009080315 Ubuntu/8.10 (intrepid) Firefox/3.0.13\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9;q=0.8\r\n"; + uint8_t httpbuf2[] = + "Accept-Language: es-es,es;q=0.8,en-us;q=0.5,en;q=0.3\r\n" + "Accept-Encoding: gzip,deflate\r\n" + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + "Date: Tue, 22 Sep 2009 19:24:48 GMT\r\n" + "Server: Apache\r\n" + "Content-Length: 500\r\n" + "\r\n" + "flow = &f; + p1->flowflags |= FLOW_PKT_TOSERVER; + p1->flowflags |= FLOW_PKT_ESTABLISHED; + p1->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + p2->flow = &f; + p2->flowflags |= FLOW_PKT_TOSERVER; + p2->flowflags |= FLOW_PKT_ESTABLISHED; + p2->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + + StreamTcpInitConfig(TRUE); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + if (de_ctx == NULL) { + goto end; + } + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, sig); + if (s == NULL) { + printf("sig parse failed: "); + goto end; + } + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + /* do detect for p1 */ + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + + if ((PacketAlertCheck(p1, 1))) { + printf("sid 1 didn't match on p1 but should have: "); + goto end; + } + + /* do detect for p2 */ + SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); + + if (!(PacketAlertCheck(p2, 1))) { + printf("sid 1 didn't match on p2 but should have: "); + goto end; + } + + FlowVar *fv = FlowVarGet(&f, 1); + if (fv == NULL) { + printf("no flowvar: "); + goto end; + } + + if (fv->data.fv_str.value_len != 1) { + printf("%u != %u: ", fv->data.fv_str.value_len, 1); + goto end; + } + + if (memcmp(fv->data.fv_str.value, "2", 1) != 0) { + PrintRawDataFp(stdout, fv->data.fv_str.value, fv->data.fv_str.value_len); + + printf("buffer mismatch: "); + goto end; + } + + result = 1; +end: + if (de_ctx != NULL) + DetectEngineCtxFree(de_ctx); + + StreamTcpFreeConfig(TRUE); + FLOW_DESTROY(&f); + UTHFreePackets(&p1, 1); + UTHFreePackets(&p2, 1); + return result; +} + +/** \test packet buffer */ +static int LuajitMatchTest03(void) { + const char script[] = + "function init (args)\n" + " local needs = {}\n" + " needs[\"packet\"] = tostring(true)\n" + " needs[\"flowvar\"] = {\"cnt\"}\n" + " return needs\n" + "end\n" + "\n" + "function match(args)\n" + " a = ScFlowvarGet(0)\n" + " if a then\n" + " a = tostring(tonumber(a)+1)\n" + " print (a)\n" + " ScFlowvarSet(0, a, #a)\n" + " else\n" + " a = tostring(1)\n" + " print (a)\n" + " ScFlowvarSet(0, a, #a)\n" + " end\n" + " \n" + " print (\"pre check: \" .. (a))\n" + " if tonumber(a) == 2 then\n" + " print \"match\"\n" + " return 1\n" + " end\n" + " return 0\n" + "end\n" + "return 0\n"; + char sig[] = "alert tcp any any -> any any (flow:to_server; luajit:unittest; sid:1;)"; + int result = 0; + uint8_t httpbuf1[] = + "POST / HTTP/1.1\r\n" + "Host: www.emergingthreats.net\r\n" + "User-Agent: Mozilla/5.0 (X11; U; Linux i686; es-ES; rv:1.9.0.13) Gecko/2009080315 Ubuntu/8.10 (intrepid) Firefox/3.0.13\r\n" + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9;q=0.8\r\n"; + uint8_t httpbuf2[] = + "Accept-Language: es-es,es;q=0.8,en-us;q=0.5,en;q=0.3\r\n" + "Accept-Encoding: gzip,deflate\r\n" + "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n" + "Date: Tue, 22 Sep 2009 19:24:48 GMT\r\n" + "Server: Apache\r\n" + "Content-Length: 500\r\n" + "\r\n" + "flow = &f; + p1->flowflags |= FLOW_PKT_TOSERVER; + p1->flowflags |= FLOW_PKT_ESTABLISHED; + p1->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + p2->flow = &f; + p2->flowflags |= FLOW_PKT_TOSERVER; + p2->flowflags |= FLOW_PKT_ESTABLISHED; + p2->flags |= PKT_HAS_FLOW|PKT_STREAM_EST; + + StreamTcpInitConfig(TRUE); + + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + if (de_ctx == NULL) { + goto end; + } + de_ctx->flags |= DE_QUIET; + + s = DetectEngineAppendSig(de_ctx, sig); + if (s == NULL) { + printf("sig parse failed: "); + goto end; + } + + SigGroupBuild(de_ctx); + DetectEngineThreadCtxInit(&th_v, (void *)de_ctx, (void *)&det_ctx); + + /* do detect for p1 */ + SigMatchSignatures(&th_v, de_ctx, det_ctx, p1); + + if ((PacketAlertCheck(p1, 1))) { + printf("sid 1 didn't match on p1 but should have: "); + goto end; + } + + /* do detect for p2 */ + SigMatchSignatures(&th_v, de_ctx, det_ctx, p2); + + if (!(PacketAlertCheck(p2, 1))) { + printf("sid 1 didn't match on p2 but should have: "); + goto end; + } + + FlowVar *fv = FlowVarGet(&f, 1); + if (fv == NULL) { + printf("no flowvar: "); + goto end; + } + + if (fv->data.fv_str.value_len != 1) { + printf("%u != %u: ", fv->data.fv_str.value_len, 1); + goto end; + } + + if (memcmp(fv->data.fv_str.value, "2", 1) != 0) { + PrintRawDataFp(stdout, fv->data.fv_str.value, fv->data.fv_str.value_len); + + printf("buffer mismatch: "); + goto end; + } + + result = 1; +end: + if (de_ctx != NULL) + DetectEngineCtxFree(de_ctx); + + StreamTcpFreeConfig(TRUE); + FLOW_DESTROY(&f); + UTHFreePackets(&p1, 1); + UTHFreePackets(&p2, 1); + return result; } #endif void DetectLuajitRegisterTests(void) { #ifdef UNITTESTS UtRegisterTest("LuajitMatchTest01", LuajitMatchTest01, 1); + UtRegisterTest("LuajitMatchTest02", LuajitMatchTest02, 1); + UtRegisterTest("LuajitMatchTest03", LuajitMatchTest03, 1); #endif } diff --git a/src/detect-luajit.h b/src/detect-luajit.h index c96dfa5872..ad14d7a3a7 100644 --- a/src/detect-luajit.h +++ b/src/detect-luajit.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2012 Open Information Security Foundation +/* Copyright (C) 2007-2013 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -36,6 +36,8 @@ typedef struct DetectLuajitThreadData { int alproto; } DetectLuajitThreadData; +#define DETECT_LUAJIT_MAX_FLOWVARS 15 + typedef struct DetectLuajitData { int thread_ctx_id; int negated; @@ -43,12 +45,16 @@ typedef struct DetectLuajitData { uint32_t flags; int alproto; char *buffername; /* buffer name in case of a single buffer */ + uint16_t flowvar[DETECT_LUAJIT_MAX_FLOWVARS]; + uint16_t flowvars; } DetectLuajitData; #endif /* prototypes */ void DetectLuajitRegister (void); -int DetectLuajitMatchBuffer(DetectEngineThreadCtx *det_ctx, Signature *s, SigMatch *sm, uint8_t *buffer, uint32_t buffer_len, uint32_t offset); +int DetectLuajitMatchBuffer(DetectEngineThreadCtx *det_ctx, Signature *s, SigMatch *sm, + uint8_t *buffer, uint32_t buffer_len, uint32_t offset, + Flow *f, int need_flow_lock); int DetectLuajitSetupStatesPool(int num, int reloads); diff --git a/src/detect-pcre.c b/src/detect-pcre.c index 6749585fe2..8af36fe867 100644 --- a/src/detect-pcre.c +++ b/src/detect-pcre.c @@ -226,8 +226,11 @@ int DetectPcrePayloadMatch(DetectEngineThreadCtx *det_ctx, Signature *s, /* regex matched and we're not negated, * considering it a match */ + SCLogDebug("ret %d capidx %u", ret, pe->capidx); + /* see if we need to do substring capturing. */ if (ret > 1 && pe->capidx != 0) { + SCLogDebug("capturing"); const char *str_ptr; ret = pcre_get_substring((char *)ptr, ov, MAX_SUBSTRINGS, 1, &str_ptr); if (ret) { @@ -240,7 +243,8 @@ int DetectPcrePayloadMatch(DetectEngineThreadCtx *det_ctx, Signature *s, /* store max 64k. Errors are ignored */ capture_len = (ret < 0xffff) ? (uint16_t)ret : 0xffff; (void)DetectFlowvarStoreMatch(det_ctx, pe->capidx, - (uint8_t *)str_ptr, capture_len); + (uint8_t *)str_ptr, capture_len, + DETECT_FLOWVAR_TYPE_POSTMATCH); } } } @@ -589,12 +593,13 @@ DetectPcreData *DetectPcreParseCapture(char *regexstr, DetectEngineCtx *de_ctx, int ov[MAX_SUBSTRINGS]; const char *capture_str_ptr = NULL, *type_str_ptr = NULL; - if(pd == NULL) + if (pd == NULL) goto error; - if(de_ctx == NULL) + if (de_ctx == NULL) goto error; - //printf("DetectPcreParseCapture: \'%s\'\n", regexstr); + + SCLogDebug("\'%s\'", regexstr); ret = pcre_exec(parse_capture_regex, parse_capture_regex_study, regexstr, strlen(regexstr), 0, 0, ov, MAX_SUBSTRINGS); if (ret > 1) { @@ -609,8 +614,8 @@ DetectPcreData *DetectPcreParseCapture(char *regexstr, DetectEngineCtx *de_ctx, goto error; } } - //printf("DetectPcreParseCapture: type \'%s\'\n", type_str_ptr ? type_str_ptr : "NULL"); - //printf("DetectPcreParseCapture: capture \'%s\'\n", capture_str_ptr ? capture_str_ptr : "NULL"); + SCLogDebug("type \'%s\'", type_str_ptr ? type_str_ptr : "NULL"); + SCLogDebug("capture \'%s\'", capture_str_ptr ? capture_str_ptr : "NULL"); if (capture_str_ptr != NULL) { pd->capname = SCStrdup((char *)capture_str_ptr); @@ -621,6 +626,7 @@ DetectPcreData *DetectPcreParseCapture(char *regexstr, DetectEngineCtx *de_ctx, pd->flags |= DETECT_PCRE_CAPTURE_PKT; } else if (strcmp(type_str_ptr,"flow") == 0) { pd->flags |= DETECT_PCRE_CAPTURE_FLOW; + SCLogDebug("flow capture"); } if (capture_str_ptr != NULL) { if (pd->flags & DETECT_PCRE_CAPTURE_PKT) @@ -629,17 +635,20 @@ DetectPcreData *DetectPcreParseCapture(char *regexstr, DetectEngineCtx *de_ctx, pd->capidx = VariableNameGetIdx(de_ctx, (char *)capture_str_ptr, DETECT_FLOWVAR); } } - //printf("DetectPcreParseCapture: pd->capname %s\n", pd->capname ? pd->capname : "NULL"); + SCLogDebug("pd->capname %s", pd->capname ? pd->capname : "NULL"); - if (type_str_ptr != NULL) pcre_free((char *)type_str_ptr); - if (capture_str_ptr != NULL) pcre_free((char *)capture_str_ptr); + if (type_str_ptr != NULL) + pcre_free((char *)type_str_ptr); + if (capture_str_ptr != NULL) + pcre_free((char *)capture_str_ptr); return pd; error: - if (pd != NULL && pd->capname != NULL) SCFree(pd->capname); - if (pd) SCFree(pd); + if (pd != NULL && pd->capname != NULL) + SCFree(pd->capname); + if (pd) + SCFree(pd); return NULL; - } static int DetectPcreSetup (DetectEngineCtx *de_ctx, Signature *s, char *regexstr) diff --git a/src/detect.c b/src/detect.c index ade75151cd..49e5d421f8 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1629,7 +1629,7 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh p->action |= s->action; } next: - DetectFlowvarCleanupList(det_ctx); + DetectFlowvarProcessList(det_ctx, p->flow); DetectReplaceFree(det_ctx->replist); det_ctx->replist = NULL; RULE_PROFILING_END(det_ctx, s, smatch); diff --git a/src/detect.h b/src/detect.h index a3e4906960..0513e639f1 100644 --- a/src/detect.h +++ b/src/detect.h @@ -468,6 +468,11 @@ typedef struct DetectReplaceList_ { struct DetectReplaceList_ *next; } DetectReplaceList; +/** only execute flowvar storage if rule matched */ +#define DETECT_FLOWVAR_TYPE_POSTMATCH 1 +/** execute flowvar storage even if rule doesn't match (for luajit) */ +#define DETECT_FLOWVAR_TYPE_ALWAYS 2 + /** list for flowvar store candidates, to be stored from * post-match function */ typedef struct DetectFlowvarList_ { @@ -475,6 +480,7 @@ typedef struct DetectFlowvarList_ { uint16_t len; /**< data len */ uint8_t *buffer; /**< alloc'd buffer, may be freed by post-match, post-non-match */ + int type; /**< type of store candidate POSTMATCH or ALWAYS */ struct DetectFlowvarList_ *next; } DetectFlowvarList; diff --git a/src/flow-var.c b/src/flow-var.c index 7532b4f84c..979475bcbd 100644 --- a/src/flow-var.c +++ b/src/flow-var.c @@ -34,7 +34,8 @@ /* puts a new value into a flowvar */ void FlowVarUpdateStr(FlowVar *fv, uint8_t *value, uint16_t size) { - if (fv->data.fv_str.value) SCFree(fv->data.fv_str.value); + if (fv->data.fv_str.value) + SCFree(fv->data.fv_str.value); fv->data.fv_str.value = value; fv->data.fv_str.value_len = size; }