aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

multi-detect: detect loader for unix socket

Browse Source 
Move the tenant load and reload commands to be executed by the detect
loader thread(s).

Limitation: no yaml parsing in parallel. The Conf API is currently not
thread safe, so don't load the tenant config (yaml) in parallel.
pull/1608/head
Victor Julien 10 years ago
parent eb09118d64
commit b7b27684c2
3 changed files with 147 additions and 39 deletions
Show Stats Download Patch File Download Diff File

138
src/detect-engine.c
Unescape Escape View File

@ -1673,7 +1673,8 @@ int DetectEngineMultiTenantEnabled(void)
return (master->multi_tenant_enabled);
}
/** \brief load a tenant from a yaml file
/** \internal
* \brief load a tenant from a yaml file
*
* \param tenant_id the tenant id by which the config is known
* \param filename full path of a yaml file
@ -1682,7 +1683,7 @@ int DetectEngineMultiTenantEnabled(void)
* \retval 0 ok
* \retval -1 failed
*/
int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *filename, int loader_id)
static int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *filename, int loader_id)
{
DetectEngineCtx *de_ctx = NULL;
char prefix[64];
@ -1708,11 +1709,6 @@ int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *filename,
goto error;
}
if (ConfYamlLoadFileWithPrefix(filename, prefix) != 0) {
SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to load yaml %s", filename);
goto error;
}
ConfNode *node = ConfGetNode(prefix);
if (node == NULL) {
SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to properly setup yaml %s", filename);
@ -1728,6 +1724,7 @@ int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *filename,
SCLogDebug("de_ctx %p with prefix %s", de_ctx, de_ctx->config_prefix);
de_ctx->tenant_id = tenant_id;
de_ctx->loader_id = loader_id;
if (SigLoadSignatures(de_ctx, NULL, 0) < 0) {
SCLogError(SC_ERR_NO_RULES_LOADED, "Loading signatures failed.");
@ -1745,6 +1742,59 @@ error:
return -1;
}
static int DetectEngineMultiTenantReloadTenant(uint32_t tenant_id, const char *filename, int reload_cnt)
{
DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
if (old_de_ctx == NULL) {
SCLogError(SC_ERR_INITIALIZATION, "tenant detect engine not found");
return -1;
}
char prefix[64];
snprintf(prefix, sizeof(prefix), "multi-detect.%d.reload.%d", tenant_id, reload_cnt);
reload_cnt++;
SCLogInfo("prefix %s", prefix);
if (ConfYamlLoadFileWithPrefix(filename, prefix) != 0) {
SCLogError(SC_ERR_INITIALIZATION,"failed to load yaml");
goto error;
}
ConfNode *node = ConfGetNode(prefix);
if (node == NULL) {
SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to properly setup yaml %s", filename);
goto error;
}
DetectEngineCtx *new_de_ctx = DetectEngineCtxInitWithPrefix(prefix);
if (new_de_ctx == NULL) {
SCLogError(SC_ERR_INITIALIZATION, "initializing detection engine "
"context failed.");
goto error;
}
SCLogDebug("de_ctx %p with prefix %s", new_de_ctx, new_de_ctx->config_prefix);
new_de_ctx->tenant_id = tenant_id;
new_de_ctx->loader_id = old_de_ctx->loader_id;
if (SigLoadSignatures(new_de_ctx, NULL, 0) < 0) {
SCLogError(SC_ERR_NO_RULES_LOADED, "Loading signatures failed.");
goto error;
}
DetectEngineAddToMaster(new_de_ctx);
/* move to free list */
DetectEngineMoveToFreeList(old_de_ctx);
DetectEngineDeReference(&old_de_ctx);
return 0;
error:
DetectEngineDeReference(&old_de_ctx);
return -1;
}
/**
* \param ctx function specific data
* \param loader_id id of the loader that executed the task
@ -1852,6 +1902,7 @@ void DetectLoadersInit(void)
typedef struct TenantLoaderCtx_ {
uint32_t tenant_id;
int reload_cnt; /**< used by reload */
const char *yaml;
} TenantLoaderCtx;
@ -1859,7 +1910,7 @@ static int DetectLoaderFuncLoadTenant(void *vctx, int loader_id)
{
TenantLoaderCtx *ctx = (TenantLoaderCtx *)vctx;
/* TODO we need to somehow store the loader id for when we free */
SCLogInfo("loader %d", loader_id);
if (DetectEngineMultiTenantLoadTenant(ctx->tenant_id, ctx->yaml, loader_id) != 0) {
return -1;
}
@ -1878,6 +1929,67 @@ int DetectLoaderSetupLoadTenant(uint32_t tenant_id, const char *yaml)
return DetectLoaderQueueTask(-1, DetectLoaderFuncLoadTenant, t);
}
static int DetectLoaderFuncReloadTenant(void *vctx, int loader_id)
{
TenantLoaderCtx *ctx = (TenantLoaderCtx *)vctx;
SCLogDebug("loader_id %d", loader_id);
if (DetectEngineMultiTenantReloadTenant(ctx->tenant_id, ctx->yaml, ctx->reload_cnt) != 0) {
return -1;
}
return 0;
}
int DetectLoaderSetupReloadTenant(uint32_t tenant_id, const char *yaml, int reload_cnt)
{
DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
if (old_de_ctx == NULL)
return -ENOENT;
int loader_id = old_de_ctx->loader_id;
DetectEngineDeReference(&old_de_ctx);
TenantLoaderCtx *t = SCCalloc(1, sizeof(*t));
if (t == NULL)
return -ENOMEM;
t->tenant_id = tenant_id;
t->yaml = yaml;
t->reload_cnt = reload_cnt;
SCLogDebug("loader_id %d", loader_id);
return DetectLoaderQueueTask(loader_id, DetectLoaderFuncReloadTenant, t);
}
/** \brief Load a tenant and wait for loading to complete
*/
int DetectEngineLoadTenantBlocking(uint32_t tenant_id, const char *yaml)
{
int r = DetectLoaderSetupLoadTenant(tenant_id, yaml);
if (r < 0)
return r;
if (DetectLoadersSync() != 0)
return -1;
return 0;
}
/** \brief Reload a tenant and wait for loading to complete
*/
int DetectEngineReloadTenantBlocking(uint32_t tenant_id, const char *yaml, int reload_cnt)
{
int r = DetectLoaderSetupReloadTenant(tenant_id, yaml, reload_cnt);
if (r < 0)
return r;
if (DetectLoadersSync() != 0)
return -1;
return 0;
}
/**
* \brief Unpauses all threads present in tv_root
*/
@ -1997,6 +2109,7 @@ static TmEcode DetectLoader(ThreadVars *th_v, void *thread_data)
SCLogDebug("woke up...");
}
return TM_ECODE_OK;
}
@ -2158,6 +2271,15 @@ void DetectEngineMultiTenantSetup(void)
}
SCLogInfo("tenant id: %u, %s", tenant_id, yaml_node->val);
/* setup the yaml in this loop so that it's not done by the loader
* threads. ConfYamlLoadFileWithPrefix is not thread safe. */
char prefix[64];
snprintf(prefix, sizeof(prefix), "multi-detect.%d", tenant_id);
if (ConfYamlLoadFileWithPrefix(yaml_node->val, prefix) != 0) {
SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to load yaml %s", yaml_node->val);
goto bad_tenant;
}
if (DetectLoaderSetupLoadTenant(tenant_id, yaml_node->val) != 0) {
/* error logged already */
goto bad_tenant;

3
src/detect-engine.h
Unescape Escape View File

@ -86,7 +86,8 @@ int DetectEngineReloadIsStart(void);
void DetectEngineReloadSetDone(void);
int DetectEngineReloadIsDone(void);
int DetectEngineMultiTenantLoadTenant(uint32_t tenant_id, const char *filename, int loader_id);
int DetectEngineLoadTenantBlocking(uint32_t tenant_id, const char *yaml);
int DetectEngineReloadTenantBlocking(uint32_t tenant_id, const char *yaml, int reload_cnt);
int DetectEngineTentantRegisterVlanId(uint32_t tenant_id, uint16_t vlan_id);
int DetectEngineTentantUnregisterVlanId(uint32_t tenant_id, uint16_t vlan_id);

45
src/runmode-unix-socket.c
Unescape Escape View File

@ -638,8 +638,18 @@ TmEcode UnixSocketRegisterTenant(json_t *cmd, json_t* answer, void *data)
SCLogDebug("add-tenant: %d %s", tenant_id, filename);
/* setup the yaml in this loop so that it's not done by the loader
* threads. ConfYamlLoadFileWithPrefix is not thread safe. */
char prefix[64];
snprintf(prefix, sizeof(prefix), "multi-detect.%d", tenant_id);
if (ConfYamlLoadFileWithPrefix(filename, prefix) != 0) {
SCLogError(SC_ERR_CONF_YAML_ERROR, "failed to load yaml %s", filename);
json_object_set_new(answer, "message", json_string("failed to load yaml"));
return TM_ECODE_FAILED;
}
/* 3 load into the system */
if (DetectEngineMultiTenantLoadTenant(tenant_id, filename, -1) != 0) {
if (DetectEngineLoadTenantBlocking(tenant_id, filename) != 0) {
json_object_set_new(answer, "message", json_string("adding tenant failed"));
return TM_ECODE_FAILED;
}
@ -704,15 +714,8 @@ TmEcode UnixSocketReloadTenant(json_t *cmd, json_t* answer, void *data)
SCLogDebug("reload-tenant: %d %s", tenant_id, filename);
DetectEngineCtx *old_de_ctx = DetectEngineGetByTenantId(tenant_id);
if (old_de_ctx == NULL) {
json_object_set_new(answer, "message", json_string("tenant detect engine not found"));
return TM_ECODE_FAILED;
}
char prefix[64];
snprintf(prefix, sizeof(prefix), "multi-detect.%d.reload.%d", tenant_id, reload_cnt);
reload_cnt++;
SCLogInfo("prefix %s", prefix);
if (ConfYamlLoadFileWithPrefix(filename, prefix) != 0) {
@ -720,31 +723,13 @@ TmEcode UnixSocketReloadTenant(json_t *cmd, json_t* answer, void *data)
return TM_ECODE_FAILED;
}
ConfNode *node = ConfGetNode(prefix);
if (node == NULL) {
json_object_set_new(answer, "message", json_string("failed to properly setup yaml"));
return TM_ECODE_FAILED;
}
DetectEngineCtx *new_de_ctx = DetectEngineCtxInitWithPrefix(prefix);
if (new_de_ctx == NULL) {
json_object_set_new(answer, "message", json_string("initializing detection engine failed"));
return TM_ECODE_FAILED;
}
SCLogDebug("de_ctx %p with prefix %s", new_de_ctx, new_de_ctx->config_prefix);
new_de_ctx->tenant_id = tenant_id;
if (SigLoadSignatures(new_de_ctx, NULL, 0) < 0) {
json_object_set_new(answer, "message", json_string("loading rules failed"));
/* 3 load into the system */
if (DetectEngineReloadTenantBlocking(tenant_id, filename, reload_cnt) != 0) {
json_object_set_new(answer, "message", json_string("reload tenant failed"));
return TM_ECODE_FAILED;
}
DetectEngineAddToMaster(new_de_ctx);
/* move to free list */
DetectEngineMoveToFreeList(old_de_ctx);
DetectEngineDeReference(&old_de_ctx);
reload_cnt++;
/* apply to the running system */
if (DetectEngineMTApply() < 0) {

Write Preview
Loading…
Cancel
Save