From fd6df00684a503b69d303d6cffe2b6ba5ffbd531 Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Thu, 25 Oct 2012 13:39:13 +0200 Subject: [PATCH] Bug 585: use per detect thread libmagic ctx --- src/detect-engine.c | 15 ++++- src/detect-filemagic.c | 128 +++++++++++++++++++++++++++++++++++++++-- src/detect-filemagic.h | 8 ++- src/detect-luajit.c | 2 +- src/detect.h | 2 +- src/log-file.c | 2 +- src/log-filestore.c | 2 +- src/util-magic.c | 45 +++++++++++---- src/util-magic.h | 5 +- 9 files changed, 186 insertions(+), 23 deletions(-) diff --git a/src/detect-engine.c b/src/detect-engine.c index c91c5975ab..e6ed137df0 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -1234,6 +1234,7 @@ void DetectEngineThreadCtxInfo(ThreadVars *t, DetectEngineThreadCtx *det_ctx) { * \param InitFunc function ptr * \param data keyword init data to pass to Func * \param FreeFunc function ptr + * \param mode 0 normal (ctx per keyword instance) 1 shared (one ctx per det_ct) * * \retval id for retrieval of ctx at runtime * \retval -1 on error @@ -1242,9 +1243,20 @@ void DetectEngineThreadCtxInfo(ThreadVars *t, DetectEngineThreadCtx *det_ctx) { * recommended to store it in the keywords global ctx so that * it's freed when the de_ctx is freed. */ -int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *)) { +int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int mode) { BUG_ON(de_ctx == NULL || InitFunc == NULL || FreeFunc == NULL || data == NULL); + if (mode) { + DetectEngineThreadKeywordCtxItem *item = de_ctx->keyword_list; + while (item != NULL) { + if (strcmp(name, item->name) == 0) { + return item->id; + } + + item = item->next; + } + } + DetectEngineThreadKeywordCtxItem *item = SCMalloc(sizeof(DetectEngineThreadKeywordCtxItem)); if (unlikely(item == NULL)) return -1; @@ -1253,6 +1265,7 @@ int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void item->InitFunc = InitFunc; item->FreeFunc = FreeFunc; item->data = data; + item->name = name; item->next = de_ctx->keyword_list; de_ctx->keyword_list = item; diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index 3100308bff..4eef0243c5 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -52,6 +52,9 @@ #include "detect-filemagic.h" +#include "conf.h" +#include "util-magic.h" + static int DetectFilemagicMatch (ThreadVars *, DetectEngineThreadCtx *, Flow *, uint8_t, File *, Signature *, SigMatch *); static int DetectFilemagicSetup (DetectEngineCtx *, Signature *, char *); @@ -83,19 +86,68 @@ void DetectFilemagicRegister(void) { * \retval -1 error * \retval 0 ok */ -int FilemagicLookup(File *file) { +int FilemagicGlobalLookup(File *file) { if (file == NULL || file->chunks_head == NULL) { SCReturnInt(-1); } /* initial chunk already matching our requirement */ if (file->chunks_head->len >= FILEMAGIC_MIN_SIZE) { - file->magic = MagicLookup(file->chunks_head->data, FILEMAGIC_MIN_SIZE); + file->magic = MagicGlobalLookup(file->chunks_head->data, FILEMAGIC_MIN_SIZE); + } else { + uint8_t *buf = SCMalloc(FILEMAGIC_MIN_SIZE); + uint32_t size = 0; + + if (likely(buf != NULL)) { + FileData *ffd = file->chunks_head; + + for ( ; ffd != NULL; ffd = ffd->next) { + uint32_t copy_len = ffd->len; + if (size + ffd->len > FILEMAGIC_MIN_SIZE) + copy_len = FILEMAGIC_MIN_SIZE - size; + + memcpy(buf + size, ffd->data, copy_len); + size += copy_len; + + if (size >= FILEMAGIC_MIN_SIZE) { + file->magic = MagicGlobalLookup(buf, size); + break; + } + /* file is done but smaller than FILEMAGIC_MIN_SIZE */ + if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) { + file->magic = MagicGlobalLookup(buf, size); + break; + } + } + + SCFree(buf); + } + } + + SCReturnInt(0); +} + +/** + * \brief run the magic check + * + * \param file the file + * + * \retval -1 error + * \retval 0 ok + */ +int FilemagicThreadLookup(magic_t *ctx, File *file) { + if (ctx == NULL || file == NULL || file->chunks_head == NULL) { + SCReturnInt(-1); + } + + /* initial chunk already matching our requirement */ + if (file->chunks_head->len >= FILEMAGIC_MIN_SIZE) { + file->magic = MagicThreadLookup(ctx, file->chunks_head->data, FILEMAGIC_MIN_SIZE); } else { uint8_t *buf = SCMalloc(FILEMAGIC_MIN_SIZE); uint32_t size = 0; - if (buf != NULL) { + if (likely(buf != NULL)) { FileData *ffd = file->chunks_head; for ( ; ffd != NULL; ffd = ffd->next) { @@ -107,12 +159,12 @@ int FilemagicLookup(File *file) { size += copy_len; if (size >= FILEMAGIC_MIN_SIZE) { - file->magic = MagicLookup(buf, size); + file->magic = MagicThreadLookup(ctx, buf, size); break; } /* file is done but smaller than FILEMAGIC_MIN_SIZE */ if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) { - file->magic = MagicLookup(buf, size); + file->magic = MagicThreadLookup(ctx, buf, size); break; } } @@ -151,8 +203,13 @@ static int DetectFilemagicMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx, if (file->txid > det_ctx->tx_id) SCReturnInt(0); + DetectFilemagicThreadData *tfilemagic = (DetectFilemagicThreadData *)DetectThreadCtxGetKeywordThreadCtx(det_ctx, filemagic->thread_ctx_id); + if (tfilemagic == NULL) { + SCReturnInt(0); + } + if (file->magic == NULL) { - FilemagicLookup(file); + FilemagicThreadLookup(&tfilemagic->ctx, file); } if (file->magic != NULL) { @@ -240,6 +297,59 @@ error: return NULL; } +static void *DetectFilemagicThreadInit(void *data) { + char *filename = NULL; + FILE *fd = NULL; + DetectFilemagicData *filemagic = (DetectFilemagicData *)data; + BUG_ON(filemagic == NULL); + + DetectFilemagicThreadData *t = SCMalloc(sizeof(DetectFilemagicThreadData)); + if (unlikely(t == NULL)) { + SCLogError(SC_ERR_LUAJIT_ERROR, "couldn't alloc ctx memory"); + return NULL; + } + memset(t, 0x00, sizeof(DetectFilemagicThreadData)); + + t->ctx = magic_open(0); + if (t->ctx == NULL) { + SCLogError(SC_ERR_MAGIC_OPEN, "magic_open failed: %s", magic_error(t->ctx)); + goto error; + } + + (void)ConfGet("magic-file", &filename); + if (filename != NULL) { + SCLogInfo("using magic-file %s", filename); + + if ( (fd = fopen(filename, "r")) == NULL) { + SCLogWarning(SC_ERR_FOPEN, "Error opening file: \"%s\": %s", filename, strerror(errno)); + goto error; + } + fclose(fd); + } + + if (magic_load(t->ctx, filename) != 0) { + SCLogError(SC_ERR_MAGIC_LOAD, "magic_load failed: %s", magic_error(t->ctx)); + goto error; + } +SCLogInfo("returning %p", t); + return (void *)t; + +error: + if (t->ctx) + magic_close(t->ctx); + SCFree(t); + return NULL; +} + +static void DetectFilemagicThreadFree(void *ctx) { + if (ctx != NULL) { + DetectFilemagicThreadData *t = (DetectFilemagicThreadData *)ctx; + if (t->ctx) + magic_close(t->ctx); + SCFree(t); + } +} + /** * \brief this function is used to parse filemagic options * \brief into the current signature @@ -260,6 +370,12 @@ static int DetectFilemagicSetup (DetectEngineCtx *de_ctx, Signature *s, char *st if (filemagic == NULL) goto error; + filemagic->thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "filemagic", + DetectFilemagicThreadInit, (void *)filemagic, + DetectFilemagicThreadFree, 1); + if (filemagic->thread_ctx_id == -1) + goto error; + /* Okay so far so good, lets get this into a SigMatch * and put it in the Signature. */ sm = SigMatchAlloc(); diff --git a/src/detect-filemagic.h b/src/detect-filemagic.h index 572b245879..97cd79543f 100644 --- a/src/detect-filemagic.h +++ b/src/detect-filemagic.h @@ -25,8 +25,14 @@ #define __DETECT_FILEMAGIC_H__ #include "util-spm-bm.h" +#include + +typedef struct DetectFilemagicThreadData { + magic_t ctx; +} DetectFilemagicThreadData; typedef struct DetectFilemagicData { + int thread_ctx_id; uint8_t *name; /** name of the file to match */ BmCtx *bm_ctx; /** BM context */ uint16_t len; /** name length */ @@ -35,6 +41,6 @@ typedef struct DetectFilemagicData { /* prototypes */ void DetectFilemagicRegister (void); -int FilemagicLookup(File *file); +int FilemagicGlobalLookup(File *file); #endif /* __DETECT_FILEMAGIC_H__ */ diff --git a/src/detect-luajit.c b/src/detect-luajit.c index d4f448caff..bb70a5efb6 100644 --- a/src/detect-luajit.c +++ b/src/detect-luajit.c @@ -711,7 +711,7 @@ static int DetectLuajitSetup (DetectEngineCtx *de_ctx, Signature *s, char *str) luajit->thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "luajit", DetectLuajitThreadInit, (void *)luajit, - DetectLuajitThreadFree); + DetectLuajitThreadFree, 0); if (luajit->thread_ctx_id == -1) goto error; diff --git a/src/detect.h b/src/detect.h index 5c9619f323..4182d13fd1 100644 --- a/src/detect.h +++ b/src/detect.h @@ -1125,7 +1125,7 @@ int SignatureIsFilemagicInspecting(Signature *); int SignatureIsFileMd5Inspecting(Signature *); int SignatureIsFilesizeInspecting(Signature *); -int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *)); +int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int); void *DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *, int); #endif /* __DETECT_H__ */ diff --git a/src/log-file.c b/src/log-file.c index 5fa0e20c0e..fc7a67a79b 100644 --- a/src/log-file.c +++ b/src/log-file.c @@ -301,7 +301,7 @@ static TmEcode LogFileLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue continue; if (FileForceMagic() && ff->magic == NULL) { - FilemagicLookup(ff); + FilemagicGlobalLookup(ff); } SCLogDebug("ff %p", ff); diff --git a/src/log-filestore.c b/src/log-filestore.c index 60e8111c29..645063a0bd 100644 --- a/src/log-filestore.c +++ b/src/log-filestore.c @@ -296,7 +296,7 @@ static TmEcode LogFilestoreLogWrap(ThreadVars *tv, Packet *p, void *data, Packet int file_fd = -1; if (FileForceMagic() && ff->magic == NULL) { - FilemagicLookup(ff); + FilemagicGlobalLookup(ff); } SCLogDebug("ff %p", ff); diff --git a/src/util-magic.c b/src/util-magic.c index 1bb2f335da..fbc9b7e51f 100644 --- a/src/util-magic.c +++ b/src/util-magic.c @@ -93,7 +93,7 @@ error: * * \retval result pointer to null terminated string */ -char *MagicLookup(uint8_t *buf, uint32_t buflen) { +char *MagicGlobalLookup(uint8_t *buf, uint32_t buflen) { const char *result = NULL; char *magic = NULL; @@ -113,6 +113,31 @@ char *MagicLookup(uint8_t *buf, uint32_t buflen) { SCReturnPtr(magic, "const char"); } +/** + * \brief Find the magic value for a buffer. + * + * \param buf the buffer + * \param buflen length of the buffer + * + * \retval result pointer to null terminated string + */ +char *MagicThreadLookup(magic_t *ctx, uint8_t *buf, uint32_t buflen) { + const char *result = NULL; + char *magic = NULL; + + if (buf != NULL && buflen > 0) { + result = magic_buffer(*ctx, (void *)buf, (size_t)buflen); + if (result != NULL) { + magic = SCStrdup(result); + if (magic == NULL) { + SCLogError(SC_ERR_MEM_ALLOC, "Unable to dup magic"); + } + } + } + + SCReturnPtr(magic, "const char"); +} + void MagicDeinit(void) { SCMutexLock(&g_magic_lock); if (g_magic_ctx != NULL) { @@ -341,7 +366,7 @@ int MagicDetectTest05(void) { MagicInit(); - result = MagicLookup(buffer, buffer_len); + result = MagicGlobalLookup(buffer, buffer_len); if (result == NULL || strncmp(result, "PDF document", 12) != 0) { printf("result %p:%s, not \"PDF document\": ", result,result?result:"(null)"); goto end; @@ -377,7 +402,7 @@ int MagicDetectTest06(void) { MagicInit(); - result = MagicLookup(buffer, buffer_len); + result = MagicGlobalLookup(buffer, buffer_len); if (result == NULL || strcmp(result, "Microsoft Office Document") != 0) { printf("result %p:%s, not \"Microsoft Office Document\": ", result,result?result:"(null)"); goto end; @@ -422,7 +447,7 @@ int MagicDetectTest07(void) { MagicInit(); - result = MagicLookup(buffer, buffer_len); + result = MagicGlobalLookup(buffer, buffer_len); if (result == NULL || strcmp(result, "OpenDocument Text") != 0) { printf("result %p:%s, not \"OpenDocument Text\": ", result,result?result:"(null)"); goto end; @@ -472,7 +497,7 @@ int MagicDetectTest08(void) { MagicInit(); - result = MagicLookup(buffer, buffer_len); + result = MagicGlobalLookup(buffer, buffer_len); if (result == NULL || strcmp(result, "OpenOffice.org 1.x Database file") != 0) { printf("result %p:%s, not \"OpenOffice.org 1.x Database file\": ", result,result?result:"(null)"); goto end; @@ -494,13 +519,13 @@ int MagicDetectTest09(void) { MagicInit(); - result1 = MagicLookup(buffer, buffer_len); + result1 = MagicGlobalLookup(buffer, buffer_len); if (result1 == NULL || strncmp(result1, "PDF document", 12) != 0) { printf("result %p:%s, not \"PDF document\": ", result1,result1?result1:"(null)"); goto end; } - result2 = MagicLookup(buffer, buffer_len); + result2 = MagicGlobalLookup(buffer, buffer_len); if (result2 == NULL || strncmp(result2, "PDF document", 12) != 0) { printf("result %p:%s, not \"PDF document\": ", result2,result2?result2:"(null)"); goto end; @@ -519,7 +544,7 @@ end: /** \test results in valgrind warning about invalid read, tested with * file 5.09 and 5.11 */ -int MagicDetectTest10(void) { +static int MagicDetectTest10ValgrindError(void) { const char *result = NULL; uint8_t buffer[] = { 0xFF,0xD8,0xFF,0xE0,0x00,0x10,0x4A,0x46,0x49,0x46,0x00,0x01,0x01,0x01,0x01,0x2C, @@ -539,7 +564,7 @@ int MagicDetectTest10(void) { MagicInit(); - result = MagicLookup(buffer, buffer_len); + result = MagicGlobalLookup(buffer, buffer_len); if (result == NULL || strncmp(result, "JPEG", 4) != 0) { printf("result %p:%s, not \"JPEG\": ", result,result?result:"(null)"); goto end; @@ -569,6 +594,6 @@ void MagicRegisterTests(void) { /* fails in valgrind, somehow it returns different pointers then. UtRegisterTest("MagicDetectTest09", MagicDetectTest09, 1); */ - UtRegisterTest("MagicDetectTest10", MagicDetectTest10, 1); + UtRegisterTest("MagicDetectTest10ValgrindError", MagicDetectTest10ValgrindError, 1); #endif /* UNITTESTS */ } diff --git a/src/util-magic.h b/src/util-magic.h index db1deb84c2..0efdbd147b 100644 --- a/src/util-magic.h +++ b/src/util-magic.h @@ -24,9 +24,12 @@ #ifndef __UTIL_MAGIC_H__ #define __UTIL_MAGIC_H__ +#include + int MagicInit(void); void MagicDeinit(void); -char *MagicLookup(uint8_t *, uint32_t); +char *MagicGlobalLookup(uint8_t *, uint32_t); +char *MagicThreadLookup(magic_t *, uint8_t *, uint32_t); void MagicRegisterTests(void); #endif /* __UTIL_MAGIC_H__ */