Bug 585: use per detect thread libmagic ctx

src/detect-engine.c

@@ -1234,6 +1234,7 @@ void DetectEngineThreadCtxInfo(ThreadVars *t, DetectEngineThreadCtx *det_ctx) {
  *  \param InitFunc function ptr
  *  \param data keyword init data to pass to Func
  *  \param FreeFunc function ptr
+ *  \param mode 0 normal (ctx per keyword instance) 1 shared (one ctx per det_ct)
  *
  *  \retval id for retrieval of ctx at runtime
  *  \retval -1 on error
@@ -1242,9 +1243,20 @@ void DetectEngineThreadCtxInfo(ThreadVars *t, DetectEngineThreadCtx *det_ctx) {
  *        recommended to store it in the keywords global ctx so that
  *        it's freed when the de_ctx is freed.
  */
-int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *)) {
+int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int mode) {
     BUG_ON(de_ctx == NULL || InitFunc == NULL || FreeFunc == NULL || data == NULL);
 
+    if (mode) {
+        DetectEngineThreadKeywordCtxItem *item = de_ctx->keyword_list;
+        while (item != NULL) {
+            if (strcmp(name, item->name) == 0) {
+                return item->id;
+            }
+
+            item = item->next;
+        }
+    }
+
     DetectEngineThreadKeywordCtxItem *item = SCMalloc(sizeof(DetectEngineThreadKeywordCtxItem));
     if (unlikely(item == NULL))
         return -1;
@@ -1253,6 +1265,7 @@ int DetectRegisterThreadCtxFuncs(DetectEngineCtx *de_ctx, const char *name, void
     item->InitFunc = InitFunc;
     item->FreeFunc = FreeFunc;
     item->data = data;
+    item->name = name;
 
     item->next = de_ctx->keyword_list;
     de_ctx->keyword_list = item;

src/detect-filemagic.c

@@ -52,6 +52,9 @@
 
 #include "detect-filemagic.h"
 
+#include "conf.h"
+#include "util-magic.h"
+
 static int DetectFilemagicMatch (ThreadVars *, DetectEngineThreadCtx *, Flow *,
         uint8_t, File *, Signature *, SigMatch *);
 static int DetectFilemagicSetup (DetectEngineCtx *, Signature *, char *);
@@ -83,19 +86,68 @@ void DetectFilemagicRegister(void) {
  *  \retval -1 error
  *  \retval 0 ok
  */
-int FilemagicLookup(File *file) {
+int FilemagicGlobalLookup(File *file) {
     if (file == NULL || file->chunks_head == NULL) {
         SCReturnInt(-1);
     }
 
     /* initial chunk already matching our requirement */
     if (file->chunks_head->len >= FILEMAGIC_MIN_SIZE) {
-        file->magic = MagicLookup(file->chunks_head->data, FILEMAGIC_MIN_SIZE);
+        file->magic = MagicGlobalLookup(file->chunks_head->data, FILEMAGIC_MIN_SIZE);
+    } else {
+        uint8_t *buf = SCMalloc(FILEMAGIC_MIN_SIZE);
+        uint32_t size = 0;
+
+        if (likely(buf != NULL)) {
+            FileData *ffd = file->chunks_head;
+
+            for ( ; ffd != NULL; ffd = ffd->next) {
+                uint32_t copy_len = ffd->len;
+                if (size + ffd->len > FILEMAGIC_MIN_SIZE)
+                    copy_len = FILEMAGIC_MIN_SIZE - size;
+
+                memcpy(buf + size, ffd->data, copy_len);
+                size += copy_len;
+
+                if (size >= FILEMAGIC_MIN_SIZE) {
+                    file->magic = MagicGlobalLookup(buf, size);
+                    break;
+                }
+                /* file is done but smaller than FILEMAGIC_MIN_SIZE */
+                if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) {
+                    file->magic = MagicGlobalLookup(buf, size);
+                    break;
+                }
+            }
+
+            SCFree(buf);
+        }
+    }
+
+    SCReturnInt(0);
+}
+
+/**
+ *  \brief run the magic check
+ *
+ *  \param file the file
+ *
+ *  \retval -1 error
+ *  \retval 0 ok
+ */
+int FilemagicThreadLookup(magic_t *ctx, File *file) {
+    if (ctx == NULL || file == NULL || file->chunks_head == NULL) {
+        SCReturnInt(-1);
+    }
+
+    /* initial chunk already matching our requirement */
+    if (file->chunks_head->len >= FILEMAGIC_MIN_SIZE) {
+        file->magic = MagicThreadLookup(ctx, file->chunks_head->data, FILEMAGIC_MIN_SIZE);
     } else {
         uint8_t *buf = SCMalloc(FILEMAGIC_MIN_SIZE);
         uint32_t size = 0;
 
-        if (buf != NULL) {
+        if (likely(buf != NULL)) {
             FileData *ffd = file->chunks_head;
 
             for ( ; ffd != NULL; ffd = ffd->next) {
@@ -107,12 +159,12 @@ int FilemagicLookup(File *file) {
                 size += copy_len;
 
                 if (size >= FILEMAGIC_MIN_SIZE) {
-                    file->magic = MagicLookup(buf, size);
+                    file->magic = MagicThreadLookup(ctx, buf, size);
                     break;
                 }
                 /* file is done but smaller than FILEMAGIC_MIN_SIZE */
                 if (ffd->next == NULL && file->state >= FILE_STATE_CLOSED) {
-                    file->magic = MagicLookup(buf, size);
+                    file->magic = MagicThreadLookup(ctx, buf, size);
                     break;
                 }
             }
@@ -151,8 +203,13 @@ static int DetectFilemagicMatch (ThreadVars *t, DetectEngineThreadCtx *det_ctx,
     if (file->txid > det_ctx->tx_id)
         SCReturnInt(0);
 
+    DetectFilemagicThreadData *tfilemagic = (DetectFilemagicThreadData *)DetectThreadCtxGetKeywordThreadCtx(det_ctx, filemagic->thread_ctx_id);
+    if (tfilemagic == NULL) {
+        SCReturnInt(0);
+    }
+
     if (file->magic == NULL) {
-        FilemagicLookup(file);
+        FilemagicThreadLookup(&tfilemagic->ctx, file);
     }
 
     if (file->magic != NULL) {
@@ -240,6 +297,59 @@ error:
     return NULL;
 }
 
+static void *DetectFilemagicThreadInit(void *data) {
+    char *filename = NULL;
+    FILE *fd = NULL;
+    DetectFilemagicData *filemagic = (DetectFilemagicData *)data;
+    BUG_ON(filemagic == NULL);
+
+    DetectFilemagicThreadData *t = SCMalloc(sizeof(DetectFilemagicThreadData));
+    if (unlikely(t == NULL)) {
+        SCLogError(SC_ERR_LUAJIT_ERROR, "couldn't alloc ctx memory");
+        return NULL;
+    }
+    memset(t, 0x00, sizeof(DetectFilemagicThreadData));
+
+    t->ctx = magic_open(0);
+    if (t->ctx == NULL) {
+        SCLogError(SC_ERR_MAGIC_OPEN, "magic_open failed: %s", magic_error(t->ctx));
+        goto error;
+    }
+
+    (void)ConfGet("magic-file", &filename);
+    if (filename != NULL) {
+        SCLogInfo("using magic-file %s", filename);
+
+        if ( (fd = fopen(filename, "r")) == NULL) {
+            SCLogWarning(SC_ERR_FOPEN, "Error opening file: \"%s\": %s", filename, strerror(errno));
+            goto error;
+        }
+        fclose(fd);
+    }
+
+    if (magic_load(t->ctx, filename) != 0) {
+        SCLogError(SC_ERR_MAGIC_LOAD, "magic_load failed: %s", magic_error(t->ctx));
+        goto error;
+    }
+SCLogInfo("returning %p", t);
+    return (void *)t;
+
+error:
+    if (t->ctx)
+        magic_close(t->ctx);
+    SCFree(t);
+    return NULL;
+}
+
+static void DetectFilemagicThreadFree(void *ctx) {
+    if (ctx != NULL) {
+        DetectFilemagicThreadData *t = (DetectFilemagicThreadData *)ctx;
+        if (t->ctx)
+            magic_close(t->ctx);
+        SCFree(t);
+    }
+}
+
 /**
  * \brief this function is used to parse filemagic options
  * \brief into the current signature
@@ -260,6 +370,12 @@ static int DetectFilemagicSetup (DetectEngineCtx *de_ctx, Signature *s, char *st
     if (filemagic == NULL)
         goto error;
 
+    filemagic->thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "filemagic",
+            DetectFilemagicThreadInit, (void *)filemagic,
+            DetectFilemagicThreadFree, 1);
+    if (filemagic->thread_ctx_id == -1)
+        goto error;
+
     /* Okay so far so good, lets get this into a SigMatch
      * and put it in the Signature. */
     sm = SigMatchAlloc();

src/detect-filemagic.h

@@ -25,8 +25,14 @@
 #define __DETECT_FILEMAGIC_H__
 
 #include "util-spm-bm.h"
+#include <magic.h>
+
+typedef struct DetectFilemagicThreadData {
+    magic_t ctx;
+} DetectFilemagicThreadData;
 
 typedef struct DetectFilemagicData {
+    int thread_ctx_id;
     uint8_t *name; /** name of the file to match */
     BmCtx *bm_ctx; /** BM context */
     uint16_t len; /** name length */
@@ -35,6 +41,6 @@ typedef struct DetectFilemagicData {
 
 /* prototypes */
 void DetectFilemagicRegister (void);
-int FilemagicLookup(File *file);
+int FilemagicGlobalLookup(File *file);
 
 #endif /* __DETECT_FILEMAGIC_H__ */

src/detect-luajit.c

@@ -711,7 +711,7 @@ static int DetectLuajitSetup (DetectEngineCtx *de_ctx, Signature *s, char *str)
 
     luajit->thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "luajit",
             DetectLuajitThreadInit, (void *)luajit,
-            DetectLuajitThreadFree);
+            DetectLuajitThreadFree, 0);
     if (luajit->thread_ctx_id == -1)
         goto error;
 

src/detect.h

@@ -1125,7 +1125,7 @@ int SignatureIsFilemagicInspecting(Signature *);
 int SignatureIsFileMd5Inspecting(Signature *);
 int SignatureIsFilesizeInspecting(Signature *);
 
-int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *));
+int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int);
 void *DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *, int);
 
 #endif /* __DETECT_H__ */

src/log-file.c

@@ -301,7 +301,7 @@ static TmEcode LogFileLogWrap(ThreadVars *tv, Packet *p, void *data, PacketQueue
                 continue;
 
             if (FileForceMagic() && ff->magic == NULL) {
-                FilemagicLookup(ff);
+                FilemagicGlobalLookup(ff);
             }
 
             SCLogDebug("ff %p", ff);

src/log-filestore.c

@@ -296,7 +296,7 @@ static TmEcode LogFilestoreLogWrap(ThreadVars *tv, Packet *p, void *data, Packet
             int file_fd = -1;
 
             if (FileForceMagic() && ff->magic == NULL) {
-                FilemagicLookup(ff);
+                FilemagicGlobalLookup(ff);
             }
 
             SCLogDebug("ff %p", ff);

src/util-magic.c

@@ -93,7 +93,7 @@ error:
  *
  *  \retval result pointer to null terminated string
  */
-char *MagicLookup(uint8_t *buf, uint32_t buflen) {
+char *MagicGlobalLookup(uint8_t *buf, uint32_t buflen) {
     const char *result = NULL;
     char *magic = NULL;
 
@@ -113,6 +113,31 @@ char *MagicLookup(uint8_t *buf, uint32_t buflen) {
     SCReturnPtr(magic, "const char");
 }
 
+/**
+ *  \brief Find the magic value for a buffer.
+ *
+ *  \param buf the buffer
+ *  \param buflen length of the buffer
+ *
+ *  \retval result pointer to null terminated string
+ */
+char *MagicThreadLookup(magic_t *ctx, uint8_t *buf, uint32_t buflen) {
+    const char *result = NULL;
+    char *magic = NULL;
+
+    if (buf != NULL && buflen > 0) {
+        result = magic_buffer(*ctx, (void *)buf, (size_t)buflen);
+        if (result != NULL) {
+            magic = SCStrdup(result);
+            if (magic == NULL) {
+                SCLogError(SC_ERR_MEM_ALLOC, "Unable to dup magic");
+            }
+        }
+    }
+
+    SCReturnPtr(magic, "const char");
+}
+
 void MagicDeinit(void) {
     SCMutexLock(&g_magic_lock);
     if (g_magic_ctx != NULL) {
@@ -341,7 +366,7 @@ int MagicDetectTest05(void) {
 
     MagicInit();
 
-    result = MagicLookup(buffer, buffer_len);
+    result = MagicGlobalLookup(buffer, buffer_len);
     if (result == NULL || strncmp(result, "PDF document", 12) != 0) {
         printf("result %p:%s, not \"PDF document\": ", result,result?result:"(null)");
         goto end;
@@ -377,7 +402,7 @@ int MagicDetectTest06(void) {
 
     MagicInit();
 
-    result = MagicLookup(buffer, buffer_len);
+    result = MagicGlobalLookup(buffer, buffer_len);
     if (result == NULL || strcmp(result, "Microsoft Office Document") != 0) {
         printf("result %p:%s, not \"Microsoft Office Document\": ", result,result?result:"(null)");
         goto end;
@@ -422,7 +447,7 @@ int MagicDetectTest07(void) {
 
     MagicInit();
 
-    result = MagicLookup(buffer, buffer_len);
+    result = MagicGlobalLookup(buffer, buffer_len);
     if (result == NULL || strcmp(result, "OpenDocument Text") != 0) {
         printf("result %p:%s, not \"OpenDocument Text\": ", result,result?result:"(null)");
         goto end;
@@ -472,7 +497,7 @@ int MagicDetectTest08(void) {
 
     MagicInit();
 
-    result = MagicLookup(buffer, buffer_len);
+    result = MagicGlobalLookup(buffer, buffer_len);
     if (result == NULL || strcmp(result, "OpenOffice.org 1.x Database file") != 0) {
         printf("result %p:%s, not \"OpenOffice.org 1.x Database file\": ", result,result?result:"(null)");
         goto end;
@@ -494,13 +519,13 @@ int MagicDetectTest09(void) {
 
     MagicInit();
 
-    result1 = MagicLookup(buffer, buffer_len);
+    result1 = MagicGlobalLookup(buffer, buffer_len);
     if (result1 == NULL || strncmp(result1, "PDF document", 12) != 0) {
         printf("result %p:%s, not \"PDF document\": ", result1,result1?result1:"(null)");
         goto end;
     }
 
-    result2 = MagicLookup(buffer, buffer_len);
+    result2 = MagicGlobalLookup(buffer, buffer_len);
     if (result2 == NULL || strncmp(result2, "PDF document", 12) != 0) {
         printf("result %p:%s, not \"PDF document\": ", result2,result2?result2:"(null)");
         goto end;
@@ -519,7 +544,7 @@ end:
 
 /** \test results in valgrind warning about invalid read, tested with
  *        file 5.09 and 5.11 */
-int MagicDetectTest10(void) {
+static int MagicDetectTest10ValgrindError(void) {
     const char *result = NULL;
     uint8_t buffer[] = {
         0xFF,0xD8,0xFF,0xE0,0x00,0x10,0x4A,0x46,0x49,0x46,0x00,0x01,0x01,0x01,0x01,0x2C,
@@ -539,7 +564,7 @@ int MagicDetectTest10(void) {
 
     MagicInit();
 
-    result = MagicLookup(buffer, buffer_len);
+    result = MagicGlobalLookup(buffer, buffer_len);
     if (result == NULL || strncmp(result, "JPEG", 4) != 0) {
         printf("result %p:%s, not \"JPEG\": ", result,result?result:"(null)");
         goto end;
@@ -569,6 +594,6 @@ void MagicRegisterTests(void) {
     /* fails in valgrind, somehow it returns different pointers then.
     UtRegisterTest("MagicDetectTest09", MagicDetectTest09, 1); */
 
-    UtRegisterTest("MagicDetectTest10", MagicDetectTest10, 1);
+    UtRegisterTest("MagicDetectTest10ValgrindError", MagicDetectTest10ValgrindError, 1);
 #endif /* UNITTESTS */
 }

src/util-magic.h

@@ -24,9 +24,12 @@
 #ifndef __UTIL_MAGIC_H__
 #define __UTIL_MAGIC_H__
 
+#include <magic.h>
+
 int MagicInit(void);
 void MagicDeinit(void);
-char *MagicLookup(uint8_t *, uint32_t);
+char *MagicGlobalLookup(uint8_t *, uint32_t);
+char *MagicThreadLookup(magic_t *, uint8_t *, uint32_t);
 void MagicRegisterTests(void);
 
 #endif /* __UTIL_MAGIC_H__ */