detect: mpm deduplication

Create hash for mpm's that we can reuse. Have packet/stream mpms
use this.

src/detect-engine-mpm.h

@@ -64,7 +64,7 @@ void PatternMatchDestroy(MpmCtx *, uint16_t);
 void PatternMatchThreadDestroy(MpmThreadCtx *mpm_thread_ctx, uint16_t);
 void PatternMatchThreadPrint(MpmThreadCtx *, uint16_t);
 
-int PatternMatchPrepareGroup(const DetectEngineCtx *, SigGroupHead *);
+int PatternMatchPrepareGroup(DetectEngineCtx *, SigGroupHead *);
 void DetectEngineThreadCtxInfo(ThreadVars *, DetectEngineThreadCtx *);
 void PatternMatchDestroyGroup(SigGroupHead *);
 
@@ -83,6 +83,11 @@ int SignatureHasStreamContent(const Signature *);
 
 SigMatch *RetrieveFPForSig(Signature *s);
 
+int MpmStoreInit(DetectEngineCtx *);
+void MpmStoreFree(DetectEngineCtx *);
+void MpmStoreReportStats(const DetectEngineCtx *de_ctx);
+MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh, enum MpmBuiltinBuffers buf);
+
 /**
  * \brief Figured out the FP and their respective content ids for all the
  *        sigs in the engine.

src/detect-engine-siggroup.c

@@ -203,133 +203,6 @@ void SigGroupHeadFree(SigGroupHead *sgh)
     return;
 }
 
-/**
- * \brief The hash function to be the used by the mpm SigGroupHead hash table -
- *        DetectEngineCtx->sgh_mpm_hash_table.
- *
- * \param ht      Pointer to the hash table.
- * \param data    Pointer to the SigGroupHead.
- * \param datalen Not used in our case.
- *
- * \retval hash The generated hash value.
- */
-uint32_t SigGroupHeadMpmHashFunc(HashListTable *ht, void *data, uint16_t datalen)
-{
-    SigGroupHead *sgh = (SigGroupHead *)data;
-    uint32_t hash = 0;
-    uint32_t b = 0;
-
-    for (b = 0; b < sgh->init->content_size; b++)
-        hash += sgh->init->content_array[b];
-
-    return hash % ht->array_size;
-}
-
-/**
- * \brief The Compare function to be used by the mpm SigGroupHead hash table -
- *        DetectEngineCtx->sgh_mpm_hash_table.
- *
- * \param data1 Pointer to the first SigGroupHead.
- * \param len1  Not used.
- * \param data2 Pointer to the second SigGroupHead.
- * \param len2  Not used.
- *
- * \retval 1 If the 2 SigGroupHeads sent as args match.
- * \retval 0 If the 2 SigGroupHeads sent as args do not match.
- */
-char SigGroupHeadMpmCompareFunc(void *data1, uint16_t len1, void *data2,
-                                uint16_t len2)
-{
-    SigGroupHead *sgh1 = (SigGroupHead *)data1;
-    SigGroupHead *sgh2 = (SigGroupHead *)data2;
-
-    if (sgh1->init->content_size != sgh2->init->content_size)
-        return 0;
-
-    if (SCMemcmp(sgh1->init->content_array, sgh2->init->content_array,
-               sgh1->init->content_size) != 0) {
-        return 0;
-    }
-
-    return 1;
-}
-
-/**
- * \brief Initializes the SigGroupHead mpm hash table to be used by the detection
- *        engine context.
- *
- * \param de_ctx Pointer to the detection engine context.
- *
- * \retval  0 On success.
- * \retval -1 On failure.
- */
-int SigGroupHeadMpmHashInit(DetectEngineCtx *de_ctx)
-{
-    de_ctx->sgh_mpm_hash_table = HashListTableInit(4096, SigGroupHeadMpmHashFunc,
-                                                   SigGroupHeadMpmCompareFunc,
-                                                   NULL);
-
-    if (de_ctx->sgh_mpm_hash_table == NULL)
-        goto error;
-
-    return 0;
-
-error:
-    return -1;
-}
-
-/**
- * \brief Adds a SigGroupHead to the detection engine context SigGroupHead
- *        mpm hash table.
- *
- * \param de_ctx Pointer to the detection engine context.
- * \param sgh    Pointer to the SigGroupHead.
- *
- * \retval ret 0 on Successfully adding the argument sgh; -1 on failure.
- */
-int SigGroupHeadMpmHashAdd(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
-{
-    int ret = HashListTableAdd(de_ctx->sgh_mpm_hash_table, (void *)sgh, 0);
-
-    return ret;
-}
-
-/**
- * \brief Used to lookup a SigGroupHead from the detection engine context
- *        SigGroupHead mpm hash table.
- *
- * \param de_ctx Pointer to the detection engine context.
- * \param sgh    Pointer to the SigGroupHead.
- *
- * \retval rsgh On success a pointer to the SigGroupHead if the SigGroupHead is
- *              found in the hash table; NULL on failure.
- */
-SigGroupHead *SigGroupHeadMpmHashLookup(DetectEngineCtx *de_ctx,
-                                        SigGroupHead *sgh)
-{
-    SigGroupHead *rsgh = HashListTableLookup(de_ctx->sgh_mpm_hash_table,
-                                             (void *)sgh, 0);
-
-    return rsgh;
-}
-
-/**
- * \brief Frees the hash table - DetectEngineCtx->sgh_mpm_hash_table, allocated by
- *        SigGroupHeadMpmHashInit() function.
- *
- * \param de_ctx Pointer to the detection engine context.
- */
-void SigGroupHeadMpmHashFree(DetectEngineCtx *de_ctx)
-{
-    if (de_ctx->sgh_mpm_hash_table == NULL)
-        return;
-
-    HashListTableFree(de_ctx->sgh_mpm_hash_table);
-    de_ctx->sgh_mpm_hash_table = NULL;
-
-    return;
-}
-
 /**
  * \brief The hash function to be the used by the hash table -
  *        DetectEngineCtx->sgh_hash_table.
@@ -1192,28 +1065,6 @@ int SigGroupHeadContainsSigId(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
 
 int SigAddressPrepareStage1(DetectEngineCtx *);
 
-/**
- * \test Check if a SigGroupHead mpm hash table is properly allocated and
- *       deallocated when calling SigGroupHeadMpmHashInit() and
- *       SigGroupHeadMpmHashFree() respectively.
- */
-static int SigGroupHeadTest01(void)
-{
-    int result = 1;
-
-    DetectEngineCtx de_ctx;
-
-    SigGroupHeadMpmHashInit(&de_ctx);
-
-    result &= (de_ctx.sgh_mpm_hash_table != NULL);
-
-    SigGroupHeadMpmHashFree(&de_ctx);
-
-    result &= (de_ctx.sgh_mpm_hash_table == NULL);
-
-    return result;
-}
-
 /**
  * \test Check if a SigGroupHead hash table is properly allocated and
  *       deallocated when calling SigGroupHeadHashInit() and
@@ -1720,7 +1571,6 @@ end:
 void SigGroupHeadRegisterTests(void)
 {
 #ifdef UNITTESTS
-    UtRegisterTest("SigGroupHeadTest01", SigGroupHeadTest01, 1);
     UtRegisterTest("SigGroupHeadTest03", SigGroupHeadTest03, 1);
     UtRegisterTest("SigGroupHeadTest04", SigGroupHeadTest04, 1);
     UtRegisterTest("SigGroupHeadTest06", SigGroupHeadTest06, 1);

src/detect-engine.c

@@ -871,7 +871,7 @@ static DetectEngineCtx *DetectEngineCtxInitReal(int minimal, const char *prefix)
     DetectEngineCtxLoadConf(de_ctx);
 
     SigGroupHeadHashInit(de_ctx);
-    SigGroupHeadMpmHashInit(de_ctx);
+    MpmStoreInit(de_ctx);
     SigGroupHeadDPortHashInit(de_ctx);
     ThresholdHashInit(de_ctx);
     VariableNameInitHash(de_ctx);
@@ -956,7 +956,7 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
      * to be sure look at them again here.
      */
     SigGroupHeadHashFree(de_ctx);
-    SigGroupHeadMpmHashFree(de_ctx);
+    MpmStoreFree(de_ctx);
     SigGroupHeadDPortHashFree(de_ctx);
     DetectParseDupSigHashFree(de_ctx);
     SCSigSignatureOrderingModuleCleanup(de_ctx);

src/detect.c

@@ -3534,6 +3534,7 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
         SigGroupHead *sgh = de_ctx->sgh_array[idx];
         if (sgh == NULL)
             continue;
+
         SigGroupHeadSetFilemagicFlag(de_ctx, sgh);
         SigGroupHeadSetFileMd5Flag(de_ctx, sgh);
         SigGroupHeadSetFilesizeFlag(de_ctx, sgh);
@@ -3549,6 +3550,8 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
     }
     SCLogInfo("Unique rule groups: %u", cnt);
 
+    MpmStoreReportStats(de_ctx);
+
     if (de_ctx->decoder_event_sgh != NULL) {
         /* no need to set filestore count here as that would make a
          * signature not decode event only. */
@@ -3563,7 +3566,7 @@ int SigAddressPrepareStage4(DetectEngineCtx *de_ctx)
      * after the initialization phase. */
     SigGroupHeadHashFree(de_ctx);
     SigGroupHeadDPortHashFree(de_ctx);
-    SigGroupHeadMpmHashFree(de_ctx);
+    MpmStoreFree(de_ctx);
 
     SCFree(de_ctx->sgh_array);
     de_ctx->sgh_array_cnt = 0;

src/detect.h

@@ -606,7 +606,7 @@ typedef struct DetectEngineCtx_ {
     /* init phase vars */
     HashListTable *sgh_hash_table;
 
-    HashListTable *sgh_mpm_hash_table;
+    HashListTable *mpm_hash_table;
 
     HashListTable *sgh_dport_hash_table;
 
@@ -959,10 +959,30 @@ typedef struct SigTableElmt_ {
 #define SIG_GROUP_HEAD_MPM_DNSQUERY     (1 << 23)
 #define SIG_GROUP_HEAD_MPM_FD_SMTP      (1 << 24)
 
+enum MpmBuiltinBuffers {
+    MPMB_TCP_PKT_TS,
+    MPMB_TCP_PKT_TC,
+    MPMB_TCP_STREAM_TS,
+    MPMB_TCP_STREAM_TC,
+    MPMB_UDP_TS,
+    MPMB_UDP_TC,
+    MPMB_OTHERIP,
+    MPMB_MAX,
+};
+
+typedef struct MpmStore_ {
+    uint8_t *sid_array;
+    uint32_t sid_array_size;
+
+    int direction;
+    enum MpmBuiltinBuffers buffer;
+
+    MpmCtx *mpm_ctx;
+
+} MpmStore;
+
 typedef struct SigGroupHeadInitData_ {
-    /* list of content containers */
-    uint8_t *content_array;
-    uint32_t content_size;
+    MpmStore mpm_store[MPMB_MAX];
 
     uint8_t *sig_array; /**< bit array of sig nums (internal id's) */
     uint32_t sig_size; /**< size in bytes */