suricata/src/detect-engine-mpm.c

/* Copyright (C) 2007-2026 Open Information Security Foundation
 *
 * You can copy, redistribute or modify this Program under the terms of
 * the GNU General Public License version 2 as published by the Free
 * Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 */

/**
 * \file
 *
 * \author Victor Julien <victor@inliniac.net>
 * \author Anoop Saldanha <anoopsaldanha@gmail.com>
 *
 * Multi pattern matcher
 */

#include "suricata.h"
#include "suricata-common.h"

#include "app-layer-protos.h"
#include "app-layer-parser.h"

#include "decode.h"
#include "detect.h"
#include "detect-engine.h"
#include "detect-engine-siggroup.h"
#include "detect-engine-mpm.h"
#include "detect-engine-iponly.h"
#include "detect-parse.h"
#include "detect-engine-prefilter.h"
#include "util-mpm.h"
#include "util-memcmp.h"
#include "util-memcpy.h"
#include "conf.h"
#include "detect-fast-pattern.h"

#include "detect-tcphdr.h"
#include "detect-udphdr.h"

#include "flow.h"
#include "flow-var.h"
#include "detect-flow.h"

#include "detect-content.h"

#include "detect-engine-payload.h"

#include "stream.h"

#include "util-misc.h"
#include "util-enum.h"
#include "util-debug.h"
#include "util-print.h"
#include "util-validate.h"
#include "util-hash-string.h"

const char *builtin_mpms[] = {
    "toserver TCP packet",
    "toclient TCP packet",
    "toserver TCP stream",
    "toclient TCP stream",
    "toserver UDP packet",
    "toclient UDP packet",
    "other IP packet",

    NULL };

/* Registry for mpm keywords
 *
 * Keywords are registered at engine start up
 */

static DetectBufferMpmRegistry *g_mpm_list[DETECT_BUFFER_MPM_TYPE_SIZE] = { NULL, NULL, NULL };
static int g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_SIZE] = { 0, 0, 0 };

/** \brief register a MPM engine
 *
 *  \note to be used at start up / registration only. Errors are fatal.
 */
static void RegisterInternal(const char *name, int direction, int priority,
        PrefilterRegisterFunc PrefilterRegister, InspectionBufferGetDataPtr GetData,
        InspectionSingleBufferGetDataPtr GetDataSingle,
        InspectionMultiBufferGetDataPtr GetMultiData, AppProto alproto, uint8_t sub_state,
        uint8_t tx_min_progress)
{
    SCLogDebug("registering %s/%d/%d/%p/%p/%u/%d", name, direction, priority,
            PrefilterRegister, GetData, alproto, tx_min_progress);

    if (!AppLayerParserIsEnabled(alproto)) {
        SCLogDebug("%s is disabled", AppProtoToString(alproto));
        return;
    }
    SCLogDebug("%s is enabled", AppProtoToString(alproto));
    DEBUG_VALIDATE_BUG_ON(AppLayerParserSupportsSubStates(alproto) && sub_state == 0);
    DEBUG_VALIDATE_BUG_ON(!AppLayerParserSupportsSubStates(alproto) && sub_state != 0);

    BUG_ON(tx_min_progress >= APP_LAYER_MAX_PROGRESS);

    // must register GetData with PrefilterGenericMpmRegister
    BUG_ON(PrefilterRegister == PrefilterGenericMpmRegister && GetData == NULL);

    DetectBufferTypeSupportsMpm(name);
    DetectBufferTypeSupportsTransformations(name);
    int sm_list = DetectBufferTypeGetByName(name);
    if (sm_list == -1) {
        FatalError("MPM engine registration for %s failed", name);
    }

    DetectBufferMpmRegistry *am = SCCalloc(1, sizeof(*am));
    BUG_ON(am == NULL);
    am->name = name;
    snprintf(am->pname, sizeof(am->pname), "%s", am->name);
    am->direction = direction;
    DEBUG_VALIDATE_BUG_ON(sm_list < 0 || sm_list > INT16_MAX);
    am->sm_list = (int16_t)sm_list;
    am->sm_list_base = (int16_t)sm_list;
    am->priority = priority;
    am->type = DETECT_BUFFER_MPM_TYPE_APP;

    am->PrefilterRegisterWithListId = PrefilterRegister;
    if (GetData != NULL) {
        am->app_v2.GetData = GetData;
    } else if (GetDataSingle != NULL) {
        am->app_v2.GetDataSingle = GetDataSingle;
    } else if (GetMultiData != NULL) {
        am->app_v2.GetMultiData = GetMultiData;
    }
    am->app_v2.alproto = alproto;
    am->app_v2.tx_min_progress = tx_min_progress;
    am->app_v2.sub_state = sub_state;

    if (g_mpm_list[DETECT_BUFFER_MPM_TYPE_APP] == NULL) {
        g_mpm_list[DETECT_BUFFER_MPM_TYPE_APP] = am;
    } else {
        DetectBufferMpmRegistry *t = g_mpm_list[DETECT_BUFFER_MPM_TYPE_APP];
        while (t->next != NULL) {
            t = t->next;
        }

        t->next = am;
        am->id = t->id + 1;
    }
    g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_APP]++;

    SupportFastPatternForSigMatchList(sm_list, priority);
    SCLogDebug("%s: sub_state %u", name, am->app_v2.sub_state);
}

void DetectAppLayerMpmRegister(const char *name, int direction, int priority,
        PrefilterRegisterFunc PrefilterRegister, InspectionBufferGetDataPtr GetData,
        AppProto alproto, uint8_t tx_min_progress)
{
    RegisterInternal(name, direction, priority, PrefilterRegister, GetData, NULL, NULL, alproto, 0,
            tx_min_progress);
}

void DetectAppLayerMpmRegisterSubState(const char *name, int direction, int priority,
        PrefilterRegisterFunc PrefilterRegister, InspectionBufferGetDataPtr GetData,
        AppProto alproto, uint8_t sub_state, uint8_t tx_min_progress)
{
    SCLogDebug("%s: sub_state %u", name, sub_state);
    RegisterInternal(name, direction, priority, PrefilterRegister, GetData, NULL, NULL, alproto,
            sub_state, tx_min_progress);
}

void DetectAppLayerMpmRegisterSingle(const char *name, int direction, int priority,
        PrefilterRegisterFunc PrefilterRegister, InspectionSingleBufferGetDataPtr GetData,
        AppProto alproto, uint8_t tx_min_progress)
{
    RegisterInternal(name, direction, priority, PrefilterRegister, NULL, GetData, NULL, alproto, 0,
            tx_min_progress);
}

void DetectAppLayerMpmMultiRegister(const char *name, int direction, int priority,
        PrefilterRegisterFunc PrefilterRegister, InspectionMultiBufferGetDataPtr GetData,
        AppProto alproto, uint8_t tx_min_progress)
{
    RegisterInternal(name, direction, priority, PrefilterRegister, NULL, NULL, GetData, alproto, 0,
            tx_min_progress);
}

void DetectAppLayerMpmMultiRegisterSubState(const char *name, int direction, int priority,
        PrefilterRegisterFunc PrefilterRegister, InspectionMultiBufferGetDataPtr GetData,
        AppProto alproto, uint8_t sub_state, uint8_t tx_min_progress)
{
    RegisterInternal(name, direction, priority, PrefilterRegister, NULL, NULL, GetData, alproto,
            sub_state, tx_min_progress);
}

/** \internal
 *  \brief build basic profiling name (pname) making sure the id is always fully printed
 */
static void BuildBasicPname(char *out, const size_t out_size, const char *name, const uint16_t id)
{
    size_t id_space;
    if (id < 10)
        id_space = 1;
    else if (id < 100)
        id_space = 2;
    else if (id < 1000)
        id_space = 3;
    else if (id < 10000)
        id_space = 4;
    else
        id_space = 5;
    size_t name_space = out_size - (id_space + 1);
    char pname[name_space];
    if (strlen(name) >= name_space) {
        ShortenString(name, pname, name_space, '~');
    } else {
        strlcpy(pname, name, sizeof(pname));
    }
    snprintf(out, out_size, "%s#%u", pname, id);
}

/** \internal
 *  \brief add transforms to the profiling name, as space permits
 *  \param out contains basic profiling name (pname), to be appended to
 *  \note out will be untouched if there isn't enough space left of if there are no transforms
 */
static void AppendTransformsToPname(
        char *out, const size_t out_size, const DetectEngineTransforms *transforms)
{
    if (transforms == NULL || transforms->cnt == 0)
        return;

    ssize_t left = (ssize_t)out_size - (ssize_t)strlen(out) - (ssize_t)4;
    /* only append xform if we can add least 5 chars */
    if (left >= 5) {
        /* create comma separated string of the names of the
         * transforms and then shorten it if necessary. Finally
         * use it to construct the 'profile' name for the engine */
        char xforms[DETECT_PROFILE_NAME_LEN + 1];
        memset(xforms, 0, DETECT_PROFILE_NAME_LEN + 1);
        for (uint8_t i = 0; i < transforms->cnt; i++) {
            char ttstr[64];
            (void)snprintf(ttstr, sizeof(ttstr), "%s,",
                    sigmatch_table[transforms->transforms[i].transform].name);
            strlcat(xforms, ttstr, sizeof(xforms));
        }
        if (strlen(xforms) == 0)
            return;
        xforms[strlen(xforms) - 1] = '\0';
        SCLogDebug("left %d '%s' %d", (int)left, xforms, (int)strlen(xforms));

        char xforms_print[out_size];
        if ((size_t)left >= strlen(xforms)) {
            snprintf(xforms_print, sizeof(xforms_print), " (%s)", xforms);
        } else {
            char xforms_short[out_size];
            ShortenString(xforms, xforms_short, left, '~');
            snprintf(xforms_print, sizeof(xforms_print), " (%s)", xforms_short);
        }
        strlcat(out, xforms_print, out_size);
    }
}

/** \brief copy a mpm engine from parent_id, add in transforms */
void DetectAppLayerMpmRegisterByParentId(DetectEngineCtx *de_ctx,
        const int id, const int parent_id,
        DetectEngineTransforms *transforms)
{
    SCLogDebug("registering %d/%d", id, parent_id);

    DetectBufferMpmRegistry *t = de_ctx->app_mpms_list;
    while (t) {
        if (t->sm_list == parent_id) {
            DetectBufferMpmRegistry *am = SCCalloc(1, sizeof(*am));
            BUG_ON(am == NULL);
            am->name = t->name;
            am->direction = t->direction;
            DEBUG_VALIDATE_BUG_ON(id < 0 || id > INT16_MAX);
            am->sm_list = (uint16_t)id; // use new id
            am->sm_list_base = t->sm_list;
            am->type = DETECT_BUFFER_MPM_TYPE_APP;
            am->PrefilterRegisterWithListId = t->PrefilterRegisterWithListId;
            am->app_v2.GetData = t->app_v2.GetData;
            am->app_v2.alproto = t->app_v2.alproto;
            am->app_v2.tx_min_progress = t->app_v2.tx_min_progress;
            am->app_v2.sub_state = t->app_v2.sub_state;
            am->priority = t->priority;
            am->sgh_mpm_context = t->sgh_mpm_context;
            am->sgh_mpm_context = MpmFactoryRegisterMpmCtxProfile(
                    de_ctx, am->name, am->sm_list, am->app_v2.alproto);
            am->next = t->next;

            BuildBasicPname(am->pname, sizeof(am->pname), am->name, (uint16_t)id);
            if (transforms) {
                memcpy(&am->transforms, transforms, sizeof(*transforms));
                AppendTransformsToPname(am->pname, sizeof(am->pname), transforms);
            }
            SCLogDebug("am->pname '%s' (%u)", am->pname, (uint32_t)strlen(am->pname));
            am->id = de_ctx->app_mpms_list_cnt++;

            DetectEngineRegisterFastPatternForId(de_ctx, am->sm_list, am->priority);
            t->next = am;
            SCLogDebug("copied mpm registration for %s id %u "
                    "with parent %u and GetData %p",
                    t->name, id, parent_id, am->app_v2.GetData);
            t = am;
        }
        t = t->next;
    }
}

void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx)
{
    const DetectBufferMpmRegistry *list = g_mpm_list[DETECT_BUFFER_MPM_TYPE_APP];
    DetectBufferMpmRegistry *toadd = de_ctx->app_mpms_list;
    while (list != NULL) {
        DetectBufferMpmRegistry *n = SCCalloc(1, sizeof(*n));
        BUG_ON(n == NULL);

        *n = *list;
        n->next = NULL;

        if (toadd == NULL) {
            toadd = n;
            de_ctx->app_mpms_list = n;
        } else {
            toadd->next = n;
            toadd = toadd->next;
        }

        /* default to whatever the global setting is */
        int shared = (de_ctx->sgh_mpm_ctx_cnf == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);

        /* see if we use a unique or shared mpm ctx for this type */
        int confshared = 0;
        char confstring[256] = "detect.mpm.";
        strlcat(confstring, n->name, sizeof(confstring));
        strlcat(confstring, ".shared", sizeof(confstring));
        if (SCConfGetBool(confstring, &confshared) == 1)
            shared = confshared;

        if (shared == 0) {
            SCLogDebug("using unique mpm ctx' for %s", n->name);
            n->sgh_mpm_context = MPM_CTX_FACTORY_UNIQUE_CONTEXT;
        } else {
            SCLogDebug("using shared mpm ctx' for %s", n->name);
            n->sgh_mpm_context =
                    MpmFactoryRegisterMpmCtxProfile(de_ctx, n->name, n->sm_list, n->app_v2.alproto);
        }

        list = list->next;
    }
    de_ctx->app_mpms_list_cnt = g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_APP];
    SCLogDebug("mpm: de_ctx app_mpms_list %p %u",
            de_ctx->app_mpms_list, de_ctx->app_mpms_list_cnt);
}

/**
 *  \brief initialize mpm contexts for applayer buffers that are in
 *         "single or "shared" mode.
 */
int DetectMpmPrepareAppMpms(DetectEngineCtx *de_ctx)
{
    int r = 0;
    const DetectBufferMpmRegistry *am = de_ctx->app_mpms_list;
    while (am != NULL) {
        int dir = (am->direction == SIG_FLAG_TOSERVER) ? 1 : 0;

        if (am->sgh_mpm_context != MPM_CTX_FACTORY_UNIQUE_CONTEXT)
        {
            MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
            if (mpm_ctx != NULL) {
                if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
                }
            }
        }
        am = am->next;
    }
    return r;
}

/** \brief register a MPM engine
 *
 *  \note to be used at start up / registration only. Errors are fatal.
 */
void DetectFrameMpmRegister(const char *name, int direction, int priority,
        int (*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
                const DetectBufferMpmRegistry *mpm_reg, int list_id),
        AppProto alproto, uint8_t type)
{
    SCLogDebug("registering %s/%d/%p/%s/%u", name, priority, PrefilterRegister,
            AppProtoToString(alproto), type);

    DetectBufferTypeSupportsMpm(name);
    DetectBufferTypeSupportsFrames(name);
    DetectBufferTypeSupportsTransformations(name);
    int sm_list = DetectBufferTypeGetByName(name);
    if (sm_list < 0 || sm_list > UINT16_MAX) {
        FatalError("MPM engine registration for %s failed", name);
    }

    DetectBufferMpmRegistry *am = SCCalloc(1, sizeof(*am));
    BUG_ON(am == NULL);
    am->name = name;
    snprintf(am->pname, sizeof(am->pname), "%s", am->name);
    am->sm_list = (uint16_t)sm_list;
    am->direction = direction;
    am->priority = priority;
    am->type = DETECT_BUFFER_MPM_TYPE_FRAME;

    am->PrefilterRegisterWithListId = PrefilterRegister;
    am->frame_v1.alproto = alproto;
    am->frame_v1.type = type;
    SCLogDebug("type %u", type);
    SCLogDebug("am type %u", am->frame_v1.type);

    if (g_mpm_list[DETECT_BUFFER_MPM_TYPE_FRAME] == NULL) {
        g_mpm_list[DETECT_BUFFER_MPM_TYPE_FRAME] = am;
    } else {
        DetectBufferMpmRegistry *t = g_mpm_list[DETECT_BUFFER_MPM_TYPE_FRAME];
        while (t->next != NULL) {
            t = t->next;
        }
        t->next = am;
        am->id = t->id + 1;
    }
    g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_FRAME]++;

    SupportFastPatternForSigMatchList(sm_list, priority);
    SCLogDebug("%s/%d done", name, sm_list);
}

/** \brief copy a mpm engine from parent_id, add in transforms */
void DetectFrameMpmRegisterByParentId(DetectEngineCtx *de_ctx, const int id, const int parent_id,
        DetectEngineTransforms *transforms)
{
    SCLogDebug("registering %d/%d", id, parent_id);

    DetectBufferMpmRegistry *t = de_ctx->frame_mpms_list;
    while (t) {
        if (t->sm_list == parent_id) {
            DetectBufferMpmRegistry *am = SCCalloc(1, sizeof(*am));
            BUG_ON(am == NULL);
            am->name = t->name;
            DEBUG_VALIDATE_BUG_ON(id < 0 || id > UINT16_MAX);
            am->sm_list = (uint16_t)id; // use new id
            am->sm_list_base = t->sm_list;
            am->type = DETECT_BUFFER_MPM_TYPE_FRAME;
            am->PrefilterRegisterWithListId = t->PrefilterRegisterWithListId;
            am->frame_v1 = t->frame_v1;
            SCLogDebug("am type %u", am->frame_v1.type);
            am->priority = t->priority;
            am->direction = t->direction;
            am->sgh_mpm_context = t->sgh_mpm_context;
            am->next = t->next;

            BuildBasicPname(am->pname, sizeof(am->pname), am->name, (uint16_t)id);
            if (transforms) {
                memcpy(&am->transforms, transforms, sizeof(*transforms));
                AppendTransformsToPname(am->pname, sizeof(am->pname), transforms);
            }
            am->id = de_ctx->frame_mpms_list_cnt++;

            DetectEngineRegisterFastPatternForId(de_ctx, am->sm_list, am->priority);
            t->next = am;
            SCLogDebug("copied mpm registration for %s id %u "
                       "with parent %u",
                    t->name, id, parent_id);
            t = am;
        }
        t = t->next;
    }
}

void DetectEngineFrameMpmRegister(DetectEngineCtx *de_ctx, const char *name, int direction,
        int priority,
        int (*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
                const DetectBufferMpmRegistry *mpm_reg, int list_id),
        AppProto alproto, uint8_t type)
{
    SCLogDebug("registering %s/%d/%p/%s/%u", name, priority, PrefilterRegister,
            AppProtoToString(alproto), type);

    const int sm_list = DetectEngineBufferTypeRegister(de_ctx, name);
    if (sm_list < 0 || sm_list > UINT16_MAX) {
        FatalError("MPM engine registration for %s failed", name);
    }

    DetectEngineBufferTypeSupportsMpm(de_ctx, name);
    DetectEngineBufferTypeSupportsFrames(de_ctx, name);
    DetectEngineBufferTypeSupportsTransformations(de_ctx, name);

    DetectBufferMpmRegistry *am = SCCalloc(1, sizeof(*am));
    BUG_ON(am == NULL);
    am->name = name;
    snprintf(am->pname, sizeof(am->pname), "%s", am->name);
    am->sm_list = (uint16_t)sm_list;
    am->direction = direction;
    am->priority = priority;
    am->type = DETECT_BUFFER_MPM_TYPE_FRAME;

    am->PrefilterRegisterWithListId = PrefilterRegister;
    am->frame_v1.alproto = alproto;
    am->frame_v1.type = type;

    // TODO is it ok to do this here?

    /* default to whatever the global setting is */
    int shared = (de_ctx->sgh_mpm_ctx_cnf == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);
    /* see if we use a unique or shared mpm ctx for this type */
    int confshared = 0;
    if (SCConfGetBool("detect.mpm.frame.shared", &confshared) == 1)
        shared = confshared;

    if (shared == 0) {
        am->sgh_mpm_context = MPM_CTX_FACTORY_UNIQUE_CONTEXT;
    } else {
        am->sgh_mpm_context =
                MpmFactoryRegisterMpmCtxProfile(de_ctx, am->name, am->sm_list, alproto);
    }

    if (de_ctx->frame_mpms_list == NULL) {
        de_ctx->frame_mpms_list = am;
    } else {
        DetectBufferMpmRegistry *t = de_ctx->frame_mpms_list;
        while (t->next != NULL) {
            t = t->next;
        }

        t->next = am;
    }
    de_ctx->frame_mpms_list_cnt++;

    DetectEngineRegisterFastPatternForId(de_ctx, sm_list, priority);
    SCLogDebug("%s/%d done", name, sm_list);
}

void DetectMpmInitializeFrameMpms(DetectEngineCtx *de_ctx)
{
    const DetectBufferMpmRegistry *list = g_mpm_list[DETECT_BUFFER_MPM_TYPE_FRAME];
    while (list != NULL) {
        DetectBufferMpmRegistry *n = SCCalloc(1, sizeof(*n));
        BUG_ON(n == NULL);

        *n = *list;
        n->next = NULL;

        if (de_ctx->frame_mpms_list == NULL) {
            de_ctx->frame_mpms_list = n;
        } else {
            DetectBufferMpmRegistry *t = de_ctx->frame_mpms_list;
            while (t->next != NULL) {
                t = t->next;
            }

            t->next = n;
        }

        /* default to whatever the global setting is */
        int shared = (de_ctx->sgh_mpm_ctx_cnf == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);

        /* see if we use a unique or shared mpm ctx for this type */
        int confshared = 0;
        char confstring[256] = "detect.mpm.";
        strlcat(confstring, n->name, sizeof(confstring));
        strlcat(confstring, ".shared", sizeof(confstring));
        if (SCConfGetBool(confstring, &confshared) == 1)
            shared = confshared;

        if (shared == 0) {
            SCLogDebug("using unique mpm ctx' for %s", n->name);
            n->sgh_mpm_context = MPM_CTX_FACTORY_UNIQUE_CONTEXT;
        } else {
            SCLogDebug("using shared mpm ctx' for %s", n->name);
            n->sgh_mpm_context = MpmFactoryRegisterMpmCtxProfile(
                    de_ctx, n->name, n->sm_list, n->frame_v1.alproto);
        }

        list = list->next;
    }
    de_ctx->frame_mpms_list_cnt = g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_FRAME];
    SCLogDebug("mpm: de_ctx frame_mpms_list %p %u", de_ctx->frame_mpms_list,
            de_ctx->frame_mpms_list_cnt);
}

/**
 *  \brief initialize mpm contexts for applayer buffers that are in
 *         "single or "shared" mode.
 */
int DetectMpmPrepareFrameMpms(DetectEngineCtx *de_ctx)
{
    SCLogDebug("preparing frame mpm");
    int r = 0;
    const DetectBufferMpmRegistry *am = de_ctx->frame_mpms_list;
    while (am != NULL) {
        SCLogDebug("am %p %s sgh_mpm_context %d", am, am->name, am->sgh_mpm_context);
        SCLogDebug("%s", am->name);
        if (am->sgh_mpm_context != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
            int dir = (am->direction == SIG_FLAG_TOSERVER) ? 1 : 0;
            MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, dir);
            SCLogDebug("%s: %d mpm_Ctx %p", am->name, r, mpm_ctx);
            if (mpm_ctx != NULL) {
                if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
                    SCLogDebug("%s: %d", am->name, r);
                }
            }
        }
        am = am->next;
    }
    return r;
}

/** \brief register a MPM engine
 *
 *  \note to be used at start up / registration only. Errors are fatal.
 */
void DetectPktMpmRegister(const char *name, int priority,
        int (*PrefilterRegister)(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx,
                const DetectBufferMpmRegistry *mpm_reg, int list_id),
        InspectionBufferGetPktDataPtr GetData)
{
    SCLogDebug("registering %s/%d/%p/%p", name, priority,
            PrefilterRegister, GetData);

    // must register GetData with PrefilterGenericMpmRegister
    BUG_ON(PrefilterRegister == PrefilterGenericMpmPktRegister && GetData == NULL);

    DetectBufferTypeSupportsMpm(name);
    DetectBufferTypeSupportsTransformations(name);
    int sm_list = DetectBufferTypeGetByName(name);
    if (sm_list == -1) {
        FatalError("MPM engine registration for %s failed", name);
    }

    DetectBufferMpmRegistry *am = SCCalloc(1, sizeof(*am));
    BUG_ON(am == NULL);
    am->name = name;
    snprintf(am->pname, sizeof(am->pname), "%s", am->name);
    DEBUG_VALIDATE_BUG_ON(sm_list < 0 || sm_list > INT16_MAX);
    am->sm_list = (uint16_t)sm_list;
    am->priority = priority;
    am->type = DETECT_BUFFER_MPM_TYPE_PKT;

    am->PrefilterRegisterWithListId = PrefilterRegister;
    am->pkt_v1.GetData = GetData;

    if (g_mpm_list[DETECT_BUFFER_MPM_TYPE_PKT] == NULL) {
        g_mpm_list[DETECT_BUFFER_MPM_TYPE_PKT] = am;
    } else {
        DetectBufferMpmRegistry *t = g_mpm_list[DETECT_BUFFER_MPM_TYPE_PKT];
        while (t->next != NULL) {
            t = t->next;
        }
        t->next = am;
        am->id = t->id + 1;
    }
    g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_PKT]++;

    SupportFastPatternForSigMatchList(sm_list, priority);
    SCLogDebug("%s/%d done", name, sm_list);
}

/** \brief copy a mpm engine from parent_id, add in transforms */
void DetectPktMpmRegisterByParentId(DetectEngineCtx *de_ctx,
        const int id, const int parent_id,
        DetectEngineTransforms *transforms)
{
    SCLogDebug("registering %d/%d", id, parent_id);

    DetectBufferMpmRegistry *t = de_ctx->pkt_mpms_list;
    while (t) {
        if (t->sm_list == parent_id) {
            DetectBufferMpmRegistry *am = SCCalloc(1, sizeof(*am));
            BUG_ON(am == NULL);
            am->name = t->name;
            DEBUG_VALIDATE_BUG_ON(id < 0 || id > INT16_MAX);
            am->sm_list = (uint16_t)id; // use new id
            am->sm_list_base = t->sm_list;
            am->type = DETECT_BUFFER_MPM_TYPE_PKT;
            am->PrefilterRegisterWithListId = t->PrefilterRegisterWithListId;
            am->pkt_v1.GetData = t->pkt_v1.GetData;
            am->priority = t->priority;
            am->sgh_mpm_context = t->sgh_mpm_context;
            am->next = t->next;

            BuildBasicPname(am->pname, sizeof(am->pname), am->name, (uint16_t)id);
            if (transforms) {
                memcpy(&am->transforms, transforms, sizeof(*transforms));
                AppendTransformsToPname(am->pname, sizeof(am->pname), transforms);
            }
            am->id = de_ctx->pkt_mpms_list_cnt++;

            DetectEngineRegisterFastPatternForId(de_ctx, am->sm_list, am->priority);
            t->next = am;
            SCLogDebug("copied mpm registration for %s id %u "
                    "with parent %u and GetData %p",
                    t->name, id, parent_id, am->pkt_v1.GetData);
            t = am;
        }
        t = t->next;
    }
}

void DetectMpmInitializePktMpms(DetectEngineCtx *de_ctx)
{
    const DetectBufferMpmRegistry *list = g_mpm_list[DETECT_BUFFER_MPM_TYPE_PKT];
    while (list != NULL) {
        DetectBufferMpmRegistry *n = SCCalloc(1, sizeof(*n));
        BUG_ON(n == NULL);

        *n = *list;
        n->next = NULL;

        if (de_ctx->pkt_mpms_list == NULL) {
            de_ctx->pkt_mpms_list = n;
        } else {
            DetectBufferMpmRegistry *t = de_ctx->pkt_mpms_list;
            while (t->next != NULL) {
                t = t->next;
            }

            t->next = n;
        }

        /* default to whatever the global setting is */
        int shared = (de_ctx->sgh_mpm_ctx_cnf == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);

        /* see if we use a unique or shared mpm ctx for this type */
        int confshared = 0;
        char confstring[256] = "detect.mpm.";
        strlcat(confstring, n->name, sizeof(confstring));
        strlcat(confstring, ".shared", sizeof(confstring));
        if (SCConfGetBool(confstring, &confshared) == 1)
            shared = confshared;

        if (shared == 0) {
            SCLogDebug("using unique mpm ctx' for %s", n->name);
            n->sgh_mpm_context = MPM_CTX_FACTORY_UNIQUE_CONTEXT;
        } else {
            SCLogDebug("using shared mpm ctx' for %s", n->name);
            n->sgh_mpm_context =
                    MpmFactoryRegisterMpmCtxProfile(de_ctx, n->name, n->sm_list, ALPROTO_UNKNOWN);
        }

        list = list->next;
    }
    de_ctx->pkt_mpms_list_cnt = g_mpm_list_cnt[DETECT_BUFFER_MPM_TYPE_PKT];
    SCLogDebug("mpm: de_ctx pkt_mpms_list %p %u",
            de_ctx->pkt_mpms_list, de_ctx->pkt_mpms_list_cnt);
}

/**
 *  \brief initialize mpm contexts for applayer buffers that are in
 *         "single or "shared" mode.
 */
int DetectMpmPreparePktMpms(DetectEngineCtx *de_ctx)
{
    SCLogDebug("preparing pkt mpm");
    int r = 0;
    const DetectBufferMpmRegistry *am = de_ctx->pkt_mpms_list;
    while (am != NULL) {
        SCLogDebug("%s", am->name);
        if (am->sgh_mpm_context != MPM_CTX_FACTORY_UNIQUE_CONTEXT)
        {
            MpmCtx *mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, am->sgh_mpm_context, 0);
            if (mpm_ctx != NULL) {
                if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
                    r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
                    SCLogDebug("%s: %d", am->name, r);
                }
            }
        }
        am = am->next;
    }
    return r;
}

static int32_t SetupBuiltinMpm(DetectEngineCtx *de_ctx, const char *name)
{
    /* default to whatever the global setting is */
    int shared = (de_ctx->sgh_mpm_ctx_cnf == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE);

    /* see if we use a unique or shared mpm ctx for this type */
    int confshared = 0;
    char confstring[256] = "detect.mpm.";
    strlcat(confstring, name, sizeof(confstring));
    strlcat(confstring, ".shared", sizeof(confstring));
    if (SCConfGetBool(confstring, &confshared) == 1)
        shared = confshared;

    int32_t ctx;
    if (shared == 0) {
        ctx = MPM_CTX_FACTORY_UNIQUE_CONTEXT;
        SCLogDebug("using unique mpm ctx' for %s", name);
    } else {
        ctx = MpmFactoryRegisterMpmCtxProfile(de_ctx, name, DETECT_SM_LIST_PMATCH, ALPROTO_UNKNOWN);
        SCLogDebug("using shared mpm ctx' for %s", name);
    }
    return ctx;
}

void DetectMpmInitializeBuiltinMpms(DetectEngineCtx *de_ctx)
{
    de_ctx->sgh_mpm_context_proto_tcp_packet = SetupBuiltinMpm(de_ctx, "tcp-packet");
    de_ctx->sgh_mpm_context_stream = SetupBuiltinMpm(de_ctx, "tcp-stream");

    de_ctx->sgh_mpm_context_proto_udp_packet = SetupBuiltinMpm(de_ctx, "udp-packet");
    de_ctx->sgh_mpm_context_proto_other_packet = SetupBuiltinMpm(de_ctx, "other-ip");
}

/**
 *  \brief initialize mpm contexts for builtin buffers that are in
 *         "single or "shared" mode.
 */
int DetectMpmPrepareBuiltinMpms(DetectEngineCtx *de_ctx)
{
    int r = 0;
    MpmCtx *mpm_ctx = NULL;

    if (de_ctx->sgh_mpm_context_proto_tcp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 0);
        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
            r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
        }
        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_tcp_packet, 1);
        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
            r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
        }
    }

    if (de_ctx->sgh_mpm_context_proto_udp_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 0);
        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
            r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
        }
        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_udp_packet, 1);
        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
            r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
        }
    }

    if (de_ctx->sgh_mpm_context_proto_other_packet != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_proto_other_packet, 0);
        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
            r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
        }
    }

    if (de_ctx->sgh_mpm_context_stream != MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 0);
        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
            r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
        }
        mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, de_ctx->sgh_mpm_context_stream, 1);
        if (mpm_table[de_ctx->mpm_matcher].Prepare != NULL) {
            r |= mpm_table[de_ctx->mpm_matcher].Prepare(de_ctx->mpm_cfg, mpm_ctx);
        }
    }

    return r;
}

/**
 *  \brief check if a signature has patterns that are to be inspected
 *         against a packets payload (as opposed to the stream payload)
 *
 *  \param s signature
 *
 *  \retval 1 true
 *  \retval 0 false
 */
int SignatureHasPacketContent(const Signature *s)
{
    SCEnter();

    if (!DetectProtoContainsProto(&s->init_data->proto, IPPROTO_TCP)) {
        SCReturnInt(1);
    }

    if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) {
        SCLogDebug("no PMATCH");
        SCReturnInt(0);
    }

    if (!(s->flags & SIG_FLAG_REQUIRE_PACKET)) {
        SCReturnInt(0);
    }

    SCReturnInt(1);
}

/**
 *  \brief check if a signature has patterns that are to be inspected
 *         against the stream payload (as opposed to the individual packets
 *         payload(s))
 *
 *  \param s signature
 *
 *  \retval 1 true
 *  \retval 0 false
 */
int SignatureHasStreamContent(const Signature *s)
{
    SCEnter();

    if (!DetectProtoContainsProto(&s->init_data->proto, IPPROTO_TCP)) {
        SCReturnInt(0);
    }

    if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] == NULL) {
        SCLogDebug("no PMATCH");
        SCReturnInt(0);
    }

    if (!(s->flags & SIG_FLAG_REQUIRE_STREAM)) {
        SCReturnInt(0);
    }

    SCReturnInt(1);
}


/**
 *  \brief  Function to return the multi pattern matcher algorithm to be
 *          used by the engine, based on the mpm-algo setting in yaml
 *          Use the default mpm if none is specified in the yaml file.
 *
 *  \retval mpm algo value
 */
uint8_t PatternMatchDefaultMatcher(void)
{
    const char *mpm_algo;
    uint8_t mpm_algo_val = mpm_default_matcher;

    /* Get the mpm algo defined in config file by the user */
    if ((SCConfGet("mpm-algo", &mpm_algo)) == 1) {
        if (mpm_algo != NULL) {
#if __BYTE_ORDER == __BIG_ENDIAN
            if (strcmp(mpm_algo, "ac-ks") == 0) {
                FatalError("ac-ks does "
                           "not work on big endian systems at this time.");
            }
#endif
            if (strcmp("auto", mpm_algo) == 0) {
                goto done;
            } else if (strcmp("ac-bs", mpm_algo) == 0) {
                SCLogWarning("mpm-algo \"ac-bs\" has been removed. See ticket #6586.");
                goto done;
            }
            for (uint8_t u = 0; u < MPM_TABLE_SIZE; u++) {
                if (mpm_table[u].name == NULL)
                    continue;

                if (strcmp(mpm_table[u].name, mpm_algo) == 0) {
                    mpm_algo_val = u;
                    goto done;
                }
            }

#ifndef BUILD_HYPERSCAN
            if ((strcmp(mpm_algo, "hs") == 0)) {
                FatalError("Hyperscan (hs) support for mpm-algo is "
                           "not compiled into Suricata.");
            }
#endif
        }
        FatalError("Invalid mpm algo supplied "
                   "in the yaml conf file: \"%s\"",
                mpm_algo);
    }

 done:
    return mpm_algo_val;
}

void PatternMatchDestroy(MpmCtx *mpm_ctx, uint16_t mpm_matcher)
{
    SCLogDebug("mpm_ctx %p, mpm_matcher %"PRIu16"", mpm_ctx, mpm_matcher);
    mpm_table[mpm_matcher].DestroyCtx(mpm_ctx);
}

void PatternMatchThreadDestroy(MpmThreadCtx *mpm_thread_ctx, uint16_t mpm_matcher)
{
    SCLogDebug("mpm_thread_ctx %p, mpm_matcher %"PRIu16"", mpm_thread_ctx, mpm_matcher);
    MpmDestroyThreadCtx(mpm_thread_ctx, mpm_matcher);
}
void PatternMatchThreadPrepare(MpmThreadCtx *mpm_thread_ctx, DetectEngineCtx *de_ctx)
{
    SCLogDebug("mpm_thread_ctx %p, type %" PRIu16, mpm_thread_ctx, de_ctx->mpm_matcher);
    MpmCtx cum_mpm_ctx = { 0 };
    for (HashListTableBucket *htb = HashListTableGetListHead(de_ctx->mpm_hash_table); htb != NULL;
            htb = HashListTableGetListNext(htb)) {
        // iterate all de_ctx mpms to merge one MpmCtx with max pattern_cnt and max max_pat_id
        const MpmStore *ms = (MpmStore *)HashListTableGetListData(htb);
        if (ms == NULL || ms->mpm_ctx == NULL) {
            continue;
        }
        if (ms->mpm_ctx->pattern_cnt > cum_mpm_ctx.pattern_cnt)
            cum_mpm_ctx.pattern_cnt = ms->mpm_ctx->pattern_cnt;
        if (ms->mpm_ctx->max_pat_id > cum_mpm_ctx.max_pat_id)
            cum_mpm_ctx.max_pat_id = ms->mpm_ctx->max_pat_id;
    }
    MpmInitThreadCtx(mpm_thread_ctx, &cum_mpm_ctx, de_ctx->mpm_matcher);
}

/** \brief Predict a strength value for patterns
 *
 *  Patterns with high character diversity score higher.
 *  Alpha chars score not so high
 *  Other printable + a few common codes a little higher
 *  Everything else highest.
 *  Longer patterns score better than short patters.
 *
 *  \param pat pattern
 *  \param patlen length of the pattern
 *
 *  \retval s pattern score
 */
uint32_t PatternStrength(uint8_t *pat, uint16_t patlen)
{
    uint8_t a[256];
    memset(&a, 0 ,sizeof(a));

    uint32_t s = 0;
    uint16_t u = 0;
    for (u = 0; u < patlen; u++) {
        if (a[pat[u]] == 0) {
            if (isalpha(pat[u]))
                s += 3;
            else if (isprint(pat[u]) || pat[u] == 0x00 || pat[u] == 0x01 || pat[u] == 0xFF)
                s += 4;
            else
                s += 6;

            a[pat[u]] = 1;
        } else {
            s++;
        }
    }

    return s;
}

static void PopulateMpmHelperAddPattern(MpmCtx *mpm_ctx, const DetectContentData *cd,
        const Signature *s, const uint8_t flags, const int chop)
{
    uint16_t pat_offset = cd->offset;
    uint16_t pat_depth = cd->depth;

    /* recompute offset/depth to cope with chop */
    if (chop && (pat_depth || pat_offset)) {
        pat_offset += cd->fp_chop_offset;
        if (pat_depth) {
            pat_depth -= cd->content_len;
            pat_depth += cd->fp_chop_offset + cd->fp_chop_len;
        }
    }

    /* We have to effectively "wild card" values that will be coming from
     * byte_extract variables
     */
    if (cd->flags & (DETECT_CONTENT_DEPTH_VAR | DETECT_CONTENT_OFFSET_VAR)) {
        pat_depth = pat_offset = 0;
    }

    if (cd->flags & DETECT_CONTENT_NOCASE) {
        if (chop) {
            SCMpmAddPatternCI(mpm_ctx, cd->content + cd->fp_chop_offset, cd->fp_chop_len,
                    pat_offset, pat_depth, cd->id, s->iid, flags | MPM_PATTERN_CTX_OWNS_ID);
        } else {
            SCMpmAddPatternCI(mpm_ctx, cd->content, cd->content_len, pat_offset, pat_depth, cd->id,
                    s->iid, flags | MPM_PATTERN_CTX_OWNS_ID);
        }
    } else {
        if (chop) {
            MpmAddPatternCS(mpm_ctx, cd->content + cd->fp_chop_offset, cd->fp_chop_len, pat_offset,
                    pat_depth, cd->id, s->iid, flags | MPM_PATTERN_CTX_OWNS_ID);
        } else {
            MpmAddPatternCS(mpm_ctx, cd->content, cd->content_len, pat_offset, pat_depth, cd->id,
                    s->iid, flags | MPM_PATTERN_CTX_OWNS_ID);
        }
    }
}

#define SGH_PROTO(sgh, p) ((sgh)->init->protos[(p)] == 1)
#define SGH_DIRECTION_TS(sgh) ((sgh)->init->direction & SIG_FLAG_TOSERVER)
#define SGH_DIRECTION_TC(sgh) ((sgh)->init->direction & SIG_FLAG_TOCLIENT)

static void SetMpm(Signature *s, SigMatch *mpm_sm, const int mpm_sm_list)
{
    if (s == NULL || mpm_sm == NULL)
        return;

    DetectContentData *cd = (DetectContentData *)mpm_sm->ctx;
    if (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
        if (DETECT_CONTENT_IS_SINGLE(cd) &&
                !(cd->flags & DETECT_CONTENT_NEGATED) &&
                !(cd->flags & DETECT_CONTENT_REPLACE) &&
                cd->content_len == cd->fp_chop_len)
        {
            cd->flags |= DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED;
        }
    } else {
        if (DETECT_CONTENT_IS_SINGLE(cd) &&
                !(cd->flags & DETECT_CONTENT_NEGATED) &&
                !(cd->flags & DETECT_CONTENT_REPLACE))
        {
            cd->flags |= DETECT_CONTENT_NO_DOUBLE_INSPECTION_REQUIRED;
        }
    }
    cd->flags |= DETECT_CONTENT_MPM;
    s->init_data->mpm_sm_list = mpm_sm_list;
    s->init_data->mpm_sm = mpm_sm;
}

static SigMatch *GetMpmForList(const Signature *s, SigMatch *list, SigMatch *mpm_sm,
        uint16_t max_len, bool skip_negated_content)
{
    for (SigMatch *sm = list; sm != NULL; sm = sm->next) {
        if (sm->type != DETECT_CONTENT)
            continue;

        const DetectContentData *cd = (DetectContentData *)sm->ctx;
        /* skip_negated_content is only set if there's absolutely no
         * non-negated content present in the sig */
        if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
            continue;
        if (cd->content_len != max_len) {
            SCLogDebug("content_len %u != max_len %u", cd->content_len, max_len);
            continue;
        }
        if (mpm_sm == NULL) {
            mpm_sm = sm;
        } else {
            DetectContentData *data1 = (DetectContentData *)sm->ctx;
            DetectContentData *data2 = (DetectContentData *)mpm_sm->ctx;
            uint32_t ls = PatternStrength(data1->content, data1->content_len);
            uint32_t ss = PatternStrength(data2->content, data2->content_len);
            if (ls > ss) {
                mpm_sm = sm;
            } else if (ls == ss) {
                /* if 2 patterns are of equal strength, we pick the longest */
                if (data1->content_len > data2->content_len)
                    mpm_sm = sm;
            } else {
                SCLogDebug("sticking with mpm_sm");
            }
        }
    }
    return mpm_sm;
}

int g_skip_prefilter = 0;

// tells if a buffer id is only used to client
bool DetectBufferToClient(const DetectEngineCtx *de_ctx, int buf_id, AppProto alproto)
{
    bool r = false;
    const DetectEngineAppInspectionEngine *app = de_ctx->app_inspect_engines;
    for (; app != NULL; app = app->next) {
        if (app->sm_list == buf_id &&
                (AppProtoEquals(alproto, app->alproto) || alproto == ALPROTO_UNKNOWN)) {
            if (app->dir == 1) {
                // do not return yet in case we have app engines on both sides
                r = true;
            } else {
                // ambiguous keywords have a app-engine to server
                return false;
            }
        }
    }
    return r;
}

void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s)
{
    if (g_skip_prefilter)
        return;

    if (s->init_data->mpm_sm != NULL)
        return;

    /* A transform that derives its key from a runtime byte_extract or byte_math
     * variable cannot have its buffer precomputed at load time, so the content
     * of such a buffer is unusable as a fast_pattern. Disqualify the whole
     * signature from prefilter; it falls back to full inspection. */
    if (s->init_data->var_transform) {
        SCLogDebug("sid %u uses a variable-key transform; skipping fast_pattern", s->id);
        return;
    }

    const int nlists = s->init_data->max_content_list_id + 1;
    DEBUG_VALIDATE_BUG_ON(nlists > UINT16_MAX);
    int pos_sm_list[nlists];
    int neg_sm_list[nlists];
    memset(pos_sm_list, 0, nlists * sizeof(int));
    memset(neg_sm_list, 0, nlists * sizeof(int));
    int pos_sm_list_cnt = 0;
    int neg_sm_list_cnt = 0;

    /* inspect rule to see if we have the fast_pattern reg to
     * force using a sig, otherwise keep stats about the patterns */
    if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) {
        if (FastPatternSupportEnabledForSigMatchList(de_ctx, DETECT_SM_LIST_PMATCH)) {
            for (SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_PMATCH]; sm != NULL;
                    sm = sm->next) {
                if (sm->type != DETECT_CONTENT)
                    continue;

                const DetectContentData *cd = (DetectContentData *)sm->ctx;
                /* fast_pattern set in rule, so using this pattern */
                if ((cd->flags & DETECT_CONTENT_FAST_PATTERN)) {
                    SetMpm(s, sm, DETECT_SM_LIST_PMATCH);
                    return;
                }

                if (cd->flags & DETECT_CONTENT_NEGATED) {
                    neg_sm_list[DETECT_SM_LIST_PMATCH] = 1;
                    neg_sm_list_cnt++;
                } else {
                    pos_sm_list[DETECT_SM_LIST_PMATCH] = 1;
                    pos_sm_list_cnt++;
                }
            }
        }
    }
    for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
        const int list_id = s->init_data->buffers[x].id;

        SCLogDebug("%u: list_id %d: %s", s->id, list_id,
                DetectEngineBufferTypeGetNameById(de_ctx, list_id));

        if (!FastPatternSupportEnabledForSigMatchList(de_ctx, list_id)) {
            SCLogDebug("skip");
            continue;
        }

        for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) {
            // a buffer with absent keyword cannot be used as fast_pattern
            if (sm->type == DETECT_ABSENT)
                break;
            if (sm->type != DETECT_CONTENT)
                continue;

            const DetectContentData *cd = (DetectContentData *)sm->ctx;
            /* fast_pattern set in rule, so using this pattern */
            if ((cd->flags & DETECT_CONTENT_FAST_PATTERN)) {
                SetMpm(s, sm, list_id);
                return;
            }

            if (cd->flags & DETECT_CONTENT_NEGATED) {
                neg_sm_list[list_id] = 1;
                neg_sm_list_cnt++;
            } else {
                pos_sm_list[list_id] = 1;
                pos_sm_list_cnt++;
                SCLogDebug("pos added for %d", list_id);
            }
        }
        SCLogDebug("ok");
    }

    SCLogDebug("neg_sm_list_cnt %d pos_sm_list_cnt %d", neg_sm_list_cnt, pos_sm_list_cnt);

    /* prefer normal not-negated over negated */
    int *curr_sm_list = NULL;
    int skip_negated_content = 1;
    if (pos_sm_list_cnt > 0) {
        curr_sm_list = pos_sm_list;
    } else if (neg_sm_list_cnt > 0) {
        curr_sm_list = neg_sm_list;
        skip_negated_content = 0;
    } else {
        return;
    }

    int final_sm_list[nlists];
    memset(&final_sm_list, 0, (nlists * sizeof(int)));

    int count_final_sm_list = 0;
    int count_txbidir_toclient_sm_list = 0;
    int priority;

    const SCFPSupportSMList *tmp = de_ctx->fp_support_smlist_list;
    while (tmp != NULL) {
        for (priority = tmp->priority;
             tmp != NULL && priority == tmp->priority;
             tmp = tmp->next)
        {
            SCLogDebug("tmp->list_id %d tmp->priority %d", tmp->list_id, tmp->priority);
            if (tmp->list_id >= nlists)
                continue;
            if (curr_sm_list[tmp->list_id] == 0)
                continue;
            if (s->flags & SIG_FLAG_TXBOTHDIR) {
                // prefer to choose a fast_pattern to server by default
                if (DetectBufferToClient(de_ctx, tmp->list_id, s->alproto)) {
                    if (count_final_sm_list == 0) {
                        // still put it in in the case we do not have toserver buffer
                        final_sm_list[count_txbidir_toclient_sm_list++] = tmp->list_id;
                    }
                    continue;
                }
            }
            // we may erase tx bidir toclient buffers here as intended if we have a better choice
            final_sm_list[count_final_sm_list++] = tmp->list_id;
            SCLogDebug("tmp->list_id %d", tmp->list_id);
        }
        if (count_final_sm_list != 0)
            break;
    }

    if ((s->flags & SIG_FLAG_TXBOTHDIR) && count_final_sm_list == 0) {
        // forced to pick a fast_pattern to client for tx bidir signature
        count_final_sm_list = count_txbidir_toclient_sm_list;
    }
    BUG_ON(count_final_sm_list == 0);
    SCLogDebug("count_final_sm_list %d skip_negated_content %d", count_final_sm_list,
            skip_negated_content);

    uint16_t max_len = 0;
    for (int i = 0; i < count_final_sm_list; i++) {
        SCLogDebug("i %d final_sm_list[i] %d", i, final_sm_list[i]);

        if (final_sm_list[i] == DETECT_SM_LIST_PMATCH) {
            for (SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_PMATCH]; sm != NULL;
                    sm = sm->next) {
                if (sm->type != DETECT_CONTENT)
                    continue;

                const DetectContentData *cd = (DetectContentData *)sm->ctx;
                /* skip_negated_content is only set if there's absolutely no
                 * non-negated content present in the sig */
                if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
                    continue;
                max_len = MAX(max_len, cd->content_len);
            }
        } else {
            for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
                if (s->init_data->buffers[x].only_tc) {
                    // prefer to choose a fast_pattern to server by default
                    continue;
                }
                const int list_id = s->init_data->buffers[x].id;

                if (final_sm_list[i] == list_id) {
                    SCLogDebug("%u: list_id %d: %s", s->id, list_id,
                            DetectEngineBufferTypeGetNameById(de_ctx, list_id));

                    for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) {
                        if (sm->type != DETECT_CONTENT)
                            continue;

                        const DetectContentData *cd = (DetectContentData *)sm->ctx;
                        /* skip_negated_content is only set if there's absolutely no
                         * non-negated content present in the sig */
                        if ((cd->flags & DETECT_CONTENT_NEGATED) && skip_negated_content)
                            continue;
                        max_len = MAX(max_len, cd->content_len);
                    }
                }
            }
        }
    }

    SigMatch *mpm_sm = NULL;
    int mpm_sm_list = -1;
    for (int i = 0; i < count_final_sm_list; i++) {
        SCLogDebug("i %d", i);
        if (final_sm_list[i] == DETECT_SM_LIST_PMATCH) {
            /* GetMpmForList may keep `mpm_sm` the same, so track if it changed */
            SigMatch *prev_mpm_sm = mpm_sm;
            mpm_sm = GetMpmForList(s, s->init_data->smlists[DETECT_SM_LIST_PMATCH], mpm_sm, max_len,
                    skip_negated_content);
            if (mpm_sm != prev_mpm_sm) {
                mpm_sm_list = final_sm_list[i];
            }
        } else {
            SCLogDebug(
                    "%u: %s", s->id, DetectEngineBufferTypeGetNameById(de_ctx, final_sm_list[i]));
            for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
                const int list_id = s->init_data->buffers[x].id;
                if (final_sm_list[i] == list_id) {
                    SCLogDebug("%u: list_id %d: %s", s->id, list_id,
                            DetectEngineBufferTypeGetNameById(de_ctx, list_id));
                    /* GetMpmForList may keep `mpm_sm` the same, so track if it changed */
                    SigMatch *prev_mpm_sm = mpm_sm;
                    mpm_sm = GetMpmForList(s, s->init_data->buffers[x].head, mpm_sm, max_len,
                            skip_negated_content);
                    SCLogDebug("mpm_sm %p from %p", mpm_sm, s->init_data->buffers[x].head);
                    if (mpm_sm != prev_mpm_sm) {
                        mpm_sm_list = list_id;
                    }
                }
            }
        }
    }

#ifdef DEBUG
    if (mpm_sm != NULL) {
        BUG_ON(mpm_sm_list == -1);
        int check_list = SigMatchListSMBelongsTo(s, mpm_sm);
        BUG_ON(check_list != mpm_sm_list);
    }
#endif
    /* assign to signature */
    SetMpm(s, mpm_sm, mpm_sm_list);
}

/** \internal
 *  \brief The hash function for MpmStore
 *
 *  \param ht      Pointer to the hash table.
 *  \param data    Pointer to the MpmStore.
 *  \param datalen Not used in our case.
 *
 *  \retval hash The generated hash value.
 */
static uint32_t MpmStoreHashFunc(HashListTable *ht, void *data, uint16_t datalen)
{
    const MpmStore *ms = (MpmStore *)data;
    uint32_t hash = ms->alproto;

    for (uint32_t b = 0; b < ms->sid_array_size; b++)
        hash += ms->sid_array[b];

    return hash % ht->array_size;
}

/**
 * \brief The Compare function for MpmStore
 *
 * \param data1 Pointer to the first MpmStore.
 * \param len1  Not used.
 * \param data2 Pointer to the second MpmStore.
 * \param len2  Not used.
 *
 * \retval 1 If the 2 MpmStores sent as args match.
 * \retval 0 If the 2 MpmStores sent as args do not match.
 */
static char MpmStoreCompareFunc(void *data1, uint16_t len1, void *data2,
                                uint16_t len2)
{
    const MpmStore *ms1 = (MpmStore *)data1;
    const MpmStore *ms2 = (MpmStore *)data2;

    if (ms1->alproto != ms2->alproto)
        return 0;

    if (ms1->sid_array_size != ms2->sid_array_size)
        return 0;

    if (ms1->buffer != ms2->buffer)
        return 0;

    if (ms1->direction != ms2->direction)
        return 0;

    if (ms1->sm_list != ms2->sm_list)
        return 0;

    if (SCMemcmp(ms1->sid_array, ms2->sid_array,
                 ms1->sid_array_size) != 0)
    {
        return 0;
    }

    return 1;
}

static void MpmStoreFreeFunc(void *ptr)
{
    MpmStore *ms = ptr;
    if (ms != NULL) {
        if (ms->mpm_ctx != NULL && !(ms->mpm_ctx->flags & MPMCTX_FLAGS_GLOBAL))
        {
            SCLogDebug("destroying mpm_ctx %p", ms->mpm_ctx);
            mpm_table[ms->mpm_ctx->mpm_type].DestroyCtx(ms->mpm_ctx);
            SCFree(ms->mpm_ctx);
        }
        ms->mpm_ctx = NULL;

        SCFree(ms->sid_array);
        SCFree(ms);
    }
}

/**
 * \brief Initializes the MpmStore mpm hash table to be used by the detection
 *        engine context.
 *
 * \param de_ctx Pointer to the detection engine context.
 *
 * \retval  0 On success.
 * \retval -1 On failure.
 */
int MpmStoreInit(DetectEngineCtx *de_ctx)
{
    de_ctx->mpm_hash_table = HashListTableInit(4096,
                                               MpmStoreHashFunc,
                                               MpmStoreCompareFunc,
                                               MpmStoreFreeFunc);
    if (de_ctx->mpm_hash_table == NULL)
        goto error;

    return 0;

error:
    return -1;
}

/**
 * \brief Adds a MpmStore to the detection engine context MpmStore
 *
 * \param de_ctx Pointer to the detection engine context.
 * \param sgh    Pointer to the MpmStore.
 *
 * \retval ret 0 on Successfully adding the argument sgh; -1 on failure.
 */
static int MpmStoreAdd(DetectEngineCtx *de_ctx, MpmStore *s)
{
    int ret = HashListTableAdd(de_ctx->mpm_hash_table, (void *)s, 0);
    return ret;
}

/**
 * \brief Used to lookup a MpmStore from the MpmStore
 *
 * \param de_ctx Pointer to the detection engine context.
 * \param sgh    Pointer to the MpmStore.
 *
 * \retval rsgh On success a pointer to the MpmStore if the MpmStore is
 *              found in the hash table; NULL on failure.
 */
static MpmStore *MpmStoreLookup(DetectEngineCtx *de_ctx, MpmStore *s)
{
    MpmStore *rs = HashListTableLookup(de_ctx->mpm_hash_table,
                                             (void *)s, 0);
    return rs;
}

static const DetectBufferMpmRegistry *GetByMpmStore(
        const DetectEngineCtx *de_ctx, const MpmStore *ms)
{
    const DetectBufferMpmRegistry *am = de_ctx->app_mpms_list;
    while (am != NULL) {
        if (ms->sm_list == am->sm_list &&
            ms->direction == am->direction) {
            return am;
        }
        am = am->next;
    }
    am = de_ctx->pkt_mpms_list;
    while (am != NULL) {
        if (ms->sm_list == am->sm_list) {
            return am;
        }
        am = am->next;
    }
    return NULL;
}

void MpmStoreReportStats(const DetectEngineCtx *de_ctx)
{
    HashListTableBucket *htb = NULL;
    uint32_t *appstats = NULL;
    uint32_t *pktstats = NULL;
    uint32_t *framestats = NULL;

    uint32_t stats[MPMB_MAX] = {0};
    appstats = SCCalloc(de_ctx->buffer_type_id, sizeof(uint32_t));
    if (appstats == NULL) {
        goto end;
    }
    pktstats = SCCalloc(de_ctx->buffer_type_id, sizeof(uint32_t));
    if (pktstats == NULL) {
        goto end;
    }
    framestats = SCCalloc(de_ctx->buffer_type_id, sizeof(uint32_t));
    if (framestats == NULL) {
        goto end;
    }

    for (htb = HashListTableGetListHead(de_ctx->mpm_hash_table);
            htb != NULL;
            htb = HashListTableGetListNext(htb))
    {
        const MpmStore *ms = (MpmStore *)HashListTableGetListData(htb);
        if (ms == NULL || ms->mpm_ctx == NULL) {
            continue;
        }
        if (ms->buffer < MPMB_MAX)
            stats[ms->buffer]++;
        else if (ms->sm_list != DETECT_SM_LIST_PMATCH) {
            const DetectBufferMpmRegistry *am = GetByMpmStore(de_ctx, ms);
            if (am != NULL) {
                switch (am->type) {
                    case DETECT_BUFFER_MPM_TYPE_PKT:
                        SCLogDebug("%s: %u patterns. Min %u, Max %u. Ctx %p",
                                am->name,
                                ms->mpm_ctx->pattern_cnt,
                                ms->mpm_ctx->minlen, ms->mpm_ctx->maxlen,
                                ms->mpm_ctx);
                        pktstats[am->sm_list]++;
                        break;
                    case DETECT_BUFFER_MPM_TYPE_APP:
                        SCLogDebug("%s %s %s: %u patterns. Min %u, Max %u. Ctx %p",
                                AppProtoToString(ms->alproto), am->name,
                                am->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
                                ms->mpm_ctx->pattern_cnt, ms->mpm_ctx->minlen, ms->mpm_ctx->maxlen,
                                ms->mpm_ctx);
                        appstats[am->sm_list]++;
                        break;
                    case DETECT_BUFFER_MPM_TYPE_FRAME:
                        SCLogDebug("%s: %u patterns. Min %u, Max %u. Ctx %p", am->name,
                                ms->mpm_ctx->pattern_cnt, ms->mpm_ctx->minlen, ms->mpm_ctx->maxlen,
                                ms->mpm_ctx);
                        framestats[am->sm_list]++;
                        break;
                    case DETECT_BUFFER_MPM_TYPE_SIZE:
                        break;
                }
            }
        }
    }

    if (!(de_ctx->flags & DE_QUIET)) {
        for (int x = 0; x < MPMB_MAX; x++) {
            SCLogPerf("Builtin MPM \"%s\": %u", builtin_mpms[x], stats[x]);
        }
        const DetectBufferMpmRegistry *am = de_ctx->app_mpms_list;
        while (am != NULL) {
            if (appstats[am->sm_list] > 0) {
                const char *name = am->name;
                const char *direction = am->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient";
                SCLogPerf("AppLayer MPM \"%s %s (%s)\": %u", direction, name,
                        AppProtoToString(am->app_v2.alproto), appstats[am->sm_list]);
            }
            am = am->next;
        }
        const DetectBufferMpmRegistry *pm = de_ctx->pkt_mpms_list;
        while (pm != NULL) {
            if (pktstats[pm->sm_list] > 0) {
                const char *name = pm->name;
                SCLogPerf("Pkt MPM \"%s\": %u", name, pktstats[pm->sm_list]);
            }
            pm = pm->next;
        }
        const DetectBufferMpmRegistry *um = de_ctx->frame_mpms_list;
        while (um != NULL) {
            if (framestats[um->sm_list] > 0) {
                const char *name = um->name;
                SCLogPerf("Frame MPM \"%s\": %u", name, framestats[um->sm_list]);
            }
            um = um->next;
        }
    }
end:
    if (appstats)
        SCFree(appstats);
    if (pktstats)
        SCFree(pktstats);
    if (framestats)
        SCFree(framestats);
}

/**
 * \brief Frees the hash table - DetectEngineCtx->mpm_hash_table, allocated by
 *        MpmStoreInit() function.
 *
 * \param de_ctx Pointer to the detection engine context.
 */
void MpmStoreFree(DetectEngineCtx *de_ctx)
{
    if (de_ctx->mpm_hash_table == NULL)
        return;

    HashListTableFree(de_ctx->mpm_hash_table);
    de_ctx->mpm_hash_table = NULL;
}

static void MpmStoreSetup(const DetectEngineCtx *de_ctx, MpmStore *ms)
{
    const Signature *s = NULL;
    uint32_t sig;
    int dir = 0;

    if (ms->buffer != MPMB_MAX) {
        BUG_ON(ms->sm_list != DETECT_SM_LIST_PMATCH);

        switch (ms->buffer) {
            /* TS is 1 */
            case MPMB_TCP_PKT_TS:
            case MPMB_TCP_STREAM_TS:
            case MPMB_UDP_TS:
                dir = 1;
                break;

                /* TC is 0 */
            default:
            case MPMB_UDP_TC:
            case MPMB_TCP_STREAM_TC:
            case MPMB_TCP_PKT_TC:
            case MPMB_OTHERIP:          /**< use 0 for other */
                dir = 0;
                break;
        }
    } else {
        BUG_ON(ms->sm_list == DETECT_SM_LIST_PMATCH);

        if (ms->direction == SIG_FLAG_TOSERVER)
            dir = 1;
        else
            dir = 0;
    }

    ms->mpm_ctx = MpmFactoryGetMpmCtxForProfile(de_ctx, ms->sgh_mpm_context, dir);
    if (ms->mpm_ctx == NULL) {
        return;
    }

    MpmInitCtx(ms->mpm_ctx, de_ctx->mpm_matcher);

    if (de_ctx->mpm_cfg && de_ctx->mpm_cfg->cache_dir_path)
        ms->mpm_ctx->flags |= MPMCTX_FLAGS_CACHE_TO_DISK;

    const bool mpm_supports_endswith =
            (mpm_table[de_ctx->mpm_matcher].feature_flags & MPM_FEATURE_FLAG_ENDSWITH) != 0;

    /* add the patterns */
    for (sig = 0; sig < (ms->sid_array_size * 8); sig++) {
        if (ms->sid_array[sig / 8] & (1 << (sig % 8))) {
            s = de_ctx->sig_array[sig];
            DEBUG_VALIDATE_BUG_ON(s == NULL);
            if (s == NULL)
                continue;

            SCLogDebug("%p: direction %d adding %u", ms, ms->direction, s->id);

            const DetectContentData *cd = (DetectContentData *)s->init_data->mpm_sm->ctx;

            int skip = 0;
            /* negated logic: if mpm match can't be used to be sure about this
             * pattern, we have to inspect the rule fully regardless of mpm
             * match. So in this case there is no point of adding it at all.
             * The non-mpm list entry for the sig will make sure the sig is
             * inspected. */
            if ((cd->flags & DETECT_CONTENT_NEGATED) &&
                !(DETECT_CONTENT_MPM_IS_CONCLUSIVE(cd)))
            {
                skip = 1;
                SCLogDebug("not adding negated mpm as it's not 'single'");
            }

            if (!skip) {
                uint8_t flags = 0;
                if ((cd->flags & DETECT_CONTENT_ENDS_WITH) && mpm_supports_endswith)
                    flags = MPM_PATTERN_FLAG_ENDSWITH;
                PopulateMpmHelperAddPattern(ms->mpm_ctx, cd, s, flags,
                        (cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) != 0);
            }
        }
    }

    if (ms->mpm_ctx->pattern_cnt == 0) {
        MpmFactoryReClaimMpmCtx(de_ctx, ms->mpm_ctx);
        ms->mpm_ctx = NULL;
    } else {
        if (ms->sgh_mpm_context == MPM_CTX_FACTORY_UNIQUE_CONTEXT) {
            if (mpm_table[ms->mpm_ctx->mpm_type].Prepare != NULL) {
                mpm_table[ms->mpm_ctx->mpm_type].Prepare(de_ctx->mpm_cfg, ms->mpm_ctx);
            }
        }
    }
}


/** \brief Get MpmStore for a built-in buffer type
 *
 */
MpmStore *MpmStorePrepareBuffer(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
                                enum MpmBuiltinBuffers buf)
{
    const Signature *s = NULL;
    uint32_t sig;
    uint32_t cnt = 0;
    int direction = 0;
    uint32_t max_sid = DetectEngineGetMaxSigId(de_ctx) / 8 + 1;
    int sgh_mpm_context = 0;
    int sm_list = DETECT_SM_LIST_PMATCH;
    uint8_t *sids_array = SCCalloc(1, max_sid);
    if (sids_array == NULL) {
        return NULL;
    }
    switch (buf) {
        case MPMB_TCP_PKT_TS:
        case MPMB_TCP_PKT_TC:
            sgh_mpm_context = de_ctx->sgh_mpm_context_proto_tcp_packet;
            break;
        case MPMB_TCP_STREAM_TS:
        case MPMB_TCP_STREAM_TC:
            sgh_mpm_context = de_ctx->sgh_mpm_context_stream;
            break;
        case MPMB_UDP_TS:
        case MPMB_UDP_TC:
            sgh_mpm_context = de_ctx->sgh_mpm_context_proto_udp_packet;
            break;
        case MPMB_OTHERIP:
            sgh_mpm_context = de_ctx->sgh_mpm_context_proto_other_packet;
            break;
        default:
            break;
    }

    switch(buf) {
        case MPMB_TCP_PKT_TS:
        case MPMB_TCP_STREAM_TS:
        case MPMB_UDP_TS:
            direction = SIG_FLAG_TOSERVER;
            break;

        case MPMB_TCP_PKT_TC:
        case MPMB_TCP_STREAM_TC:
        case MPMB_UDP_TC:
            direction = SIG_FLAG_TOCLIENT;
            break;

        case MPMB_OTHERIP:
            direction = (SIG_FLAG_TOCLIENT|SIG_FLAG_TOSERVER);
            break;

        case MPMB_MAX:
            BUG_ON(1);
            break;
    }

    for (sig = 0; sig < sgh->init->sig_cnt; sig++) {
        s = sgh->init->match_array[sig];
        if (s == NULL)
            continue;

        if (s->init_data->mpm_sm == NULL)
            continue;

        int list = s->init_data->mpm_sm_list;
        if (list < 0)
            continue;

        if (list != DETECT_SM_LIST_PMATCH)
            continue;

        switch (buf) {
            case MPMB_TCP_PKT_TS:
            case MPMB_TCP_PKT_TC:
                if (SignatureHasPacketContent(s) == 1)
                {
                    sids_array[s->iid / 8] |= 1 << (s->iid % 8);
                    cnt++;
                }
                break;
            case MPMB_TCP_STREAM_TS:
            case MPMB_TCP_STREAM_TC:
                if (SignatureHasStreamContent(s) == 1)
                {
                    sids_array[s->iid / 8] |= 1 << (s->iid % 8);
                    cnt++;
                }
                break;
            case MPMB_UDP_TS:
            case MPMB_UDP_TC:
                sids_array[s->iid / 8] |= 1 << (s->iid % 8);
                cnt++;
                break;
            case MPMB_OTHERIP:
                sids_array[s->iid / 8] |= 1 << (s->iid % 8);
                cnt++;
                break;
            default:
                break;
        }
    }

    if (cnt == 0) {
        SCFree(sids_array);
        return NULL;
    }

    MpmStore lookup = { sids_array, max_sid, direction, buf, sm_list, 0, 0, NULL };

    MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
    if (result == NULL) {
        MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
        if (copy == NULL)
            return NULL;
        uint8_t *sids = SCCalloc(1, max_sid);
        if (sids == NULL) {
            SCFree(copy);
            SCFree(sids_array);
            return NULL;
        }

        memcpy(sids, sids_array, max_sid);
        copy->sid_array = sids;
        copy->sid_array_size = max_sid;
        copy->buffer = buf;
        copy->direction = direction;
        copy->sm_list = sm_list;
        copy->sgh_mpm_context = sgh_mpm_context;

        MpmStoreSetup(de_ctx, copy);
        MpmStoreAdd(de_ctx, copy);
        SCFree(sids_array);
        return copy;
    } else {
        SCFree(sids_array);
        return result;
    }
}

struct SidsArray {
    uint8_t *sids_array;
    uint32_t sids_array_size;
    /* indicates this has an active engine */
    bool active;

    enum DetectBufferMpmType type;
};

static MpmStore *MpmStorePrepareBufferAppLayer(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
        const DetectBufferMpmRegistry *am, const struct SidsArray *sa)
{
    if (sa->sids_array_size == 0 || sa->sids_array == NULL)
        return NULL;

    SCLogDebug("handling %s direction %s for list %d", am->name,
            am->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient",
            am->sm_list);

    MpmStore lookup = { sa->sids_array, sa->sids_array_size, am->direction, MPMB_MAX, am->sm_list,
        0, am->app_v2.alproto, NULL };
    SCLogDebug("am->direction %d am->sm_list %d sgh_mpm_context %d", am->direction, am->sm_list,
            am->sgh_mpm_context);

    MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
    if (result == NULL) {
        SCLogDebug("new unique mpm for %s %s", am->name,
                am->direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient");

        MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
        if (copy == NULL)
            return NULL;
        uint8_t *sids = SCCalloc(1, sa->sids_array_size);
        if (sids == NULL) {
            SCFree(copy);
            return NULL;
        }

        memcpy(sids, sa->sids_array, sa->sids_array_size);
        copy->sid_array = sids;
        copy->sid_array_size = sa->sids_array_size;
        copy->buffer = MPMB_MAX;
        copy->direction = am->direction;
        copy->sm_list = am->sm_list;
        copy->sgh_mpm_context = am->sgh_mpm_context;
        copy->alproto = am->app_v2.alproto;

        MpmStoreSetup(de_ctx, copy);
        MpmStoreAdd(de_ctx, copy);
        return copy;
    } else {
        SCLogDebug("using existing mpm %p", result);
        return result;
    }
    return NULL;
}

static MpmStore *MpmStorePrepareBufferPkt(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
        const DetectBufferMpmRegistry *am, const struct SidsArray *sa)
{
    SCLogDebug("handling %s for list %d", am->name,
            am->sm_list);

    if (sa->sids_array_size == 0 || sa->sids_array == NULL)
        return NULL;

    MpmStore lookup = { sa->sids_array, sa->sids_array_size, SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT,
        MPMB_MAX, am->sm_list, 0, 0, NULL };
    SCLogDebug("am->sm_list %d", am->sm_list);

    MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
    if (result == NULL) {
        SCLogDebug("new unique mpm for %s", am->name);

        MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
        if (copy == NULL)
            return NULL;
        uint8_t *sids = SCCalloc(1, sa->sids_array_size);
        if (sids == NULL) {
            SCFree(copy);
            return NULL;
        }

        memcpy(sids, sa->sids_array, sa->sids_array_size);
        copy->sid_array = sids;
        copy->sid_array_size = sa->sids_array_size;
        copy->buffer = MPMB_MAX;
        copy->direction = SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT;
        copy->sm_list = am->sm_list;
        copy->sgh_mpm_context = am->sgh_mpm_context;

        MpmStoreSetup(de_ctx, copy);
        MpmStoreAdd(de_ctx, copy);
        return copy;
    } else {
        SCLogDebug("using existing mpm %p", result);
        return result;
    }
    return NULL;
}

static MpmStore *MpmStorePrepareBufferFrame(DetectEngineCtx *de_ctx, SigGroupHead *sgh,
        const DetectBufferMpmRegistry *am, const struct SidsArray *sa)
{
    SCLogDebug("handling %s for list %d", am->name, am->sm_list);

    if (sa->sids_array_size == 0 || sa->sids_array == NULL)
        return NULL;

    MpmStore lookup = { sa->sids_array, sa->sids_array_size, am->direction, MPMB_MAX, am->sm_list,
        0, am->frame_v1.alproto, NULL };
    SCLogDebug("am->sm_list %d", am->sm_list);

    MpmStore *result = MpmStoreLookup(de_ctx, &lookup);
    if (result == NULL) {
        SCLogDebug("new unique mpm for %s", am->name);

        MpmStore *copy = SCCalloc(1, sizeof(MpmStore));
        if (copy == NULL)
            return NULL;
        uint8_t *sids = SCCalloc(1, sa->sids_array_size);
        if (sids == NULL) {
            SCFree(copy);
            return NULL;
        }

        memcpy(sids, sa->sids_array, sa->sids_array_size);
        copy->sid_array = sids;
        copy->sid_array_size = sa->sids_array_size;
        copy->buffer = MPMB_MAX;
        copy->direction = am->direction;
        copy->sm_list = am->sm_list;
        copy->sgh_mpm_context = am->sgh_mpm_context;
        copy->alproto = am->frame_v1.alproto;

        MpmStoreSetup(de_ctx, copy);
        MpmStoreAdd(de_ctx, copy);
        return copy;
    } else {
        SCLogDebug("using existing mpm %p", result);
        return result;
    }
    return NULL;
}

static void SetRawReassemblyFlag(DetectEngineCtx *de_ctx, SigGroupHead *sgh)
{
    const Signature *s = NULL;
    uint32_t sig;

    for (sig = 0; sig < sgh->init->sig_cnt; sig++) {
        s = sgh->init->match_array[sig];
        if (s == NULL)
            continue;

        if (SignatureHasStreamContent(s) == 1) {
            sgh->flags |= SIG_GROUP_HEAD_HAVERAWSTREAM;
            SCLogDebug("rule group %p has SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
            return;
        }
    }
    SCLogDebug("rule group %p does NOT have SIG_GROUP_HEAD_HAVERAWSTREAM set", sgh);
}

typedef struct DetectBufferInstance {
    // key
    int list;
    AppProto alproto;

    struct SidsArray ts;
    struct SidsArray tc;
} DetectBufferInstance;

static uint32_t DetectBufferInstanceHashFunc(HashListTable *ht, void *data, uint16_t datalen)
{
    const DetectBufferInstance *ms = (const DetectBufferInstance *)data;
    uint32_t hash = ms->list + ms->alproto;
    return hash % ht->array_size;
}

static char DetectBufferInstanceCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
{
    const DetectBufferInstance *ms1 = (DetectBufferInstance *)data1;
    const DetectBufferInstance *ms2 = (DetectBufferInstance *)data2;
    return (ms1->list == ms2->list && ms1->alproto == ms2->alproto);
}

static void DetectBufferInstanceFreeFunc(void *ptr)
{
    DetectBufferInstance *ms = ptr;
    if (ms->ts.sids_array != NULL)
        SCFree(ms->ts.sids_array);
    if (ms->tc.sids_array != NULL)
        SCFree(ms->tc.sids_array);
    SCFree(ms);
}

static HashListTable *DetectBufferInstanceInit(void)
{
    return HashListTableInit(4096, DetectBufferInstanceHashFunc, DetectBufferInstanceCompareFunc,
            DetectBufferInstanceFreeFunc);
}

typedef struct MpmEngineList {
    // We check for overflows before filling this array
    // ALPROTO_MAX_STATIC constant should be enough for all engines
    // (file.data should be the engine with most AppProtos and that would be 6)
    AppProto array[ALPROTO_MAX_STATIC];
    size_t idx;
} MpmEngineList;

static void PrepareMpms(DetectEngineCtx *de_ctx, SigGroupHead *sh)
{
    HashListTable *bufs = DetectBufferInstanceInit();
    BUG_ON(bufs == NULL);

    const int max_buffer_id = de_ctx->buffer_type_id + 1;
    const uint32_t max_sid = DetectEngineGetMaxSigId(de_ctx) / 8 + 1;

    MpmEngineList *engines = SCCalloc(max_buffer_id, sizeof(MpmEngineList));
    BUG_ON(engines == NULL);
    int *types = SCCalloc(max_buffer_id, sizeof(int));
    BUG_ON(types == NULL);

    /* flag the list+directions we have engines for as active */
    for (DetectBufferMpmRegistry *a = de_ctx->pkt_mpms_list; a != NULL; a = a->next) {
        types[a->sm_list] = a->type;

        DetectBufferInstance lookup = { .list = a->sm_list, .alproto = ALPROTO_UNKNOWN };
        DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
        if (instance == NULL) {
            instance = SCCalloc(1, sizeof(*instance));
            BUG_ON(instance == NULL);
            instance->list = a->sm_list;
            instance->alproto = ALPROTO_UNKNOWN;
            HashListTableAdd(bufs, instance, 0);
        }
        instance->ts.active = true;
        instance->tc.active = true;
    }
    for (DetectBufferMpmRegistry *a = de_ctx->frame_mpms_list; a != NULL; a = a->next) {
        const bool add_ts = ((a->direction == SIG_FLAG_TOSERVER) && SGH_DIRECTION_TS(sh));
        const bool add_tc = ((a->direction == SIG_FLAG_TOCLIENT) && SGH_DIRECTION_TC(sh));
        if (add_ts || add_tc) {
            types[a->sm_list] = a->type;
            BUG_ON(engines[a->sm_list].idx >= ARRAY_SIZE(engines[a->sm_list].array));
            engines[a->sm_list].array[engines[a->sm_list].idx++] = a->frame_v1.alproto;

            DetectBufferInstance lookup = { .list = a->sm_list, .alproto = a->frame_v1.alproto };
            DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
            if (instance == NULL) {
                instance = SCCalloc(1, sizeof(*instance));
                BUG_ON(instance == NULL);
                instance->list = a->sm_list;
                instance->alproto = a->frame_v1.alproto;
                HashListTableAdd(bufs, instance, 0);
            }
            instance->ts.active |= add_ts;
            instance->tc.active |= add_tc;
        }
    }
    for (DetectBufferMpmRegistry *a = de_ctx->app_mpms_list; a != NULL; a = a->next) {
        const bool add_ts = ((a->direction == SIG_FLAG_TOSERVER) && SGH_DIRECTION_TS(sh));
        const bool add_tc = ((a->direction == SIG_FLAG_TOCLIENT) && SGH_DIRECTION_TC(sh));
        if (add_ts || add_tc) {
            types[a->sm_list] = a->type;
            BUG_ON(engines[a->sm_list].idx >= ARRAY_SIZE(engines[a->sm_list].array));
            engines[a->sm_list].array[engines[a->sm_list].idx++] = a->app_v2.alproto;

            DetectBufferInstance lookup = { .list = a->sm_list, .alproto = a->app_v2.alproto };
            DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
            if (instance == NULL) {
                instance = SCCalloc(1, sizeof(*instance));
                BUG_ON(instance == NULL);
                instance->list = a->sm_list;
                instance->alproto = a->app_v2.alproto;
                HashListTableAdd(bufs, instance, 0);
            }
            instance->ts.active |= add_ts;
            instance->tc.active |= add_tc;
        }
    }

    for (uint32_t sig = 0; sig < sh->init->sig_cnt; sig++) {
        const Signature *s = sh->init->match_array[sig];
        if (s == NULL)
            continue;
        if (s->init_data->mpm_sm == NULL)
            continue;
        const int list = s->init_data->mpm_sm_list;
        if (list < 0)
            continue;
        if (list == DETECT_SM_LIST_PMATCH)
            continue;

        switch (types[list]) {
            /* app engines are direction aware */
            case DETECT_BUFFER_MPM_TYPE_FRAME:
            case DETECT_BUFFER_MPM_TYPE_APP: {
                for (size_t e = 0; e < engines[list].idx; e++) {
                    const AppProto alproto = engines[list].array[e];
                    if (s->init_data->hook.type == SIGNATURE_HOOK_TYPE_APP) {
                        /* SIGNATURE_HOOK_TYPE_APP rules are exact about their protocol */
                        if (!(AppProtoEqualsStrict(s->alproto, alproto))) {
                            continue;
                        }
                    } else {
                        /* other rules use the more relax AppProtoEquals logic */
                        if (!(AppProtoEquals(s->alproto, alproto) || s->alproto == 0))
                            continue;
                    }

                    DetectBufferInstance lookup = { .list = list, .alproto = alproto };
                    DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
                    if (instance == NULL)
                        continue;
                    if (s->flags & SIG_FLAG_TOSERVER) {
                        struct SidsArray *sa = &instance->ts;
                        if (sa->active) {
                            if (sa->sids_array == NULL) {
                                sa->sids_array = SCCalloc(1, max_sid);
                                sa->sids_array_size = max_sid;
                                BUG_ON(sa->sids_array == NULL); // TODO
                            }
                            sa->sids_array[s->iid / 8] |= 1 << (s->iid % 8);
                            SCLogDebug("instance %p: stored %u/%u ts", instance, s->id, s->iid);
                        }
                    }
                    if (s->flags & SIG_FLAG_TOCLIENT) {
                        struct SidsArray *sa = &instance->tc;
                        if (sa->active) {
                            if (sa->sids_array == NULL) {
                                sa->sids_array = SCCalloc(1, max_sid);
                                sa->sids_array_size = max_sid;
                                BUG_ON(sa->sids_array == NULL); // TODO
                            }
                            sa->sids_array[s->iid / 8] |= 1 << (s->iid % 8);
                            SCLogDebug("instance %p: stored %u/%u tc", instance, s->id, s->iid);
                        }
                    }
                }
                break;
            }
            /* pkt engines are directionless, so only use ts */
            case DETECT_BUFFER_MPM_TYPE_PKT: {
                DetectBufferInstance lookup = { .list = list, .alproto = ALPROTO_UNKNOWN };
                DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
                if (instance == NULL)
                    continue;
                struct SidsArray *sa = &instance->ts;
                if (sa->active) {
                    if (sa->sids_array == NULL) {
                        sa->sids_array = SCCalloc(1, max_sid);
                        sa->sids_array_size = max_sid;
                        BUG_ON(sa->sids_array == NULL); // TODO
                    }
                    sa->sids_array[s->iid / 8] |= 1 << (s->iid % 8);
                }
                break;
            }
            default:
                BUG_ON(1);
        }
    }

    sh->init->app_mpms = SCCalloc(de_ctx->app_mpms_list_cnt, sizeof(MpmCtx *));
    BUG_ON(sh->init->app_mpms == NULL);

    sh->init->pkt_mpms = SCCalloc(de_ctx->pkt_mpms_list_cnt, sizeof(MpmCtx *));
    BUG_ON(sh->init->pkt_mpms == NULL);

    sh->init->frame_mpms = SCCalloc(de_ctx->frame_mpms_list_cnt, sizeof(MpmCtx *));
    BUG_ON(sh->init->frame_mpms == NULL);

    for (DetectBufferMpmRegistry *a = de_ctx->pkt_mpms_list; a != NULL; a = a->next) {
        DetectBufferInstance lookup = { .list = a->sm_list, .alproto = ALPROTO_UNKNOWN };
        DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
        if (instance == NULL) {
            continue;
        }
        struct SidsArray *sa = &instance->ts;
        if (!sa->active)
            continue;

        MpmStore *mpm_store = MpmStorePrepareBufferPkt(de_ctx, sh, a, sa);
        if (mpm_store != NULL) {
            sh->init->pkt_mpms[a->id] = mpm_store->mpm_ctx;

            SCLogDebug("a %p a->name %s a->reg->PrefilterRegisterWithListId %p "
                    "mpm_store->mpm_ctx %p", a, a->name,
                    a->PrefilterRegisterWithListId, mpm_store->mpm_ctx);

            /* if we have just certain types of negated patterns,
             * mpm_ctx can be NULL */
            if (a->PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
                BUG_ON(a->PrefilterRegisterWithListId(de_ctx,
                            sh, mpm_store->mpm_ctx,
                            a, a->sm_list) != 0);
                SCLogDebug("mpm %s %d set up", a->name, a->sm_list);
            }
        }
    }
    for (DetectBufferMpmRegistry *a = de_ctx->frame_mpms_list; a != NULL; a = a->next) {
        if ((a->direction == SIG_FLAG_TOSERVER && SGH_DIRECTION_TS(sh)) ||
                (a->direction == SIG_FLAG_TOCLIENT && SGH_DIRECTION_TC(sh))) {
            DetectBufferInstance lookup = { .list = a->sm_list, .alproto = a->frame_v1.alproto };
            DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
            if (instance == NULL) {
                continue;
            }
            struct SidsArray *sa =
                    (a->direction == SIG_FLAG_TOSERVER) ? &instance->ts : &instance->tc;
            if (!sa->active)
                continue;

            SCLogDebug("a %s direction %d PrefilterRegisterWithListId %p", a->name, a->direction,
                    a->PrefilterRegisterWithListId);
            MpmStore *mpm_store = MpmStorePrepareBufferFrame(de_ctx, sh, a, sa);
            if (mpm_store != NULL) {
                sh->init->frame_mpms[a->id] = mpm_store->mpm_ctx;

                SCLogDebug("a %p a->name %s a->reg->PrefilterRegisterWithListId %p "
                           "mpm_store->mpm_ctx %p",
                        a, a->name, a->PrefilterRegisterWithListId, mpm_store->mpm_ctx);

                /* if we have just certain types of negated patterns,
                 * mpm_ctx can be NULL */
                SCLogDebug("mpm_store %p mpm_ctx %p", mpm_store, mpm_store->mpm_ctx);
                if (a->PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
                    BUG_ON(a->PrefilterRegisterWithListId(
                                   de_ctx, sh, mpm_store->mpm_ctx, a, a->sm_list) != 0);
                    SCLogDebug("mpm %s %d set up", a->name, a->sm_list);
                }
            }
        }
    }
    for (DetectBufferMpmRegistry *a = de_ctx->app_mpms_list; a != NULL; a = a->next) {
        if ((a->direction == SIG_FLAG_TOSERVER && SGH_DIRECTION_TS(sh)) ||
                (a->direction == SIG_FLAG_TOCLIENT && SGH_DIRECTION_TC(sh))) {

            DetectBufferInstance lookup = { .list = a->sm_list, .alproto = a->app_v2.alproto };
            DetectBufferInstance *instance = HashListTableLookup(bufs, &lookup, 0);
            if (instance == NULL) {
                continue;
            }
            struct SidsArray *sa =
                    (a->direction == SIG_FLAG_TOSERVER) ? &instance->ts : &instance->tc;
            if (!sa->active)
                continue;

            MpmStore *mpm_store = MpmStorePrepareBufferAppLayer(de_ctx, sh, a, sa);
            if (mpm_store != NULL) {
                sh->init->app_mpms[a->id] = mpm_store->mpm_ctx;

                SCLogDebug("a %p a->name %s a->PrefilterRegisterWithListId %p "
                           "mpm_store->mpm_ctx %p",
                        a, a->name, a->PrefilterRegisterWithListId, mpm_store->mpm_ctx);

                /* if we have just certain types of negated patterns,
                 * mpm_ctx can be NULL */
                if (a->PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
                    BUG_ON(a->PrefilterRegisterWithListId(
                                   de_ctx, sh, mpm_store->mpm_ctx, a, a->sm_list) != 0);
                    SCLogDebug("mpm %s %d set up", a->name, a->sm_list);
                }
            }
        }
    }
    HashListTableFree(bufs);
    SCFree(engines);
    SCFree(types);
}

/** \brief Prepare the pattern matcher ctx in a sig group head.
 *
 */
int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
{
    MpmStore *mpm_store = NULL;
    if (SGH_PROTO(sh, IPPROTO_TCP)) {
        if (SGH_DIRECTION_TS(sh)) {
            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_PKT_TS);
            if (mpm_store != NULL) {
                PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
            }

            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_STREAM_TS);
            SetRawReassemblyFlag(de_ctx, sh);
            if (mpm_store != NULL && mpm_store->mpm_ctx) {
                PrefilterPktStreamRegister(de_ctx, sh, mpm_store->mpm_ctx);
            } else if (sh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM) {
                MpmCtx *mpm_ctx = SCCalloc(1, sizeof(MpmCtx));
                if (mpm_ctx) {
                    // we want to update raw_stream_progress
                    // also when sgh does not have prefilter
                    PrefilterPktStreamRegisterNoMpm(de_ctx, sh, mpm_ctx);
                }
            }
        }
        if (SGH_DIRECTION_TC(sh)) {
            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_PKT_TC);
            if (mpm_store != NULL) {
                PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
            }

            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_TCP_STREAM_TC);
            SetRawReassemblyFlag(de_ctx, sh);
            if (mpm_store != NULL && mpm_store->mpm_ctx) {
                PrefilterPktStreamRegister(de_ctx, sh, mpm_store->mpm_ctx);
            } else if (sh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM) {
                MpmCtx *mpm_ctx = SCCalloc(1, sizeof(MpmCtx));
                if (mpm_ctx) {
                    PrefilterPktStreamRegisterNoMpm(de_ctx, sh, mpm_ctx);
                }
            }
       }
    } else if (SGH_PROTO(sh, IPPROTO_UDP)) {
        if (SGH_DIRECTION_TS(sh)) {
            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_UDP_TS);
            if (mpm_store != NULL) {
                PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
            }
        }
        if (SGH_DIRECTION_TC(sh)) {
            mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_UDP_TC);
            if (mpm_store != NULL) {
                PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
            }
        }
    } else {
        mpm_store = MpmStorePrepareBuffer(de_ctx, sh, MPMB_OTHERIP);
        if (mpm_store != NULL) {
            PrefilterPktPayloadRegister(de_ctx, sh, mpm_store->mpm_ctx);
        }
    }

    PrepareMpms(de_ctx, sh);
    return 0;
}

static inline uint32_t ContentFlagsForHash(const DetectContentData *cd)
{
    return cd->flags & (DETECT_CONTENT_NOCASE | DETECT_CONTENT_OFFSET | DETECT_CONTENT_DEPTH |
                               DETECT_CONTENT_NEGATED | DETECT_CONTENT_ENDS_WITH);
}

/** \internal
 *  \brief The hash function for Pattern. Hashes pattern after chop is applied.
 *
 *  \param ht      Pointer to the hash table.
 *  \param data    Pointer to the Pattern.
 *  \param datalen Not used in our case.
 *
 *  \retval hash The generated hash value.
 */
static uint32_t PatternChopHashFunc(HashListTable *ht, void *data, uint16_t datalen)
{
    const DetectPatternTracker *p = (DetectPatternTracker *)data;
    uint32_t hash = p->sm_list + ContentFlagsForHash(p->cd);
    uint16_t content_len = p->cd->content_len;
    const uint8_t *content = p->cd->content;
    if (p->cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
        content += p->cd->fp_chop_offset;
        content_len = p->cd->fp_chop_len;
    }
    hash += StringHashDjb2(content, content_len);
    return hash % ht->array_size;
}

/** \internal
 *  \brief The hash function for Pattern. Ignores chop.
 *
 *  \param ht      Pointer to the hash table.
 *  \param data    Pointer to the Pattern.
 *  \param datalen Not used in our case.
 *
 *  \retval hash The generated hash value.
 */
static uint32_t PatternNoChopHashFunc(HashListTable *ht, void *data, uint16_t datalen)
{
    const DetectPatternTracker *p = (DetectPatternTracker *)data;
    uint32_t hash = p->sm_list + ContentFlagsForHash(p->cd);
    hash += StringHashDjb2(p->cd->content, p->cd->content_len);
    return hash % ht->array_size;
}

/**
 * \brief The Compare function for Pattern. Compares patterns after chop is applied.
 *
 * \param data1 Pointer to the first Pattern.
 * \param len1  Not used.
 * \param data2 Pointer to the second Pattern.
 * \param len2  Not used.
 *
 * \retval 1 If the 2 Patterns sent as args match.
 * \retval 0 If the 2 Patterns sent as args do not match.
 */
static char PatternChopCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
{
    const DetectPatternTracker *p1 = (DetectPatternTracker *)data1;
    const DetectPatternTracker *p2 = (DetectPatternTracker *)data2;

    if (p1->sm_list != p2->sm_list)
        return 0;

    if (ContentFlagsForHash(p1->cd) != ContentFlagsForHash(p2->cd))
        return 0;

    uint16_t p1_content_len = p1->cd->content_len;
    uint8_t *p1_content = p1->cd->content;
    if (p1->cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
        p1_content += p1->cd->fp_chop_offset;
        p1_content_len = p1->cd->fp_chop_len;
    }
    uint16_t p2_content_len = p2->cd->content_len;
    uint8_t *p2_content = p2->cd->content;
    if (p2->cd->flags & DETECT_CONTENT_FAST_PATTERN_CHOP) {
        p2_content += p2->cd->fp_chop_offset;
        p2_content_len = p2->cd->fp_chop_len;
    }

    if (p1_content_len != p2_content_len)
        return 0;

    if (memcmp(p1_content, p2_content, p1_content_len) != 0) {
        return 0;
    }

    return 1;
}

/**
 * \brief The Compare function for Pattern. Ignores chop settings
 *
 * \param data1 Pointer to the first Pattern.
 * \param len1  Not used.
 * \param data2 Pointer to the second Pattern.
 * \param len2  Not used.
 *
 * \retval 1 If the 2 Patterns sent as args match.
 * \retval 0 If the 2 Patterns sent as args do not match.
 */
static char PatternNoChopCompareFunc(void *data1, uint16_t len1, void *data2, uint16_t len2)
{
    const DetectPatternTracker *p1 = (DetectPatternTracker *)data1;
    const DetectPatternTracker *p2 = (DetectPatternTracker *)data2;

    if (p1->sm_list != p2->sm_list)
        return 0;

    if (ContentFlagsForHash(p1->cd) != ContentFlagsForHash(p2->cd))
        return 0;

    if (p1->cd->content_len != p2->cd->content_len)
        return 0;

    if (memcmp(p1->cd->content, p2->cd->content, p1->cd->content_len) != 0) {
        return 0;
    }

    return 1;
}

static void PatternFreeFunc(void *ptr)
{
    SCFree(ptr);
}

/**
 * \brief Figure out the FP and their respective content ids for all the
 *        sigs in the engine.
 *
 * \param de_ctx Detection engine context.
 *
 * \retval  0 On success.
 * \retval -1 On failure.
 */
int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx)
{
    uint32_t cnt = 0;
    for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
        if (s->init_data->mpm_sm != NULL) {
            cnt++;
        }
    }
    /* no mpm rules */
    if (cnt == 0)
        return 0;

    HashListTable *ht =
            HashListTableInit(4096, PatternChopHashFunc, PatternChopCompareFunc, PatternFreeFunc);
    BUG_ON(ht == NULL);
    PatIntId max_id = 0;

    for (Signature *s = de_ctx->sig_list; s != NULL; s = s->next) {
        if (s->init_data->mpm_sm == NULL)
            continue;

        const int sm_list = s->init_data->mpm_sm_list;
        BUG_ON(sm_list == -1);

        DetectContentData *cd = (DetectContentData *)s->init_data->mpm_sm->ctx;

        DetectPatternTracker lookup = { .cd = cd, .sm_list = sm_list, .cnt = 0, .mpm = 0 };
        DetectPatternTracker *res = HashListTableLookup(ht, &lookup, 0);
        if (res) {
            res->cnt++;
            res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);

            cd->id = res->cd->id;
            SCLogDebug("%u: res id %u cnt %u", s->id, res->cd->id, res->cnt);
        } else {
            DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
            BUG_ON(add == NULL);
            add->cd = cd;
            add->sm_list = sm_list;
            add->cnt = 1;
            add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
            HashListTableAdd(ht, (void *)add, 0);

            cd->id = max_id++;
            SCLogDebug("%u: add id %u cnt %u", s->id, add->cd->id, add->cnt);
        }
    }

    HashListTableFree(ht);

    return 0;
}

/** \brief add all patterns on our stats hash
 *  Used to fill the hash later used by DumpPatterns()
 *  \note sets up the hash table on first call
 */
void EngineAnalysisAddAllRulePatterns(DetectEngineCtx *de_ctx, const Signature *s)
{
    if (de_ctx->pattern_hash_table == NULL) {
        de_ctx->pattern_hash_table = HashListTableInit(
                4096, PatternNoChopHashFunc, PatternNoChopCompareFunc, PatternFreeFunc);
        BUG_ON(de_ctx->pattern_hash_table == NULL);
    }
    if (s->sm_arrays[DETECT_SM_LIST_PMATCH]) {
        SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_PMATCH];
        do {
            switch (smd->type) {
                case DETECT_CONTENT: {
                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;
                    DetectPatternTracker lookup = {
                        .cd = cd, .sm_list = DETECT_SM_LIST_PMATCH, .cnt = 0, .mpm = 0
                    };
                    DetectPatternTracker *res =
                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
                    if (res) {
                        res->cnt++;
                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
                    } else {
                        DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
                        BUG_ON(add == NULL);
                        add->cd = cd;
                        add->sm_list = DETECT_SM_LIST_PMATCH;
                        add->cnt = 1;
                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
                    }
                    break;
                }
            }
            if (smd->is_last)
                break;
            smd++;
        } while (1);
    }

    const DetectEngineAppInspectionEngine *app = s->app_inspect;
    for (; app != NULL; app = app->next) {
        SigMatchData *smd = app->smd;
        while (smd) {
            switch (smd->type) {
                case DETECT_CONTENT: {
                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;

                    DetectPatternTracker lookup = {
                        .cd = cd, .sm_list = app->sm_list, .cnt = 0, .mpm = 0
                    };
                    DetectPatternTracker *res =
                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
                    if (res) {
                        res->cnt++;
                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
                    } else {
                        DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
                        BUG_ON(add == NULL);
                        add->cd = cd;
                        add->sm_list = app->sm_list;
                        add->cnt = 1;
                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
                    }
                    break;
                }
            }
            if (smd->is_last)
                break;
            smd++;
        }
    }
    const DetectEnginePktInspectionEngine *pkt = s->pkt_inspect;
    for (; pkt != NULL; pkt = pkt->next) {
        SigMatchData *smd = pkt->smd;
        do {
            if (smd == NULL) {
                BUG_ON(!(pkt->sm_list < DETECT_SM_LIST_DYNAMIC_START));
                smd = s->sm_arrays[pkt->sm_list];
            }
            switch (smd->type) {
                case DETECT_CONTENT: {
                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;

                    DetectPatternTracker lookup = {
                        .cd = cd, .sm_list = pkt->sm_list, .cnt = 0, .mpm = 0
                    };
                    DetectPatternTracker *res =
                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
                    if (res) {
                        res->cnt++;
                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
                    } else {
                        DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
                        BUG_ON(add == NULL);
                        add->cd = cd;
                        add->sm_list = pkt->sm_list;
                        add->cnt = 1;
                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
                    }
                    break;
                }
            }
            if (smd->is_last)
                break;
            smd++;
        } while (1);
    }
    const DetectEngineFrameInspectionEngine *frame = s->frame_inspect;
    for (; frame != NULL; frame = frame->next) {
        SigMatchData *smd = frame->smd;
        do {
            if (smd == NULL) {
                BUG_ON(!(frame->sm_list < DETECT_SM_LIST_DYNAMIC_START));
                smd = s->sm_arrays[frame->sm_list];
            }
            switch (smd->type) {
                case DETECT_CONTENT: {
                    const DetectContentData *cd = (const DetectContentData *)smd->ctx;

                    DetectPatternTracker lookup = {
                        .cd = cd, .sm_list = frame->sm_list, .cnt = 0, .mpm = 0
                    };
                    DetectPatternTracker *res =
                            HashListTableLookup(de_ctx->pattern_hash_table, &lookup, 0);
                    if (res) {
                        res->cnt++;
                        res->mpm += ((cd->flags & DETECT_CONTENT_MPM) != 0);
                    } else {
                        DetectPatternTracker *add = SCCalloc(1, sizeof(*add));
                        BUG_ON(add == NULL);
                        add->cd = cd;
                        add->sm_list = frame->sm_list;
                        add->cnt = 1;
                        add->mpm = ((cd->flags & DETECT_CONTENT_MPM) != 0);
                        HashListTableAdd(de_ctx->pattern_hash_table, (void *)add, 0);
                    }
                    break;
                }
            }
            if (smd->is_last)
                break;
            smd++;
        } while (1);
    }
}