detect: prefilter/inspect API v2, with transforms

Introduce InspectionBuffer a structure for passing data between
prefilters, transforms and inspection engines.

At rule parsing time, we'll register new unique 'DetectBufferType's
for a 'parent' buffer (e.g. pure file_data) with its transformations.
Each unique combination of buffer with transformations gets it's
own buffer id.

Similarly, mpm registration and inspect engine registration will be
copied from the 'parent' (again, e.g. pure file_data) to the new id's.

The transforms are called from within the prefilter engines themselves.

Provide generic MPM matching and setup callbacks. Can be used by
keywords to avoid needless code duplication. Supports transformations.

Use unique name for profiling, to distinguish between pure buffers
and buffers with transformation.

Add new registration calls for mpm/prefilters and inspect engines.

Inspect engine api v2: Pass engine to itself. Add generic engine that
uses GetData callback and other registered settings.

The generic engine should be usable for every 'simple' case where
there is just a single non-streaming buffer. For example HTTP uri.

The v2 API assumes that registered MPM implements transformations.

Add util func to set new transform in rule and add util funcs for rule
parsing.

src/detect-engine-mpm.c

@@ -36,6 +36,7 @@
 #include "detect-engine-mpm.h"
 #include "detect-engine-iponly.h"
 #include "detect-parse.h"
+#include "detect-engine-prefilter.h"
 #include "util-mpm.h"
 #include "util-memcmp.h"
 #include "util-memcpy.h"
@@ -90,10 +91,69 @@ const char *builtin_mpms[] = {
 static DetectMpmAppLayerRegistery *g_app_mpms_list = NULL;
 static int g_app_mpms_list_cnt = 0;
 
+/** \brief register a MPM engine
+ *
+ *  \note to be used at start up / registration only. Errors are fatal.
+ */
+void DetectAppLayerMpmRegister2(const char *name,
+        int direction, int priority,
+        int (*PrefilterRegister)(SigGroupHead *sgh, MpmCtx *mpm_ctx,
+            const DetectMpmAppLayerRegistery *mpm_reg, int list_id),
+        InspectionBufferGetDataPtr GetData,
+        AppProto alproto, int tx_min_progress)
+{
+    SCLogDebug("registering %s/%d/%d/%p/%p/%u/%d", name, direction, priority,
+            PrefilterRegister, GetData, alproto, tx_min_progress);
+
+    if (PrefilterRegister == PrefilterGenericMpmRegister && GetData == NULL) {
+        // must register GetData with PrefilterGenericMpmRegister
+        abort();
+    }
+
+    DetectBufferTypeSupportsMpm(name);
+    DetectBufferTypeSupportsTransformations(name);
+    int sm_list = DetectBufferTypeGetByName(name);
+    if (sm_list == -1) {
+        FatalError(SC_ERR_INITIALIZATION,
+                "MPM engine registration for %s failed", name);
+    }
+
+    DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
+    BUG_ON(am == NULL);
+    am->name = name;
+    snprintf(am->pname, sizeof(am->pname), "%s", am->name);
+    am->direction = direction;
+    am->sm_list = sm_list;
+    am->priority = priority;
+
+    am->v2.PrefilterRegisterWithListId = PrefilterRegister;
+    am->v2.GetData = GetData;
+    am->v2.alproto = alproto;
+    am->v2.tx_min_progress = tx_min_progress;
+
+    if (g_app_mpms_list == NULL) {
+        g_app_mpms_list = am;
+    } else {
+        DetectMpmAppLayerRegistery *t = g_app_mpms_list;
+        while (t->next != NULL) {
+            t = t->next;
+        }
+
+        t->next = am;
+        am->id = t->id + 1;
+    }
+    g_app_mpms_list_cnt++;
+
+    SupportFastPatternForSigMatchList(sm_list, priority);
+}
+
 void DetectAppLayerMpmRegister(const char *name,
         int direction, int priority,
         int (*PrefilterRegister)(SigGroupHead *sgh, MpmCtx *mpm_ctx))
 {
+    SCLogDebug("registering %s/%d/%d/%p",
+            name, direction, priority, PrefilterRegister);
+
     DetectBufferTypeSupportsMpm(name);
     int sm_list = DetectBufferTypeGetByName(name);
     BUG_ON(sm_list == -1);
@@ -101,8 +161,10 @@ void DetectAppLayerMpmRegister(const char *name,
     DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
     BUG_ON(am == NULL);
     am->name = name;
+    snprintf(am->pname, sizeof(am->pname), "%s", am->name);
     am->direction = direction;
     am->sm_list = sm_list;
+    am->priority = priority;
     am->PrefilterRegister = PrefilterRegister;
 
     if (g_app_mpms_list == NULL) {
@@ -121,6 +183,42 @@ void DetectAppLayerMpmRegister(const char *name,
     SupportFastPatternForSigMatchList(sm_list, priority);
 }
 
+void DetectAppLayerMpmRegisterByParentId(const int id, const int parent_id,
+        DetectEngineTransforms *transforms)
+{
+    SCLogDebug("registering %d/%d", id, parent_id);
+
+    DetectMpmAppLayerRegistery *t = g_app_mpms_list;
+    while (t) {
+        if (t->sm_list == parent_id) {
+            DetectMpmAppLayerRegistery *am = SCCalloc(1, sizeof(*am));
+            BUG_ON(am == NULL);
+            am->name = t->name;
+            snprintf(am->pname, sizeof(am->pname), "%s#%d", am->name, id);
+            am->direction = t->direction;
+            am->sm_list = id;           // use new id
+            am->PrefilterRegister = t->PrefilterRegister;
+            am->v2.PrefilterRegisterWithListId = t->v2.PrefilterRegisterWithListId;
+            am->v2.GetData = t->v2.GetData;
+            am->v2.alproto = t->v2.alproto;
+            am->v2.tx_min_progress = t->v2.tx_min_progress;
+            am->priority = t->priority;
+            am->next = t->next;
+            if (transforms) {
+                memcpy(&am->v2.transforms, transforms, sizeof(*transforms));
+            }
+            am->id = g_app_mpms_list_cnt++;
+
+            SupportFastPatternForSigMatchList(am->sm_list, am->priority);
+            t->next = am;
+            SCLogDebug("copied mpm registration for %s id %u with parent %u and GetData %p",
+                    t->name, id, parent_id, am->v2.GetData);
+            t = am;
+        }
+        t = t->next;
+    }
+}
+
 void DetectMpmInitializeAppMpms(DetectEngineCtx *de_ctx)
 {
     BUG_ON(g_app_mpms_list_cnt == 0);
@@ -1285,10 +1383,22 @@ int PatternMatchPrepareGroup(DetectEngineCtx *de_ctx, SigGroupHead *sh)
             if (mpm_store != NULL) {
                 sh->init->app_mpms[a->reg->id] = mpm_store->mpm_ctx;
 
+                SCLogDebug("a->reg->PrefilterRegister %p mpm_store->mpm_ctx %p",
+                        a->reg->PrefilterRegister, mpm_store->mpm_ctx);
+                SCLogDebug("a %p a->reg->name %s a->reg->PrefilterRegisterWithListId %p "
+                        "mpm_store->mpm_ctx %p", a, a->reg->name,
+                        a->reg->v2.PrefilterRegisterWithListId, mpm_store->mpm_ctx);
+
                 /* if we have just certain types of negated patterns,
                  * mpm_ctx can be NULL */
-                if (a->reg->PrefilterRegister && mpm_store->mpm_ctx) {
+                if (a->reg->v2.PrefilterRegisterWithListId && mpm_store->mpm_ctx) {
+                    BUG_ON(a->reg->v2.PrefilterRegisterWithListId(sh, mpm_store->mpm_ctx,
+                                a->reg, a->reg->sm_list) != 0);
+                    SCLogDebug("mpm %s %d set up", a->reg->name, a->reg->sm_list);
+                }
+                else if (a->reg->PrefilterRegister && mpm_store->mpm_ctx) {
                     BUG_ON(a->reg->PrefilterRegister(sh, mpm_store->mpm_ctx) != 0);
+                    SCLogDebug("mpm %s %d set up", a->reg->name, a->reg->sm_list);
                 }
             }
         }

src/detect-engine-mpm.h

@@ -89,6 +89,14 @@ int DetectSetFastPatternAndItsId(DetectEngineCtx *de_ctx);
 void DetectAppLayerMpmRegister(const char *name,
         int direction, int priority,
         int (*PrefilterRegister)(SigGroupHead *sgh, MpmCtx *mpm_ctx));
+void DetectAppLayerMpmRegister2(const char *name,
+        int direction, int priority,
+        int (*PrefilterRegister)(SigGroupHead *sgh, MpmCtx *mpm_ctx,
+            const DetectMpmAppLayerRegistery *mpm_reg, int list_id),
+        InspectionBufferGetDataPtr GetData,
+        AppProto alproto, int tx_min_progress);
+void DetectAppLayerMpmRegisterByParentId(const int id, const int parent_id,
+        DetectEngineTransforms *transforms);
 
 #endif /* __DETECT_ENGINE_MPM_H__ */
 

src/detect-engine-prefilter.c

@@ -575,3 +575,74 @@ const char *PrefilterStoreGetName(const uint32_t id)
     return name;
 }
 #endif
+
+#include "util-print.h"
+
+typedef struct PrefilterMpmCtx {
+    int list_id;
+    InspectionBufferGetDataPtr GetData;
+    const MpmCtx *mpm_ctx;
+    const DetectEngineTransforms *transforms;
+} PrefilterMpmCtx;
+
+/** \brief Generic Mpm prefilter callback
+ *
+ *  \param det_ctx detection engine thread ctx
+ *  \param p packet to inspect
+ *  \param f flow to inspect
+ *  \param txv tx to inspect
+ *  \param pectx inspection context
+ */
+static void PrefilterMpm(DetectEngineThreadCtx *det_ctx,
+        const void *pectx,
+        Packet *p, Flow *f, void *txv,
+        const uint64_t idx, const uint8_t flags)
+{
+    SCEnter();
+
+    const PrefilterMpmCtx *ctx = (const PrefilterMpmCtx *)pectx;
+    const MpmCtx *mpm_ctx = ctx->mpm_ctx;
+    SCLogDebug("running on list %d", ctx->list_id);
+
+    InspectionBuffer *buffer = ctx->GetData(det_ctx, ctx->transforms,
+            f, flags, txv, ctx->list_id);
+    if (buffer == NULL)
+        return;
+
+    const uint32_t data_len = buffer->inspect_len;
+    const uint8_t *data = buffer->inspect;
+
+    SCLogDebug("mpm'ing buffer:");
+    //PrintRawDataFp(stdout, data, data_len);
+
+    if (data != NULL && data_len >= mpm_ctx->minlen) {
+        (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx,
+                &det_ctx->mtcu, &det_ctx->pmq, data, data_len);
+    }
+}
+
+static void PrefilterGenericMpmFree(void *ptr)
+{
+    SCFree(ptr);
+}
+
+int PrefilterGenericMpmRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx,
+        const DetectMpmAppLayerRegistery *mpm_reg, int list_id)
+{
+    SCEnter();
+    PrefilterMpmCtx *pectx = SCCalloc(1, sizeof(*pectx));
+    if (pectx == NULL)
+        return -1;
+    pectx->list_id = list_id;
+    pectx->GetData = mpm_reg->v2.GetData;
+    pectx->mpm_ctx = mpm_ctx;
+    pectx->transforms = &mpm_reg->v2.transforms;
+
+    int r = PrefilterAppendTxEngine(sgh, PrefilterMpm,
+        mpm_reg->v2.alproto, mpm_reg->v2.tx_min_progress,
+        pectx, PrefilterGenericMpmFree, mpm_reg->pname);
+    if (r != 0) {
+        SCFree(pectx);
+    }
+    return r;
+}

src/detect-engine-prefilter.h

@@ -69,4 +69,7 @@ void PrefilterCleanupRuleGroup(SigGroupHead *sgh);
 const char *PrefilterStoreGetName(const uint32_t id);
 #endif
 
+int PrefilterGenericMpmRegister(SigGroupHead *sgh, MpmCtx *mpm_ctx,
+        const DetectMpmAppLayerRegistery *mpm_reg, int list_id);
+
 #endif

src/detect-engine.c

@@ -32,6 +32,7 @@
 #include "conf.h"
 #include "conf-yaml-loader.h"
 
+#include "app-layer-parser.h"
 #include "app-layer-htp.h"
 
 #include "detect-parse.h"
@@ -55,6 +56,7 @@
 #include "detect-content.h"
 #include "detect-uricontent.h"
 #include "detect-engine-threshold.h"
+#include "detect-engine-content-inspection.h"
 
 #include "detect-engine-loader.h"
 
@@ -121,13 +123,19 @@ SCEnumCharMap det_ctx_event_table[ ] = {
     { NULL,                         -1 },
 };
 
+/** \brief register inspect engine at start up time
+ *
+ *  \note errors are fatal */
 void DetectAppLayerInspectEngineRegister(const char *name,
         AppProto alproto, uint32_t dir,
         int progress, InspectEngineFuncPtr Callback)
 {
     DetectBufferTypeRegister(name);
-    int sm_list = DetectBufferTypeGetByName(name);
-    BUG_ON(sm_list == -1);
+    const int sm_list = DetectBufferTypeGetByName(name);
+    if (sm_list == -1) {
+        FatalError(SC_ERR_INITIALIZATION,
+            "failed to register inspect engine %s", name);
+    }
 
     if ((alproto >= ALPROTO_FAILED) ||
         (!(dir == SIG_FLAG_TOSERVER || dir == SIG_FLAG_TOCLIENT)) ||
@@ -169,6 +177,92 @@ void DetectAppLayerInspectEngineRegister(const char *name,
     }
 }
 
+/** \brief register inspect engine at start up time
+ *
+ *  \note errors are fatal */
+void DetectAppLayerInspectEngineRegister2(const char *name,
+        AppProto alproto, uint32_t dir, int progress,
+        InspectEngineFuncPtr2 Callback2,
+        InspectionBufferGetDataPtr GetData)
+{
+    DetectBufferTypeRegister(name);
+    const int sm_list = DetectBufferTypeGetByName(name);
+    if (sm_list == -1) {
+        FatalError(SC_ERR_INITIALIZATION,
+            "failed to register inspect engine %s", name);
+    }
+
+    if ((alproto >= ALPROTO_FAILED) ||
+        (!(dir == SIG_FLAG_TOSERVER || dir == SIG_FLAG_TOCLIENT)) ||
+        (sm_list < DETECT_SM_LIST_MATCH) || (sm_list >= SHRT_MAX) ||
+        (progress < 0 || progress >= SHRT_MAX) ||
+        (Callback2 == NULL))
+    {
+        SCLogError(SC_ERR_INVALID_ARGUMENTS, "Invalid arguments");
+        BUG_ON(1);
+    } else if (Callback2 == DetectEngineInspectBufferGeneric && GetData == NULL) {
+        SCLogError(SC_ERR_INVALID_ARGUMENTS, "Invalid arguments: must register "
+                "GetData with DetectEngineInspectBufferGeneric");
+        BUG_ON(1);
+    }
+
+    int direction;
+    if (dir == SIG_FLAG_TOSERVER) {
+        direction = 0;
+    } else {
+        direction = 1;
+    }
+
+    DetectEngineAppInspectionEngine *new_engine = SCMalloc(sizeof(DetectEngineAppInspectionEngine));
+    if (unlikely(new_engine == NULL)) {
+        exit(EXIT_FAILURE);
+    }
+    memset(new_engine, 0, sizeof(*new_engine));
+    new_engine->alproto = alproto;
+    new_engine->dir = direction;
+    new_engine->sm_list = sm_list;
+    new_engine->progress = progress;
+    new_engine->v2.Callback = Callback2;
+    new_engine->v2.GetData = GetData;
+
+    if (g_app_inspect_engines == NULL) {
+        g_app_inspect_engines = new_engine;
+    } else {
+        DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
+        while (t->next != NULL) {
+            t = t->next;
+        }
+
+        t->next = new_engine;
+    }
+}
+
+/* copy an inspect engine to a new list id. */
+static void DetectAppLayerInspectEngineCopy(int sm_list, int new_list,
+        const DetectEngineTransforms *transforms)
+{
+    DetectEngineAppInspectionEngine *t = g_app_inspect_engines;
+    while (t) {
+        if (t->sm_list == sm_list) {
+            DetectEngineAppInspectionEngine *new_engine = SCCalloc(1, sizeof(DetectEngineAppInspectionEngine));
+            if (unlikely(new_engine == NULL)) {
+                exit(EXIT_FAILURE);
+            }
+            new_engine->alproto = t->alproto;
+            new_engine->dir = t->dir;
+            new_engine->sm_list = new_list;
+            new_engine->progress = t->progress;
+            new_engine->Callback = t->Callback;
+            new_engine->v2 = t->v2;
+            new_engine->v2.transforms = transforms;
+            new_engine->next = t->next;
+            t->next = new_engine;
+            t = new_engine;
+        }
+        t = t->next;
+    }
+}
+
 /** \internal
  *  \brief append the stream inspection
  *
@@ -239,6 +333,7 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
             continue;
 
         ptrs[i] = SigMatchList2DataArray(s->init_data->smlists[i]);
+        SCLogDebug("ptrs[%d] is set", i);
     }
 
     bool head_is_mpm = false;
@@ -249,6 +344,9 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
 
         if (ptrs[t->sm_list] == NULL)
             goto next;
+
+        SCLogDebug("ptrs[%d] is set", t->sm_list);
+
         if (t->alproto == ALPROTO_UNKNOWN) {
             /* special case, inspect engine applies to all protocols */
         } else if (s->alproto != ALPROTO_UNKNOWN && s->alproto != t->alproto)
@@ -278,6 +376,10 @@ int DetectEngineAppInspectionEngine2Signature(Signature *s)
         new_engine->smd = ptrs[new_engine->sm_list];
         new_engine->Callback = t->Callback;
         new_engine->progress = t->progress;
+        new_engine->v2 = t->v2;
+        SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p",
+                new_engine->sm_list, new_engine->v2.Callback,
+                new_engine->v2.GetData, new_engine->v2.transforms);
 
         if (s->app_inspect == NULL) {
             s->app_inspect = new_engine;
@@ -406,17 +508,23 @@ static HashListTable *g_buffer_type_hash = NULL;
 static int g_buffer_type_id = DETECT_SM_LIST_DYNAMIC_START;
 static int g_buffer_type_reg_closed = 0;
 
+static DetectEngineTransforms no_transforms = { .transforms = { 0 }, .cnt = 0, };
+
 typedef struct DetectBufferType_ {
     const char *string;
     const char *description;
     int id;
+    int parent_id;
     _Bool mpm;
     _Bool packet; /**< compat to packet matches */
+    bool supports_transforms;
     void (*SetupCallback)(Signature *);
     _Bool (*ValidateCallback)(const Signature *, const char **sigerror);
+    DetectEngineTransforms transforms;
 } DetectBufferType;
 
 static DetectBufferType **g_buffer_type_map = NULL;
+static uint32_t g_buffer_type_map_elements = 0;
 
 int DetectBufferTypeMaxId(void)
 {
@@ -429,6 +537,7 @@ static uint32_t DetectBufferTypeHashFunc(HashListTable *ht, void *data, uint16_t
     uint32_t hash = 0;
 
     hash = hashlittle_safe(map->string, strlen(map->string), 0);
+    hash += hashlittle_safe((uint8_t *)&map->transforms, sizeof(map->transforms), 0);
     hash %= ht->array_size;
 
     return hash;
@@ -441,6 +550,7 @@ static char DetectBufferTypeCompareFunc(void *data1, uint16_t len1, void *data2,
     DetectBufferType *map2 = (DetectBufferType *)data2;
 
     int r = (strcmp(map1->string, map2->string) == 0);
+    r &= (memcmp((uint8_t *)&map1->transforms, (uint8_t *)&map2->transforms, sizeof(map2->transforms)) == 0);
     return r;
 }
 
@@ -491,7 +601,7 @@ static int DetectBufferTypeAdd(const char *string)
 
 static DetectBufferType *DetectBufferTypeLookupByName(const char *string)
 {
-    DetectBufferType map = { (char *)string, NULL, 0, 0, 0, NULL, NULL };
+    DetectBufferType map = { (char *)string, NULL, 0, 0, 0, 0, false, NULL, NULL, no_transforms };
 
     DetectBufferType *res = HashListTableLookup(g_buffer_type_hash, &map, 0);
     return res;
@@ -531,6 +641,16 @@ void DetectBufferTypeSupportsMpm(const char *name)
     SCLogDebug("%p %s -- %d supports mpm", exists, name, exists->id);
 }
 
+void DetectBufferTypeSupportsTransformations(const char *name)
+{
+    BUG_ON(g_buffer_type_reg_closed);
+    DetectBufferTypeRegister(name);
+    DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    BUG_ON(!exists);
+    exists->supports_transforms = true;
+    SCLogDebug("%p %s -- %d supports transformations", exists, name, exists->id);
+}
+
 int DetectBufferTypeGetByName(const char *name)
 {
     DetectBufferType *exists = DetectBufferTypeLookupByName(name);
@@ -641,6 +761,114 @@ _Bool DetectBufferRunValidateCallback(const int id, const Signature *s, const ch
     return TRUE;
 }
 
+int DetectBufferSetActiveList(Signature *s, const int list)
+{
+    BUG_ON(s->init_data == NULL);
+
+    if (s->init_data->list && s->init_data->transform_cnt) {
+        return -1;
+    }
+    s->init_data->list = list;
+    s->init_data->list_set = true;
+
+    return 0;
+}
+
+int DetectBufferGetActiveList(Signature *s)
+{
+    BUG_ON(s->init_data == NULL);
+
+    if (s->init_data->list && s->init_data->transform_cnt) {
+        SCLogDebug("buffer %d has transform(s) registered: %d",
+                s->init_data->list, s->init_data->transforms[0]);
+        int new_list = DetectBufferTypeGetByIdTransforms(s->init_data->list,
+                s->init_data->transforms, s->init_data->transform_cnt);
+        if (new_list == -1) {
+            return -1;
+        }
+        SCLogDebug("new_list %d", new_list);
+        s->init_data->list = new_list;
+        s->init_data->list_set = false;
+        // reset transforms now that we've set up the list
+        s->init_data->transform_cnt = 0;
+    }
+
+    return 0;
+}
+
+void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size)
+{
+    memset(buffer, 0, sizeof(*buffer));
+    buffer->buf = SCCalloc(initial_size, sizeof(uint8_t));
+    if (buffer->buf != NULL) {
+        buffer->size = initial_size;
+    }
+}
+
+/** \brief setup the buffer with our initial data */
+void InspectionBufferSetup(InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len)
+{
+    buffer->inspect = buffer->orig = data;
+    buffer->inspect_len = buffer->orig_len = data_len;
+    buffer->len = 0;
+}
+
+void InspectionBufferFree(InspectionBuffer *buffer)
+{
+    if (buffer->buf != NULL) {
+        SCFree(buffer->buf);
+    }
+    memset(buffer, 0, sizeof(*buffer));
+}
+
+/**
+ * \brief make sure that the buffer has at least 'min_size' bytes
+ * Expand the buffer if necessary
+ */
+void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size)
+{
+    if (likely(buffer->size >= min_size))
+        return;
+
+    uint32_t new_size = (buffer->size == 0) ? 4096 : buffer->size;
+    while (new_size < min_size) {
+        new_size *= 2;
+    }
+
+    void *ptr = SCRealloc(buffer->buf, new_size);
+    if (ptr != NULL) {
+        buffer->buf = ptr;
+        buffer->size = new_size;
+    }
+}
+
+void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len)
+{
+    InspectionBufferCheckAndExpand(buffer, buf_len);
+
+    if (buffer->size) {
+        uint32_t copy_size = MIN(buf_len, buffer->size);
+        memcpy(buffer->buf, buf, copy_size);
+        buffer->inspect = buffer->buf;
+        buffer->inspect_len = copy_size;
+    }
+}
+
+void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
+        const DetectEngineTransforms *transforms)
+{
+    if (transforms) {
+        for (int i = 0; i < DETECT_TRANSFORMS_MAX; i++) {
+            const int id = transforms->transforms[i];
+            if (id == 0)
+                break;
+            BUG_ON(sigmatch_table[id].Transform == NULL);
+            sigmatch_table[id].Transform(buffer);
+            SCLogDebug("applied transform %s", sigmatch_table[id].name);
+        }
+    }
+}
+
 void DetectBufferTypeFinalizeRegistration(void)
 {
     BUG_ON(g_buffer_type_hash == NULL);
@@ -650,6 +878,8 @@ void DetectBufferTypeFinalizeRegistration(void)
 
     g_buffer_type_map = SCCalloc(size, sizeof(DetectBufferType *));
     BUG_ON(!g_buffer_type_map);
+    g_buffer_type_map_elements = size;
+    SCLogDebug("g_buffer_type_map %p with %u members", g_buffer_type_map, size);
 
     SCLogDebug("DETECT_SM_LIST_DYNAMIC_START %u", DETECT_SM_LIST_DYNAMIC_START);
     HashListTableBucket *b = HashListTableGetListHead(g_buffer_type_hash);
@@ -665,6 +895,62 @@ void DetectBufferTypeFinalizeRegistration(void)
     g_buffer_type_reg_closed = 1;
 }
 
+int DetectBufferTypeGetByIdTransforms(const int id, int *transforms, int transform_cnt)
+{
+    const DetectBufferType *base_map = DetectBufferTypeGetById(id);
+    if (!base_map) {
+        return -1;
+    }
+    if (!base_map->supports_transforms) {
+        SCLogError(SC_ERR_INVALID_SIGNATURE, "buffer '%s' does not support transformations",
+                base_map->string);
+        return -1;
+    }
+
+    DetectEngineTransforms t;
+    memset(&t, 0, sizeof(t));
+    for (int i = 0; i < transform_cnt; i++) {
+        t.transforms[i] = transforms[i];
+    }
+    t.cnt = transform_cnt;
+
+    DetectBufferType lookup_map = { (char *)base_map->string, NULL, 0, 0, 0, 0, false, NULL, NULL, t };
+    DetectBufferType *res = HashListTableLookup(g_buffer_type_hash, &lookup_map, 0);
+
+    SCLogDebug("res %p", res);
+    if (res != NULL) {
+        return res->id;
+    }
+
+    DetectBufferType *map = SCCalloc(1, sizeof(*map));
+    if (map == NULL)
+        return -1;
+
+    map->string = base_map->string;
+    map->id = g_buffer_type_id++;
+    map->parent_id = base_map->id;
+    map->transforms = t;
+    map->mpm = base_map->mpm;
+    map->SetupCallback = base_map->SetupCallback;
+    map->ValidateCallback = base_map->ValidateCallback;
+    DetectAppLayerMpmRegisterByParentId(map->id, map->parent_id, &map->transforms);
+
+    BUG_ON(HashListTableAdd(g_buffer_type_hash, (void *)map, 0) != 0);
+    SCLogDebug("buffer %s registered with id %d, parent %d", map->string, map->id, map->parent_id);
+
+    if (map->id >= 0 && (uint32_t)map->id >= g_buffer_type_map_elements) {
+        void *ptr = SCRealloc(g_buffer_type_map, (map->id + 1) * sizeof(DetectBufferType *));
+        BUG_ON(ptr == NULL);
+        SCLogDebug("g_buffer_type_map resized to %u (was %u)", (map->id + 1), g_buffer_type_map_elements);
+        g_buffer_type_map = ptr;
+        g_buffer_type_map[map->id] = map;
+        g_buffer_type_map_elements = map->id + 1;
+
+        DetectAppLayerInspectEngineCopy(map->parent_id, map->id, &map->transforms);
+    }
+    return map->id;
+}
+
 /* code to control the main thread to do a reload */
 
 enum DetectEngineSyncState {
@@ -773,6 +1059,74 @@ int DetectEngineInspectGenericList(ThreadVars *tv,
     return DETECT_ENGINE_INSPECT_SIG_MATCH;
 }
 
+
+/**
+ * \brief Do the content inspection & validation for a signature
+ *
+ * \param de_ctx Detection engine context
+ * \param det_ctx Detection engine thread context
+ * \param s Signature to inspect
+ * \param f Flow
+ * \param flags app layer flags
+ * \param state App layer state
+ *
+ * \retval 0 no match.
+ * \retval 1 match.
+ * \retval 2 Sig can't match.
+ */
+int DetectEngineInspectBufferGeneric(
+        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
+        const DetectEngineAppInspectionEngine *engine,
+        const Signature *s,
+        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id)
+{
+    const int list_id = engine->sm_list;
+    SCLogDebug("running inspect on %d", list_id);
+
+    SCLogDebug("list %d mpm? %s transforms %p",
+            engine->sm_list, engine->mpm ? "true" : "false", engine->v2.transforms);
+
+    /* if prefilter didn't already run, we need to consider transformations */
+    const DetectEngineTransforms *transforms = NULL;
+    if (!engine->mpm) {
+        transforms = engine->v2.transforms;
+    }
+
+    const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms,
+            f, flags, txv, list_id);
+    if (buffer == NULL) {
+        if (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress)
+            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
+        else
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    }
+
+    const uint32_t data_len = buffer->inspect_len;
+    const uint8_t *data = buffer->inspect;
+    const uint64_t offset = buffer->inspect_offset;
+
+    det_ctx->discontinue_matching = 0;
+    det_ctx->buffer_offset = 0;
+    det_ctx->inspection_recursion_counter = 0;
+
+    /* Inspect all the uricontents fetched on each
+     * transaction at the app layer */
+    int r = DetectEngineContentInspection(de_ctx, det_ctx,
+                                          s, engine->smd,
+                                          f,
+                                          (uint8_t *)data, data_len, offset,
+                                          DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE, NULL);
+    if (r == 1) {
+        return DETECT_ENGINE_INSPECT_SIG_MATCH;
+    } else {
+        if (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > engine->progress)
+            return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH;
+        else
+            return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+    }
+}
+
+
 /* nudge capture loops to wake up */
 static void BreakCapture(void)
 {
@@ -1804,6 +2158,17 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx *
         det_ctx->base64_decoded_len = 0;
     }
 
+    det_ctx->inspect_buffers_size = (uint32_t)DetectBufferTypeMaxId();
+    det_ctx->inspect_buffers = SCCalloc(det_ctx->inspect_buffers_size, sizeof(InspectionBuffer));
+    if (det_ctx->inspect_buffers == NULL) {
+        return TM_ECODE_FAILED;
+    }
+    det_ctx->multi_inspect_buffers_size = (uint32_t)DetectBufferTypeMaxId();
+    det_ctx->multi_inspect_buffers = SCCalloc(det_ctx->multi_inspect_buffers_size, sizeof(InspectionBufferMultipleForList));
+    if (det_ctx->multi_inspect_buffers == NULL) {
+        return TM_ECODE_FAILED;
+    }
+
     DetectEngineThreadCtxInitKeywords(de_ctx, det_ctx);
     DetectEngineThreadCtxInitGlobalKeywords(det_ctx);
 #ifdef PROFILING
@@ -2014,17 +2379,29 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx)
         SCFree(det_ctx->hcbd);
     }
 
-    /* file data */
-    if (det_ctx->file_data != NULL) {
-        SCLogDebug("det_ctx file_data %u", det_ctx->file_data_buffers_size);
-        SCFree(det_ctx->file_data);
-    }
-
     /* Decoded base64 data. */
     if (det_ctx->base64_decoded != NULL) {
         SCFree(det_ctx->base64_decoded);
     }
 
+    if (det_ctx->inspect_buffers) {
+        for (uint32_t i = 0; i < det_ctx->inspect_buffers_size; i++) {
+            InspectionBufferFree(&det_ctx->inspect_buffers[i]);
+        }
+        SCFree(det_ctx->inspect_buffers);
+    }
+    if (det_ctx->multi_inspect_buffers) {
+        for (uint32_t i = 0; i < det_ctx->multi_inspect_buffers_size; i++) {
+            InspectionBufferMultipleForList *fb = &det_ctx->multi_inspect_buffers[i];
+            for (uint32_t x = 0; x < fb->size; x++) {
+                InspectionBufferFree(&fb->inspection_buffers[x]);
+            }
+            SCFree(fb->inspection_buffers);
+        }
+        SCFree(det_ctx->multi_inspect_buffers);
+    }
+
+
     DetectEngineThreadCtxDeinitGlobalKeywords(det_ctx);
     if (det_ctx->de_ctx != NULL) {
         DetectEngineThreadCtxDeinitKeywords(det_ctx->de_ctx, det_ctx);

src/detect-engine.h

@@ -28,11 +28,21 @@
 #include "tm-threads.h"
 #include "flow-private.h"
 
+void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size);
+void InspectionBufferSetup(InspectionBuffer *buffer, const uint8_t *data, const uint32_t data_len);
+void InspectionBufferFree(InspectionBuffer *buffer);
+void InspectionBufferCheckAndExpand(InspectionBuffer *buffer, uint32_t min_size);
+void InspectionBufferCopy(InspectionBuffer *buffer, uint8_t *buf, uint32_t buf_len);
+void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
+        const DetectEngineTransforms *transforms);
+
 int DetectBufferTypeRegister(const char *name);
 int DetectBufferTypeGetByName(const char *name);
+int DetectBufferTypeGetByIdTransforms(const int id, int *transforms, int transform_cnt);
 const char *DetectBufferTypeGetNameById(const int id);
 void DetectBufferTypeSupportsMpm(const char *name);
 void DetectBufferTypeSupportsPacket(const char *name);
+void DetectBufferTypeSupportsTransformations(const char *name);
 _Bool DetectBufferTypeSupportsMpmGetById(const int id);
 _Bool DetectBufferTypeSupportsPacketGetById(const int id);
 int DetectBufferTypeMaxId(void);
@@ -100,6 +110,12 @@ int DetectEngineInspectGenericList(ThreadVars *, const DetectEngineCtx *,
                                    Flow *, const uint8_t, void *, void *,
                                    uint64_t);
 
+int DetectEngineInspectBufferGeneric(
+        DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx,
+        const DetectEngineAppInspectionEngine *engine,
+        const Signature *s,
+        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
+
 /**
  * \brief Registers an app inspection engine.
  *
@@ -113,6 +129,10 @@ int DetectEngineInspectGenericList(ThreadVars *, const DetectEngineCtx *,
 void DetectAppLayerInspectEngineRegister(const char *name,
         AppProto alproto, uint32_t dir,
         int progress, InspectEngineFuncPtr Callback);
+void DetectAppLayerInspectEngineRegister2(const char *name,
+        AppProto alproto, uint32_t dir, int progress,
+        InspectEngineFuncPtr2 Callback2,
+        InspectionBufferGetDataPtr GetData);
 
 int DetectEngineAppInspectionEngine2Signature(Signature *s);
 void DetectEngineAppInspectionEngineSignatureFree(Signature *s);
@@ -121,4 +141,7 @@ void DetectEngineSetParseMetadata(void);
 void DetectEngineUnsetParseMetadata(void);
 int DetectEngineMustParseMetadata(void);
 
+int DetectBufferSetActiveList(Signature *s, const int list);
+int DetectBufferGetActiveList(Signature *s);
+
 #endif /* __DETECT_ENGINE_H__ */

src/detect-parse.c

@@ -1355,6 +1355,23 @@ void SigFree(Signature *s)
     SCFree(s);
 }
 
+int DetectSignatureAddTransform(Signature *s, int transform)
+{
+    /* we only support buffers */
+    if (s->init_data->list == 0) {
+        SCReturnInt(-1);
+    }
+    if (!s->init_data->list_set) {
+        SCLogError(SC_ERR_INVALID_SIGNATURE, "transforms must directly follow stickybuffers");
+        SCReturnInt(-1);
+    }
+    if (s->init_data->transform_cnt >= DETECT_TRANSFORMS_MAX) {
+        SCReturnInt(-1);
+    }
+    s->init_data->transforms[s->init_data->transform_cnt++] = transform;
+    SCReturnInt(0);
+}
+
 int DetectSignatureSetAppProto(Signature *s, AppProto alproto)
 {
     if (alproto == ALPROTO_UNKNOWN ||

src/detect-parse.h

@@ -69,6 +69,7 @@ SigMatch *DetectGetLastSMFromLists(const Signature *s, ...);
 SigMatch *DetectGetLastSMByListPtr(const Signature *s, SigMatch *sm_list, ...);
 SigMatch *DetectGetLastSMByListId(const Signature *s, int list_id, ...);
 
+int DetectSignatureAddTransform(Signature *s, int transform);
 int DetectSignatureSetAppProto(Signature *s, AppProto alproto);
 
 /* parse regex setup and free util funcs */

src/detect.c

@@ -1033,6 +1033,13 @@ static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx,
     PacketPatternCleanup(det_ctx);
 
     if (pflow != NULL) {
+        // TODO clean this up
+        const int nlists = DetectBufferTypeMaxId();
+        for (int i = 0; i < nlists; i++) {
+            InspectionBuffer *buffer = &det_ctx->inspect_buffers[i];
+            buffer->inspect = NULL;
+        }
+
         /* update inspected tracker for raw reassembly */
         if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL) {
             StreamReassembleRawUpdateProgress(pflow->protoctx, p,
@@ -1206,8 +1213,14 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
                 continue;
             } else {
                 KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list);
-                match = engine->Callback(tv, de_ctx, det_ctx,
-                        s, engine->smd, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
+                if (engine->Callback) {
+                    match = engine->Callback(tv, de_ctx, det_ctx,
+                            s, engine->smd, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
+                } else {
+                    BUG_ON(engine->v2.Callback == NULL);
+                    match = engine->v2.Callback(de_ctx, det_ctx, engine,
+                            s, f, flow_flags, alstate, tx->tx_ptr, tx->tx_id);
+                }
                 TRACE_SID_TXS(s->id, tx, "engine %p match %d", engine, match);
                 if (engine->stream) {
                     can->stream_stored = true;

src/detect.h

@@ -54,6 +54,8 @@
 
 #define DETECT_MAX_RULE_SIZE 8192
 
+#define DETECT_TRANSFORMS_MAX 16
+
 /* forward declarations for the structures from detect-engine-sigorder.h */
 struct SCSigOrderFunc_;
 struct SCSigSignatureWrapper_;
@@ -256,6 +258,7 @@ typedef struct DetectPort_ {
 #define SIG_FLAG_INIT_FLOW           (1<<2)  /**< signature has a flow setting */
 #define SIG_FLAG_INIT_BIDIREC        (1<<3)  /**< signature has bidirectional operator */
 #define SIG_FLAG_INIT_FIRST_IPPROTO_SEEN (1 << 4) /** < signature has seen the first ip_proto keyword */
+#define SIG_FLAG_INIT_HAS_TRANSFORM (1<<5)
 
 /* signature mask flags */
 #define SIG_MASK_REQUIRE_PAYLOAD            (1<<0)
@@ -324,12 +327,63 @@ typedef struct SigMatchData_ {
 
 struct DetectEngineThreadCtx_;// DetectEngineThreadCtx;
 
+/* inspection buffer is a simple structure that is passed between prefilter,
+ * transformation functions and inspection functions.
+ * Initialy setup with 'orig' ptr and len, transformations can then take
+ * then and fill the 'buf'. Multiple transformations can update the buffer,
+ * both growing and shrinking it.
+ * Prefilter and inspection will only deal with 'inspect'. */
+
+typedef struct InspectionBuffer {
+    const uint8_t *inspect;     /**< active pointer, points either to ::buf or ::orig */
+    uint32_t inspect_len; /**< size of active data. See to ::len or ::orig_len */
+    uint64_t inspect_offset;
+
+    uint8_t *buf;
+    uint32_t len;   /**< how much is in use */
+    uint32_t size;  /**< size of the memory allocation */
+
+    const uint8_t *orig;
+    uint32_t orig_len;
+} InspectionBuffer;
+
+/* inspection buffers are kept per tx (in det_ctx), but some protocols
+ * need a bit more. A single TX might have multiple buffers, e.g. files in
+ * SMTP or DNS queries. Since all prefilters+transforms run before the
+ * individual rules need the same buffers, we need a place to store the
+ * transformed data. This array of arrays is that place. */
+
+typedef struct InspectionBufferMultipleForList {
+    InspectionBuffer *inspection_buffers;
+    uint32_t size;  /**< size in number of elements */
+} InspectionBufferMultipleForList;
+
+typedef struct DetectEngineTransforms {
+    int transforms[DETECT_TRANSFORMS_MAX];
+    int cnt;
+} DetectEngineTransforms;
+
+/** callback for getting the buffer we need to prefilter/inspect */
+typedef InspectionBuffer *(*InspectionBufferGetDataPtr)(
+        struct DetectEngineThreadCtx_ *det_ctx,
+        const DetectEngineTransforms *transforms,
+        Flow *f, const uint8_t flow_flags,
+        void *txv, const int list_id);
+
 typedef int (*InspectEngineFuncPtr)(ThreadVars *tv,
         struct DetectEngineCtx_ *de_ctx, struct DetectEngineThreadCtx_ *det_ctx,
         const struct Signature_ *sig, const SigMatchData *smd,
         Flow *f, uint8_t flags, void *alstate,
         void *tx, uint64_t tx_id);
 
+struct DetectEngineAppInspectionEngine_;
+
+typedef int (*InspectEngineFuncPtr2)(
+        struct DetectEngineCtx_ *de_ctx, struct DetectEngineThreadCtx_ *det_ctx,
+        const struct DetectEngineAppInspectionEngine_ *engine,
+        const struct Signature_ *s,
+        Flow *f, uint8_t flags, void *alstate, void *txv, uint64_t tx_id);
+
 typedef struct DetectEngineAppInspectionEngine_ {
     AppProto alproto;
     uint8_t dir;
@@ -347,6 +401,13 @@ typedef struct DetectEngineAppInspectionEngine_ {
      */
     InspectEngineFuncPtr Callback;
 
+    struct {
+        InspectionBufferGetDataPtr GetData;
+        InspectEngineFuncPtr2 Callback;
+        /** pointer to the transforms in the 'DetectBuffer entry for this list */
+        const DetectEngineTransforms *transforms;
+    } v2;
+
     SigMatchData *smd;
 
     struct DetectEngineAppInspectionEngine_ *next;
@@ -379,6 +440,10 @@ typedef struct SignatureInitData_ {
 
     /* SigMatch list used for adding content and friends. E.g. file_data; */
     int list;
+    bool list_set;
+
+    int transforms[DETECT_TRANSFORMS_MAX];
+    int transform_cnt;
 
     /** score to influence rule grouping. A higher value leads to a higher
      *  likelyhood of a rulegroup with this sig ending up as a contained
@@ -475,12 +540,25 @@ typedef struct Signature_ {
 /** \brief one time registration of keywords at start up */
 typedef struct DetectMpmAppLayerRegistery_ {
     const char *name;
+    char pname[32];             /**< name used in profiling */
     int direction;              /**< SIG_FLAG_TOSERVER or SIG_FLAG_TOCLIENT */
     int sm_list;
 
     int (*PrefilterRegister)(struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx);
 
+    int priority;
+
+    struct {
+        int (*PrefilterRegisterWithListId)(struct SigGroupHead_ *sgh, MpmCtx *mpm_ctx,
+                const struct DetectMpmAppLayerRegistery_ *mpm_reg, int list_id);
+        InspectionBufferGetDataPtr GetData;
+        AppProto alproto;
+        int tx_min_progress;
+        DetectEngineTransforms transforms;
+    } v2;
+
     int id;                     /**< index into this array and result arrays */
+
     struct DetectMpmAppLayerRegistery_ *next;
 } DetectMpmAppLayerRegistery;
 
@@ -891,6 +969,16 @@ typedef struct DetectEngineThreadCtx_ {
     uint16_t counter_match_list;
 #endif
 
+    int inspect_list; /**< list we're currently inspecting, DETECT_SM_LIST_* */
+    InspectionBuffer *inspect_buffers;
+
+    uint32_t inspect_buffers_size;          /**< in number of elements */
+    uint32_t multi_inspect_buffers_size;    /**< in number of elements */
+
+    /* inspection buffers for more complete case. As we can inspect multiple
+     * buffers in parallel, we need this extra wrapper struct */
+    InspectionBufferMultipleForList *multi_inspect_buffers;
+
     /* used to discontinue any more matching */
     uint16_t discontinue_matching;
     uint16_t flags;
@@ -1008,6 +1096,9 @@ typedef struct SigTableElmt_ {
         Flow *,                     /**< *LOCKED* flow */
         uint8_t flags, File *, const Signature *, const SigMatchCtx *);
 
+    /** InspectionBuffer transformation callback */
+    void (*Transform)(InspectionBuffer *);
+
     /** keyword setup function pointer */
     int (*Setup)(DetectEngineCtx *, Signature *, const char *);