detect: buffer type API

To replace the hardcoded SigMatch list id's, use this API to register
and query lists by name.

Also allow for registering descriptions and whether mpm is supported.

Registration is only allowed at startup.

src/detect-engine-analyzer.c

@@ -134,6 +134,13 @@ void EngineAnalysisFP(Signature *s, char *line)
         fprintf(fp_engine_analysis_FD, "http stat msg content\n");
     else if (list_type == DETECT_SM_LIST_HUADMATCH)
         fprintf(fp_engine_analysis_FD, "http user agent content\n");
+    else {
+        const char *desc = DetectBufferTypeGetDescriptionById(list_type);
+        const char *name = DetectBufferTypeGetNameById(list_type);
+        if (desc && name) {
+            fprintf(fp_engine_analysis_FD, "%s (%s)\n", desc, name);
+        }
+    }
 
     int flags_set = 0;
     fprintf(fp_engine_analysis_FD, "        Flags:");
@@ -493,6 +500,13 @@ static void EngineAnalysisRulesPrintFP(const Signature *s)
         fprintf(rule_engine_analysis_FD, "tls subject content");
     else if (list_type == DETECT_SM_LIST_DNP3_DATA_MATCH)
         fprintf(rule_engine_analysis_FD, "dnp3 data content");
+    else {
+        const char *desc = DetectBufferTypeGetDescriptionById(list_type);
+        const char *name = DetectBufferTypeGetNameById(list_type);
+        if (desc && name) {
+            fprintf(rule_engine_analysis_FD, "%s (%s)", desc, name);
+        }
+    }
 
     fprintf(rule_engine_analysis_FD, "\" buffer.\n");
 

src/detect-engine.c

@@ -247,6 +247,272 @@ void DetectEngineAppInspectionEngineSignatureFree(Signature *s)
     }
 }
 
+/* code for registering buffers */
+
+#include "util-hash-lookup3.h"
+
+static HashListTable *g_buffer_type_hash = NULL;
+static int g_buffer_type_id = DETECT_SM_LIST_MAX; // past DETECT_SM_LIST_NOTSET
+static int g_buffer_type_reg_closed = 0;
+
+typedef struct DetectBufferType_ {
+    const char *string;
+    const char *description;
+    int id;
+    _Bool mpm;
+    _Bool packet; /**< compat to packet matches */
+    void (*SetupCallback)(Signature *);
+    _Bool (*ValidateCallback)(const Signature *);
+} DetectBufferType;
+
+static DetectBufferType **g_buffer_type_map = NULL;
+
+int DetectBufferTypeMaxId(void)
+{
+    return g_buffer_type_id;
+}
+
+static uint32_t DetectBufferTypeHashFunc(HashListTable *ht, void *data, uint16_t datalen)
+{
+    const DetectBufferType *map = (DetectBufferType *)data;
+    uint32_t hash = 0;
+
+    hash = hashlittle_safe(map->string, strlen(map->string), 0);
+    hash %= ht->array_size;
+
+    return hash;
+}
+
+static char DetectBufferTypeCompareFunc(void *data1, uint16_t len1, void *data2,
+                                        uint16_t len2)
+{
+    DetectBufferType *map1 = (DetectBufferType *)data1;
+    DetectBufferType *map2 = (DetectBufferType *)data2;
+
+    int r = (strcmp(map1->string, map2->string) == 0);
+    return r;
+}
+
+static void DetectBufferTypeFreeFunc(void *data)
+{
+    DetectBufferType *map = (DetectBufferType *)data;
+    if (map != NULL) {
+        SCFree(map);
+    }
+}
+
+int DetectBufferTypeInit(void)
+{
+    BUG_ON(g_buffer_type_hash);
+    g_buffer_type_hash = HashListTableInit(256,
+            DetectBufferTypeHashFunc,
+            DetectBufferTypeCompareFunc,
+            DetectBufferTypeFreeFunc);
+    if (g_buffer_type_hash == NULL)
+        return -1;
+
+    return 0;
+}
+
+void DetectBufferTypeFree(void)
+{
+    if (g_buffer_type_hash == NULL)
+        return;
+
+    HashListTableFree(g_buffer_type_hash);
+    g_buffer_type_hash = NULL;
+    return;
+}
+
+int DetectBufferTypeAdd(const char *string)
+{
+    DetectBufferType *map = SCCalloc(1, sizeof(*map));
+    if (map == NULL)
+        return -1;
+
+    map->string = string;
+    map->id = g_buffer_type_id++;
+
+    BUG_ON(HashListTableAdd(g_buffer_type_hash, (void *)map, 0) != 0);
+    SCLogDebug("buffer %s registered with id %d", map->string, map->id);
+    return map->id;
+}
+
+DetectBufferType *DetectBufferTypeLookupByName(const char *string)
+{
+    DetectBufferType map = { (char *)string, NULL, 0, 0, 0, NULL, NULL };
+
+    DetectBufferType *res = HashListTableLookup(g_buffer_type_hash, &map, 0);
+    return res;
+}
+
+int DetectBufferTypeRegister(const char *name)
+{
+    BUG_ON(g_buffer_type_reg_closed);
+    if (g_buffer_type_hash == NULL)
+        DetectBufferTypeInit();
+
+    DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    if (!exists) {
+        return DetectBufferTypeAdd(name);
+    } else {
+        return exists->id;
+    }
+}
+
+void DetectBufferTypeSupportsPacket(const char *name)
+{
+    BUG_ON(g_buffer_type_reg_closed);
+    DetectBufferTypeRegister(name);
+    DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    BUG_ON(!exists);
+    exists->packet = TRUE;
+    SCLogDebug("%p %s -- %d supports packet inspection", exists, name, exists->id);
+}
+
+void DetectBufferTypeSupportsMpm(const char *name)
+{
+    BUG_ON(g_buffer_type_reg_closed);
+    DetectBufferTypeRegister(name);
+    DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    BUG_ON(!exists);
+    exists->mpm = TRUE;
+    SCLogDebug("%p %s -- %d supports mpm", exists, name, exists->id);
+}
+
+int DetectBufferTypeGetByName(const char *name)
+{
+    DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    if (!exists) {
+        return -1;
+    }
+    return exists->id;
+}
+
+const char *DetectBufferTypeGetNameById(const int id)
+{
+    BUG_ON(id < 0 || id >= g_buffer_type_id);
+    BUG_ON(g_buffer_type_map == NULL);
+
+    if (g_buffer_type_map[id] == NULL)
+        return NULL;
+
+    return g_buffer_type_map[id]->string;
+}
+
+const DetectBufferType *DetectBufferTypeGetById(const int id)
+{
+    BUG_ON(id < 0 || id >= g_buffer_type_id);
+    BUG_ON(g_buffer_type_map == NULL);
+
+    return g_buffer_type_map[id];
+}
+
+void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc)
+{
+    DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    if (!exists) {
+        return;
+    }
+    exists->description = desc;
+}
+
+const char *DetectBufferTypeGetDescriptionById(const int id)
+{
+    const DetectBufferType *exists = DetectBufferTypeGetById(id);
+    if (!exists) {
+        return NULL;
+    }
+    return exists->description;
+}
+
+const char *DetectBufferTypeGetDescriptionByName(const char *name)
+{
+    const DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    if (!exists) {
+        return NULL;
+    }
+    return exists->description;
+}
+
+_Bool DetectBufferTypeSupportsPacketGetById(const int id)
+{
+    const DetectBufferType *map = DetectBufferTypeGetById(id);
+    if (map == NULL)
+        return FALSE;
+    SCLogDebug("map %p id %d packet? %d", map, id, map->packet);
+    return map->packet;
+}
+
+_Bool DetectBufferTypeSupportsMpmGetById(const int id)
+{
+    const DetectBufferType *map = DetectBufferTypeGetById(id);
+    if (map == NULL)
+        return FALSE;
+    SCLogDebug("map %p id %d mpm? %d", map, id, map->mpm);
+    return map->mpm;
+}
+
+void DetectBufferTypeRegisterSetupCallback(const char *name,
+        void (*SetupCallback)(Signature *))
+{
+    BUG_ON(g_buffer_type_reg_closed);
+    DetectBufferTypeRegister(name);
+    DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    BUG_ON(!exists);
+    exists->SetupCallback = SetupCallback;
+}
+
+void DetectBufferRunSetupCallback(const int id, Signature *s)
+{
+    const DetectBufferType *map = DetectBufferTypeGetById(id);
+    if (map && map->SetupCallback) {
+        map->SetupCallback(s);
+    }
+}
+
+void DetectBufferTypeRegisterValidateCallback(const char *name,
+        _Bool (*ValidateCallback)(const Signature *))
+{
+    BUG_ON(g_buffer_type_reg_closed);
+    DetectBufferTypeRegister(name);
+    DetectBufferType *exists = DetectBufferTypeLookupByName(name);
+    BUG_ON(!exists);
+    exists->ValidateCallback = ValidateCallback;
+}
+
+_Bool DetectBufferRunValidateCallback(const int id, const Signature *s)
+{
+    const DetectBufferType *map = DetectBufferTypeGetById(id);
+    if (map && map->ValidateCallback) {
+        return map->ValidateCallback(s);
+    }
+    return TRUE;
+}
+
+void DetectBufferTypeFinalizeRegistration(void)
+{
+    BUG_ON(g_buffer_type_hash == NULL);
+
+    const int size = g_buffer_type_id;
+    BUG_ON(!(size > 0));
+
+    g_buffer_type_map = SCCalloc(size, sizeof(DetectBufferType *));
+    BUG_ON(!g_buffer_type_map);
+
+    HashListTableBucket *b = HashListTableGetListHead(g_buffer_type_hash);
+    while (b) {
+        DetectBufferType *map = HashListTableGetListData(b);
+        g_buffer_type_map[map->id] = map;
+        SCLogDebug("name %s id %d mpm %s packet %s -- %s. "
+                "Callbacks: Setup %p Validate %p", map->string, map->id,
+                map->mpm ? "true" : "false", map->packet ? "true" : "false",
+                map->description, map->SetupCallback, map->ValidateCallback);
+        b = HashListTableGetListNext(b);
+    }
+    g_buffer_type_reg_closed = 1;
+}
+
 /* code to control the main thread to do a reload */
 
 enum DetectEngineSyncState {

src/detect-engine.h

@@ -28,6 +28,25 @@
 #include "tm-threads.h"
 #include "flow-private.h"
 
+int DetectBufferTypeRegister(const char *name);
+int DetectBufferTypeGetByName(const char *name);
+const char *DetectBufferTypeGetNameById(const int id);
+void DetectBufferTypeSupportsMpm(const char *name);
+void DetectBufferTypeSupportsPacket(const char *name);
+_Bool DetectBufferTypeSupportsMpmGetById(const int id);
+_Bool DetectBufferTypeSupportsPacketGetById(const int id);
+int DetectBufferTypeMaxId(void);
+void DetectBufferTypeFinalizeRegistration(void);
+void DetectBufferTypeSetDescriptionByName(const char *name, const char *desc);
+const char *DetectBufferTypeGetDescriptionById(const int id);
+const char *DetectBufferTypeGetDescriptionByName(const char *name);
+void DetectBufferTypeRegisterSetupCallback(const char *name,
+        void (*Callback)(Signature *));
+void DetectBufferRunSetupCallback(const int id, Signature *s);
+void DetectBufferTypeRegisterValidateCallback(const char *name,
+        _Bool (*ValidateCallback)(const Signature *));
+_Bool DetectBufferRunValidateCallback(const int id, const Signature *s);
+
 /* prototypes */
 DetectEngineCtx *DetectEngineCtxInitWithPrefix(const char *prefix);
 DetectEngineCtx *DetectEngineCtxInit(void);

src/detect-parse.c

@@ -1278,9 +1278,20 @@ int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
     uint32_t u = 0;
     uint32_t sig_flags = 0;
     SigMatch *sm, *pm;
+    const int nlists = DetectBufferTypeMaxId();
 
     SCEnter();
 
+    /* run buffer type validation callbacks if any */
+    int x;
+    for (x = 0; x < nlists; x++) {
+        if (s->init_data->smlists[x]) {
+            if (DetectBufferRunValidateCallback(x, s) == FALSE) {
+                SCReturnInt(0);
+            }
+        }
+    }
+
     if ((s->flags & SIG_FLAG_REQUIRE_PACKET) &&
         (s->flags & SIG_FLAG_REQUIRE_STREAM)) {
         SCLogError(SC_ERR_INVALID_SIGNATURE, "can't mix packet keywords with "
@@ -1319,7 +1330,7 @@ int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
             }
         }
     }
-
+#if 0 // TODO figure out why this is even necessary
     if ((s->init_data->smlists[DETECT_SM_LIST_FILEDATA] != NULL && s->alproto == ALPROTO_SMTP) ||
         s->init_data->smlists[DETECT_SM_LIST_UMATCH] != NULL ||
         s->init_data->smlists[DETECT_SM_LIST_HRUDMATCH] != NULL ||
@@ -1337,6 +1348,7 @@ int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
         s->flags |= SIG_FLAG_TOCLIENT;
         s->flags &= ~SIG_FLAG_TOSERVER;
     }
+#endif
     if ((sig_flags & (SIG_FLAG_TOCLIENT | SIG_FLAG_TOSERVER)) == (SIG_FLAG_TOCLIENT | SIG_FLAG_TOSERVER)) {
         SCLogError(SC_ERR_INVALID_SIGNATURE,"You seem to have mixed keywords "
                    "that require inspection in both directions.  Atm we only "
@@ -1447,6 +1459,21 @@ int SigValidate(DetectEngineCtx *de_ctx, Signature *s)
             SCReturnInt(0);
         }
 
+        for (int i = 0; i < nlists; i++) {
+            if (s->init_data->smlists[i] == NULL)
+                continue;
+            if (!(DetectBufferTypeGetNameById(i)))
+                continue;
+
+            if (!(DetectBufferTypeSupportsPacketGetById(i))) {
+                SCLogError(SC_ERR_INVALID_SIGNATURE, "Signature combines packet "
+                        "specific matches (like dsize, flags, ttl) with stream / "
+                        "state matching by matching on app layer proto (like using "
+                        "http_* keywords).");
+                SCReturnInt(0);
+            }
+        }
+
         if (s->init_data->smlists_tail[DETECT_SM_LIST_UMATCH] ||
                 s->init_data->smlists_tail[DETECT_SM_LIST_HRUDMATCH] ||
                 s->init_data->smlists_tail[DETECT_SM_LIST_HCBDMATCH] ||

src/detect.c

@@ -1954,6 +1954,17 @@ int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s)
     if (s->init_data->smlists[DETECT_SM_LIST_AMATCH] != NULL)
         return 0;
 
+    /* for now assume that all registered buffer types are incompatible */
+    const int nlists = DetectBufferTypeMaxId();
+    for (int i = 0; i < nlists; i++) {
+        if (s->init_data->smlists[i] == NULL)
+            continue;
+        if (!(DetectBufferTypeGetNameById(i)))
+            continue;
+
+        SCReturnInt(0);
+    }
+
     /* TMATCH list can be ignored, it contains TAGs and
      * tags are compatible to IP-only. */
 
@@ -2054,6 +2065,17 @@ static int SignatureIsPDOnly(const Signature *s)
     if (s->init_data->smlists[DETECT_SM_LIST_AMATCH] != NULL)
         return 0;
 
+    /* for now assume that all registered buffer types are incompatible */
+    const int nlists = DetectBufferTypeMaxId();
+    for (int i = 0; i < nlists; i++) {
+        if (s->init_data->smlists[i] == NULL)
+            continue;
+        if (!(DetectBufferTypeGetNameById(i)))
+            continue;
+
+        SCReturnInt(0);
+    }
+
     /* TMATCH list can be ignored, it contains TAGs and
      * tags are compatible to DP-only. */
 
@@ -2157,6 +2179,17 @@ static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s)
         SCReturnInt(0);
     }
 
+    /* for now assume that all registered buffer types are incompatible */
+    const int nlists = DetectBufferTypeMaxId();
+    for (int i = 0; i < nlists; i++) {
+        if (s->init_data->smlists[i] == NULL)
+            continue;
+        if (!(DetectBufferTypeGetNameById(i)))
+            continue;
+
+        SCReturnInt(0);
+    }
+
     /* check for conflicting keywords */
     SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH];
     for ( ;sm != NULL; sm = sm->next) {
@@ -3358,6 +3391,7 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
     uint32_t cnt_payload = 0;
     uint32_t cnt_applayer = 0;
     uint32_t cnt_deonly = 0;
+    const int nlists = DetectBufferTypeMaxId();
 
     if (!(de_ctx->flags & DE_QUIET)) {
         SCLogDebug("building signature grouping structure, stage 1: "
@@ -3481,6 +3515,13 @@ int SigAddressPrepareStage1(DetectEngineCtx *de_ctx)
             }
         }
 
+        /* run buffer type callbacks if any */
+        int x;
+        for (x = 0; x < nlists; x++) {
+            if (tmp_s->init_data->smlists[x])
+                DetectBufferRunSetupCallback(x, tmp_s);
+        }
+
         de_ctx->sig_cnt++;
     }
 
@@ -4302,6 +4343,9 @@ void SigTableSetup(void)
     DetectBypassRegister();
     DetectHttpRequestLineRegister();
     DetectHttpResponseLineRegister();
+
+    /* close keyword registration */
+    DetectBufferTypeFinalizeRegistration();
 }
 
 void SigTableRegisterTests(void)