eve/metadata: create preformatted json string at start up

Avoid runtime overhead of assembling metadata json string by
pre-creating it at rule parsing time.

src/detect-metadata.c

@@ -33,6 +33,8 @@
 #include "detect-metadata.h"
 #include "util-hash-string.h"
 #include "util-unittest.h"
+#include "rust.h"
+#include "util-validate.h"
 
 static int DetectMetadataSetup (DetectEngineCtx *, Signature *, const char *);
 #ifdef UNITTESTS
@@ -101,8 +103,82 @@ static const char *DetectMedatataHashAdd(DetectEngineCtx *de_ctx, const char *st
     return NULL;
 }
 
+static int SortHelper(const void *a, const void *b)
+{
+    const DetectMetadata *ma = *(const DetectMetadata **)a;
+    const DetectMetadata *mb = *(const DetectMetadata **)b;
+    return strcasecmp(ma->key, mb->key);
+}
+
+static char *CraftPreformattedJSON(const DetectMetadata *head)
+{
+    int cnt = 0;
+    for (const DetectMetadata *m = head; m != NULL; m = m->next) {
+        cnt++;
+    }
+    if (cnt == 0)
+        return NULL;
+
+    const DetectMetadata *array[cnt];
+    int i = 0;
+    for (const DetectMetadata *m = head; m != NULL; m = m->next) {
+        array[i++] = m;
+    }
+    BUG_ON(i != cnt);
+    qsort(array, cnt, sizeof(DetectMetadata *), SortHelper);
+
+    JsonBuilder *js = jb_new_object();
+    if (js == NULL)
+        return NULL;
+
+    /* array is sorted by key, so we can create a jsonbuilder object
+     * with each key appearing just once with one or more values */
+    bool array_open = false;
+    for (int j = 0; j < cnt; j++) {
+        const DetectMetadata *m = array[j];
+        const DetectMetadata *nm = j + 1 < cnt ? array[j + 1] : NULL;
+        DEBUG_VALIDATE_BUG_ON(m == NULL); // for scan-build
+
+        if (nm && strcasecmp(m->key, nm->key) == 0) {
+            if (!array_open) {
+                jb_open_array(js, m->key);
+                array_open = true;
+            }
+            jb_append_string(js, m->value);
+        } else {
+            if (!array_open) {
+                jb_open_array(js, m->key);
+            }
+            jb_append_string(js, m->value);
+            jb_close(js);
+            array_open = false;
+        }
+    }
+    jb_close(js);
+    /* we have a complete json builder. Now store it as a C string */
+    const size_t len = jb_len(js);
+#define MD_STR "\"metadata\":"
+#define MD_STR_LEN (sizeof(MD_STR) - 1)
+    char *str = SCMalloc(len + MD_STR_LEN + 1);
+    if (str == NULL) {
+        jb_free(js);
+        return NULL;
+    }
+    char *ptr = str;
+    memcpy(ptr, MD_STR, MD_STR_LEN);
+    ptr += MD_STR_LEN;
+    memcpy(ptr, jb_ptr(js), len);
+    ptr += len;
+    *ptr = '\0';
+#undef MD_STR
+#undef MD_STR_LEN
+    jb_free(js);
+    return str;
+}
+
 static int DetectMetadataParse(DetectEngineCtx *de_ctx, Signature *s, const char *metadatastr)
 {
+    DetectMetadata *head = s->metadata ? s->metadata->list : NULL;
     char copy[strlen(metadatastr)+1];
     strlcpy(copy, metadatastr, sizeof(copy));
     char *xsaveptr = NULL;
@@ -147,13 +223,28 @@ static int DetectMetadataParse(DetectEngineCtx *de_ctx, Signature *s, const char
         }
         dkv->key = hkey;
         dkv->value = hval;
-        dkv->next = s->metadata;
-        s->metadata = dkv;
+        dkv->next = head;
+        head = dkv;
 
     next:
         key = strtok_r(NULL, ",", &xsaveptr);
     }
-
+    if (head != NULL) {
+        if (s->metadata == NULL) {
+            s->metadata = SCCalloc(1, sizeof(*s->metadata));
+            if (s->metadata == NULL) {
+                for (DetectMetadata *m = head; m != NULL; ) {
+                    DetectMetadata *next_m = m->next;
+                    DetectMetadataFree(m);
+                    m = next_m;
+                }
+                return -1;
+            }
+        }
+        s->metadata->list = head;
+        SCFree(s->metadata->json_str);
+        s->metadata->json_str = CraftPreformattedJSON(head);
+    }
     return 0;
 }
 
@@ -187,10 +278,7 @@ static int DetectMetadataParseTest01(void)
 static int DetectMetadataParseTest02(void)
 {
     DetectEngineSetParseMetadata();
-    DetectEngineCtx *de_ctx = NULL;
-    DetectMetadata *dm;
-
-    de_ctx = DetectEngineCtxInit();
+    DetectEngineCtx *de_ctx = DetectEngineCtxInit();
     FAIL_IF_NULL(de_ctx);
     Signature *sig = DetectEngineAppendSig(de_ctx,
                                            "alert tcp any any -> any any "
@@ -199,16 +287,18 @@ static int DetectMetadataParseTest02(void)
                                            "sid:1; rev:1;)");
     FAIL_IF_NULL(sig);
     FAIL_IF_NULL(sig->metadata); 
-    FAIL_IF_NULL(sig->metadata->key); 
-    FAIL_IF(strcmp("jaivu", sig->metadata->key));
-    FAIL_IF(strcmp("gros_minet", sig->metadata->value));
-    FAIL_IF_NULL(sig->metadata->next); 
-    dm = sig->metadata->next;
+    FAIL_IF_NULL(sig->metadata->list); 
+    FAIL_IF_NULL(sig->metadata->list->key); 
+    FAIL_IF(strcmp("jaivu", sig->metadata->list->key));
+    FAIL_IF(strcmp("gros_minet", sig->metadata->list->value));
+    FAIL_IF_NULL(sig->metadata->list->next); 
+    DetectMetadata *dm = sig->metadata->list->next;
     FAIL_IF(strcmp("titi", dm->key));
     dm = dm->next;
     FAIL_IF_NULL(dm);
     FAIL_IF(strcmp("toto", dm->key));
-
+    FAIL_IF_NOT(strcmp(sig->metadata->json_str,
+        "\"metadata\":{\"jaivu\":[\"gros_minet\"],\"titi\":[\"2\"],\"toto\":[\"1\"]}") == 0);
     DetectEngineCtxFree(de_ctx);
     PASS;
 }
@@ -221,4 +311,4 @@ static void DetectMetadataRegisterTests(void)
     UtRegisterTest("DetectMetadataParseTest01", DetectMetadataParseTest01);
     UtRegisterTest("DetectMetadataParseTest02", DetectMetadataParseTest02);
 }
-#endif /* UNITTESTS */
+#endif /* UNITTESTS */

src/detect-metadata.h

@@ -36,6 +36,11 @@ typedef struct DetectMetadata_ {
     struct DetectMetadata_ *next;
 } DetectMetadata;
 
+typedef struct DetectMetadataHead {
+    char *json_str;
+    DetectMetadata *list;
+} DetectMetadataHead;
+
 /* prototypes */
 void DetectMetadataRegister (void);
 

src/detect-parse.c

@@ -1298,18 +1298,19 @@ static void SigMetadataFree(Signature *s)
     DetectMetadata *mdata = NULL;
     DetectMetadata *next_mdata = NULL;
 
-    if (s == NULL) {
+    if (s == NULL || s->metadata == NULL) {
         SCReturn;
     }
 
     SCLogDebug("s %p, s->metadata %p", s, s->metadata);
 
-    for (mdata = s->metadata; mdata != NULL;)   {
+    for (mdata = s->metadata->list; mdata != NULL;)   {
         next_mdata = mdata->next;
         DetectMetadataFree(mdata);
         mdata = next_mdata;
     }
-
+    SCFree(s->metadata->json_str);
+    SCFree(s->metadata);
     s->metadata = NULL;
 
     SCReturn;

src/detect.h

@@ -589,7 +589,7 @@ typedef struct Signature_ {
     /** Reference */
     DetectReference *references;
     /** Metadata */
-    DetectMetadata *metadata;
+    DetectMetadataHead *metadata;
 
     char *sig_str;
 

src/output-json-alert.c

@@ -278,33 +278,10 @@ static void AlertJsonSourceTarget(const Packet *p, const PacketAlert *pa,
 }
 
 static void AlertJsonMetadata(AlertJsonOutputCtx *json_output_ctx,
-        const PacketAlert *pa, JsonBuilder *ajs)
+        const PacketAlert *pa, JsonBuilder *js)
 {
-    if (pa->s->metadata) {
-        const DetectMetadata* kv = pa->s->metadata;
-        json_t *mjs = json_object();
-        if (unlikely(mjs == NULL)) {
-            return;
-        }
-        while (kv) {
-            json_t *jkey = json_object_get(mjs, kv->key);
-            if (jkey == NULL) {
-                jkey = json_array();
-                if (unlikely(jkey == NULL))
-                    break;
-                json_array_append_new(jkey, json_string(kv->value));
-                json_object_set_new(mjs, kv->key, jkey);
-            } else {
-                json_array_append_new(jkey, json_string(kv->value));
-            }
-
-            kv = kv->next;
-        }
-
-        if (json_object_size(mjs) > 0) {
-            jb_set_jsont(ajs, "metadata", mjs);
-        }
-        json_decref(mjs);
+    if (pa->s->metadata && pa->s->metadata->json_str) {
+        jb_set_formatted(js, pa->s->metadata->json_str);
     }
 }