From a8e2399ea9aea93166fb9d5280dda47882b291ef Mon Sep 17 00:00:00 2001 From: Victor Julien Date: Tue, 28 Jul 2020 17:23:50 +0200 Subject: [PATCH] eve/metadata: create preformatted json string at start up Avoid runtime overhead of assembling metadata json string by pre-creating it at rule parsing time. --- src/detect-metadata.c | 118 +++++++++++++++++++++++++++++++++++----- src/detect-metadata.h | 5 ++ src/detect-parse.c | 7 ++- src/detect.h | 2 +- src/output-json-alert.c | 29 +--------- 5 files changed, 117 insertions(+), 44 deletions(-) diff --git a/src/detect-metadata.c b/src/detect-metadata.c index 0195521c63..eec83ab30d 100644 --- a/src/detect-metadata.c +++ b/src/detect-metadata.c @@ -33,6 +33,8 @@ #include "detect-metadata.h" #include "util-hash-string.h" #include "util-unittest.h" +#include "rust.h" +#include "util-validate.h" static int DetectMetadataSetup (DetectEngineCtx *, Signature *, const char *); #ifdef UNITTESTS @@ -101,8 +103,82 @@ static const char *DetectMedatataHashAdd(DetectEngineCtx *de_ctx, const char *st return NULL; } +static int SortHelper(const void *a, const void *b) +{ + const DetectMetadata *ma = *(const DetectMetadata **)a; + const DetectMetadata *mb = *(const DetectMetadata **)b; + return strcasecmp(ma->key, mb->key); +} + +static char *CraftPreformattedJSON(const DetectMetadata *head) +{ + int cnt = 0; + for (const DetectMetadata *m = head; m != NULL; m = m->next) { + cnt++; + } + if (cnt == 0) + return NULL; + + const DetectMetadata *array[cnt]; + int i = 0; + for (const DetectMetadata *m = head; m != NULL; m = m->next) { + array[i++] = m; + } + BUG_ON(i != cnt); + qsort(array, cnt, sizeof(DetectMetadata *), SortHelper); + + JsonBuilder *js = jb_new_object(); + if (js == NULL) + return NULL; + + /* array is sorted by key, so we can create a jsonbuilder object + * with each key appearing just once with one or more values */ + bool array_open = false; + for (int j = 0; j < cnt; j++) { + const DetectMetadata *m = array[j]; + const DetectMetadata *nm = j + 1 < cnt ? array[j + 1] : NULL; + DEBUG_VALIDATE_BUG_ON(m == NULL); // for scan-build + + if (nm && strcasecmp(m->key, nm->key) == 0) { + if (!array_open) { + jb_open_array(js, m->key); + array_open = true; + } + jb_append_string(js, m->value); + } else { + if (!array_open) { + jb_open_array(js, m->key); + } + jb_append_string(js, m->value); + jb_close(js); + array_open = false; + } + } + jb_close(js); + /* we have a complete json builder. Now store it as a C string */ + const size_t len = jb_len(js); +#define MD_STR "\"metadata\":" +#define MD_STR_LEN (sizeof(MD_STR) - 1) + char *str = SCMalloc(len + MD_STR_LEN + 1); + if (str == NULL) { + jb_free(js); + return NULL; + } + char *ptr = str; + memcpy(ptr, MD_STR, MD_STR_LEN); + ptr += MD_STR_LEN; + memcpy(ptr, jb_ptr(js), len); + ptr += len; + *ptr = '\0'; +#undef MD_STR +#undef MD_STR_LEN + jb_free(js); + return str; +} + static int DetectMetadataParse(DetectEngineCtx *de_ctx, Signature *s, const char *metadatastr) { + DetectMetadata *head = s->metadata ? s->metadata->list : NULL; char copy[strlen(metadatastr)+1]; strlcpy(copy, metadatastr, sizeof(copy)); char *xsaveptr = NULL; @@ -147,13 +223,28 @@ static int DetectMetadataParse(DetectEngineCtx *de_ctx, Signature *s, const char } dkv->key = hkey; dkv->value = hval; - dkv->next = s->metadata; - s->metadata = dkv; + dkv->next = head; + head = dkv; next: key = strtok_r(NULL, ",", &xsaveptr); } - + if (head != NULL) { + if (s->metadata == NULL) { + s->metadata = SCCalloc(1, sizeof(*s->metadata)); + if (s->metadata == NULL) { + for (DetectMetadata *m = head; m != NULL; ) { + DetectMetadata *next_m = m->next; + DetectMetadataFree(m); + m = next_m; + } + return -1; + } + } + s->metadata->list = head; + SCFree(s->metadata->json_str); + s->metadata->json_str = CraftPreformattedJSON(head); + } return 0; } @@ -187,10 +278,7 @@ static int DetectMetadataParseTest01(void) static int DetectMetadataParseTest02(void) { DetectEngineSetParseMetadata(); - DetectEngineCtx *de_ctx = NULL; - DetectMetadata *dm; - - de_ctx = DetectEngineCtxInit(); + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); FAIL_IF_NULL(de_ctx); Signature *sig = DetectEngineAppendSig(de_ctx, "alert tcp any any -> any any " @@ -199,16 +287,18 @@ static int DetectMetadataParseTest02(void) "sid:1; rev:1;)"); FAIL_IF_NULL(sig); FAIL_IF_NULL(sig->metadata); - FAIL_IF_NULL(sig->metadata->key); - FAIL_IF(strcmp("jaivu", sig->metadata->key)); - FAIL_IF(strcmp("gros_minet", sig->metadata->value)); - FAIL_IF_NULL(sig->metadata->next); - dm = sig->metadata->next; + FAIL_IF_NULL(sig->metadata->list); + FAIL_IF_NULL(sig->metadata->list->key); + FAIL_IF(strcmp("jaivu", sig->metadata->list->key)); + FAIL_IF(strcmp("gros_minet", sig->metadata->list->value)); + FAIL_IF_NULL(sig->metadata->list->next); + DetectMetadata *dm = sig->metadata->list->next; FAIL_IF(strcmp("titi", dm->key)); dm = dm->next; FAIL_IF_NULL(dm); FAIL_IF(strcmp("toto", dm->key)); - + FAIL_IF_NOT(strcmp(sig->metadata->json_str, + "\"metadata\":{\"jaivu\":[\"gros_minet\"],\"titi\":[\"2\"],\"toto\":[\"1\"]}") == 0); DetectEngineCtxFree(de_ctx); PASS; } @@ -221,4 +311,4 @@ static void DetectMetadataRegisterTests(void) UtRegisterTest("DetectMetadataParseTest01", DetectMetadataParseTest01); UtRegisterTest("DetectMetadataParseTest02", DetectMetadataParseTest02); } -#endif /* UNITTESTS */ \ No newline at end of file +#endif /* UNITTESTS */ diff --git a/src/detect-metadata.h b/src/detect-metadata.h index 99a5424980..fafa63f5ae 100644 --- a/src/detect-metadata.h +++ b/src/detect-metadata.h @@ -36,6 +36,11 @@ typedef struct DetectMetadata_ { struct DetectMetadata_ *next; } DetectMetadata; +typedef struct DetectMetadataHead { + char *json_str; + DetectMetadata *list; +} DetectMetadataHead; + /* prototypes */ void DetectMetadataRegister (void); diff --git a/src/detect-parse.c b/src/detect-parse.c index ca0bce5cde..03205a8f0d 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -1298,18 +1298,19 @@ static void SigMetadataFree(Signature *s) DetectMetadata *mdata = NULL; DetectMetadata *next_mdata = NULL; - if (s == NULL) { + if (s == NULL || s->metadata == NULL) { SCReturn; } SCLogDebug("s %p, s->metadata %p", s, s->metadata); - for (mdata = s->metadata; mdata != NULL;) { + for (mdata = s->metadata->list; mdata != NULL;) { next_mdata = mdata->next; DetectMetadataFree(mdata); mdata = next_mdata; } - + SCFree(s->metadata->json_str); + SCFree(s->metadata); s->metadata = NULL; SCReturn; diff --git a/src/detect.h b/src/detect.h index 2499552ef1..8060a1a69c 100644 --- a/src/detect.h +++ b/src/detect.h @@ -589,7 +589,7 @@ typedef struct Signature_ { /** Reference */ DetectReference *references; /** Metadata */ - DetectMetadata *metadata; + DetectMetadataHead *metadata; char *sig_str; diff --git a/src/output-json-alert.c b/src/output-json-alert.c index e6b73fd0a6..61262e64a3 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -278,33 +278,10 @@ static void AlertJsonSourceTarget(const Packet *p, const PacketAlert *pa, } static void AlertJsonMetadata(AlertJsonOutputCtx *json_output_ctx, - const PacketAlert *pa, JsonBuilder *ajs) + const PacketAlert *pa, JsonBuilder *js) { - if (pa->s->metadata) { - const DetectMetadata* kv = pa->s->metadata; - json_t *mjs = json_object(); - if (unlikely(mjs == NULL)) { - return; - } - while (kv) { - json_t *jkey = json_object_get(mjs, kv->key); - if (jkey == NULL) { - jkey = json_array(); - if (unlikely(jkey == NULL)) - break; - json_array_append_new(jkey, json_string(kv->value)); - json_object_set_new(mjs, kv->key, jkey); - } else { - json_array_append_new(jkey, json_string(kv->value)); - } - - kv = kv->next; - } - - if (json_object_size(mjs) > 0) { - jb_set_jsont(ajs, "metadata", mjs); - } - json_decref(mjs); + if (pa->s->metadata && pa->s->metadata->json_str) { + jb_set_formatted(js, pa->s->metadata->json_str); } }