datajson: add remove_key option to dataset

This option allows to remove the key corresponding to the match
value from the JSON object before creating the JSON object that
will be added to the `extra` data.

For example, matching on the following JSON on the `ip` key:

```json
{"ip": "10.16.1.11", "test": "success", "context":3}
```

with a match like:

```
dataset:isset,src_ip,type ip,load src.lst,format jsonline,enrichment_key src_ip,value_key ip;
```

will produce the following:

```json
"extra": {
  "src_ip": {
    "ip": "10.16.1.11",
    "test": "success",
    "context": 3
  }
```

if we add the `remove_key` option to the match:

```
dataset:isset,src_ip,type ip,load src.lst,format jsonline,enrichment_key src_ip,value_key ip, remove_key;
```

it will produce the following:

```json
"extra": {
  "src_ip": {
    "test": "success",
    "context": 3
  }
```

The option is set to false by default.

Ticket: #7372

src/datajson.c

@@ -347,8 +347,12 @@ static uint32_t DatajsonAddStringElement(Dataset *set, json_t *value, char *json
 
     *found = true;
 
-    const char *val = json_string_value(key);
+    char val[DATAJSON_JSON_LENGTH];
+    strlcpy(val, json_string_value(key), DATAJSON_JSON_LENGTH - 1);
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -408,6 +412,9 @@ static uint32_t DatajsonAddMd5Element(Dataset *set, json_t *value, char *json_ke
         return 0;
     }
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -467,6 +474,9 @@ static uint32_t DatajsonAddSha256Element(Dataset *set, json_t *value, char *json
         return 0;
     }
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -521,6 +531,9 @@ static uint32_t DatajsonAddIpv4Element(Dataset *set, json_t *value, char *json_k
         return 0;
     }
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -577,6 +590,9 @@ static uint32_t DatajsonAddIPv6Element(Dataset *set, json_t *value, char *json_k
         return 0;
     }
     DataJsonType elt = { .value = NULL, .len = 0 };
+    if (set->remove_key) {
+        json_object_del(value, json_key);
+    }
     elt.value = json_dumps(value, JSON_COMPACT);
     elt.len = strlen(elt.value);
 
@@ -616,7 +632,8 @@ static int DatajsonLoadIPv6(Dataset *set, char *json_key, char *array_key, Datas
 }
 
 Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, uint64_t memcap,
-        uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format)
+        uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format,
+        bool remove_key)
 {
     uint64_t default_memcap = 0;
     uint32_t default_hashsize = 0;
@@ -665,6 +682,7 @@ Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load,
 
     strlcpy(set->name, name, sizeof(set->name));
     set->type = type;
+    set->remove_key = remove_key;
     if (load && strlen(load)) {
         strlcpy(set->load, load, sizeof(set->load));
         SCLogDebug("set \'%s\' loading \'%s\' from \'%s\'", set->name, load, set->load);

src/datajson.h

@@ -42,7 +42,8 @@ typedef struct DataJsonResultType {
 /* Common functions */
 
 Dataset *DatajsonGet(const char *name, enum DatasetTypes type, const char *load, uint64_t memcap,
-        uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format);
+        uint32_t hashsize, char *json_key_value, char *json_array_key, DatasetFormats format,
+        bool remove_key);
 
 DataJsonResultType DatajsonLookup(Dataset *set, const uint8_t *data, const uint32_t data_len);
 

src/datasets.h

@@ -50,6 +50,7 @@ typedef struct Dataset {
     uint32_t id;
     bool from_yaml;                     /* Mark whether the set was retrieved from YAML */
     bool hidden;                        /* Mark the old sets hidden in case of reload */
+    bool remove_key;                    /* Mark that value key should be removed from extra data */
     THashTableContext *hash;
 
     char load[PATH_MAX];

src/detect-dataset.c

@@ -162,7 +162,7 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
         enum DatasetTypes *type, char *load, size_t load_size, char *save, size_t save_size,
         uint64_t *memcap, uint32_t *hashsize, DatasetFormats *format, char *value_key,
         size_t value_key_size, char *array_key, size_t array_key_size, char *enrichment_key,
-        size_t enrichment_key_size)
+        size_t enrichment_key_size, bool *remove_key)
 {
     bool cmd_set = false;
     bool name_set = false;
@@ -209,10 +209,12 @@ static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *nam
             name_set = true;
         } else {
             if (val == NULL) {
-                return -1;
-            }
-
-            if (strcmp(key, "type") == 0) {
+                /* only non fixed place option without value is remove_key */
+                if (strcmp(key, "remove_key") == 0) {
+                    *remove_key = true;
+                } else
+                    return -1;
+            } else if (strcmp(key, "type") == 0) {
                 SCLogDebug("type %s", val);
 
                 if (strcmp(val, "md5") == 0) {
@@ -455,6 +457,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     char value_key[SIG_JSON_CONTENT_KEY_LEN] = "";
     char array_key[SIG_JSON_CONTENT_KEY_LEN] = "";
     char enrichment_key[SIG_JSON_CONTENT_KEY_LEN] = "";
+    bool remove_key = false;
 
     if (DetectBufferGetActiveList(de_ctx, s) == -1) {
         SCLogError("datasets are only supported for sticky buffers");
@@ -470,7 +473,7 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     if (!DetectDatasetParse(rawstr, cmd_str, sizeof(cmd_str), name, sizeof(name), &type, load,
                 sizeof(load), save, sizeof(save), &memcap, &hashsize, &format, value_key,
                 sizeof(value_key), array_key, sizeof(array_key), enrichment_key,
-                sizeof(enrichment_key))) {
+                sizeof(enrichment_key), &remove_key)) {
         return -1;
     }
 
@@ -529,11 +532,11 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     Dataset *set = NULL;
 
     if (format == DATASET_FORMAT_JSON) {
-        set = DatajsonGet(
-                name, type, load, memcap, hashsize, value_key, array_key, DATASET_FORMAT_JSON);
+        set = DatajsonGet(name, type, load, memcap, hashsize, value_key, array_key,
+                DATASET_FORMAT_JSON, remove_key);
     } else if (format == DATASET_FORMAT_JSONLINE) {
-        set = DatajsonGet(
-                name, type, load, memcap, hashsize, value_key, NULL, DATASET_FORMAT_JSONLINE);
+        set = DatajsonGet(name, type, load, memcap, hashsize, value_key, NULL,
+                DATASET_FORMAT_JSONLINE, remove_key);
     } else {
         set = DatasetGet(name, type, save, load, memcap, hashsize);
     }