eve/json: introduce community flow id

Add support for community flow id, meant to give a records a
predictable flow id that can be used to match records to
output of other tools.

Takes a 'seed' that needs to be same across sensors and tools
to make the id less predictable.

src/output-json-email-common.h

@@ -24,6 +24,7 @@
 #ifndef __OUTPUT_JSON_EMAIL_COMMON_H__
 #define __OUTPUT_JSON_EMAIL_COMMON_H__
 
+#ifdef HAVE_LIBJANSSON
 typedef struct OutputJsonEmailCtx_ {
     LogFileCtx *file_ctx;
     uint32_t flags; /** Store mode */
@@ -31,8 +32,6 @@ typedef struct OutputJsonEmailCtx_ {
     OutputJsonCommonSettings cfg;
 } OutputJsonEmailCtx;
 
-
-#ifdef HAVE_LIBJANSSON
 typedef struct JsonEmailLogThread_ {
     OutputJsonEmailCtx *emaillog_ctx;
     MemBuffer *buffer;
@@ -40,8 +39,8 @@ typedef struct JsonEmailLogThread_ {
 
 TmEcode JsonEmailLogJson(JsonEmailLogThread *aft, json_t *js, const Packet *p, Flow *f, void *state, void *vtx, uint64_t tx_id);
 json_t *JsonEmailAddMetadata(const Flow *f, uint32_t tx_id);
-#endif
 
 void OutputEmailInitConf(ConfNode *conf, OutputJsonEmailCtx *email_ctx);
 
+#endif /* HAVE_LIBJANSSON */
 #endif /* __OUTPUT_JSON_EMAIL_COMMON_H__ */

src/output-json.c

@@ -59,6 +59,7 @@
 #include "util-log-redis.h"
 #include "util-device.h"
 #include "util-validate.h"
+#include "util-crypt.h"
 
 #include "flow-var.h"
 #include "flow-bit.h"
@@ -90,6 +91,7 @@ void OutputJsonRegister (void)
 #define MAX_JSON_SIZE 2048
 
 static void OutputJsonDeInitCtx(OutputCtx *);
+static void CreateJSONCommunityFlowId(json_t *js, const Flow *f, const uint16_t seed);
 
 static const char *TRAFFIC_ID_PREFIX = "traffic/id/";
 static const char *TRAFFIC_LABEL_PREFIX = "traffic/label/";
@@ -391,6 +393,9 @@ void JsonAddCommonOptions(const OutputJsonCommonSettings *cfg,
     if (cfg->include_metadata) {
         JsonAddMetadata(p, f, js);
     }
+    if (cfg->include_community_id && f != NULL) {
+        CreateJSONCommunityFlowId(js, f, cfg->community_id_seed);
+    }
 }
 
 /** \brief jsonify tcp flags field
@@ -546,6 +551,124 @@ void JsonFiveTuple(const Packet *p, enum OutputJsonLogDirection dir, json_t *js)
     json_object_set_new(js, "proto", json_string(proto));
 }
 
+static void CreateJSONCommunityFlowIdv4(json_t *js, const Flow *f,
+        const uint16_t seed)
+{
+    struct {
+        uint16_t seed;
+        uint32_t src;
+        uint32_t dst;
+        uint8_t proto;
+        uint8_t pad0;
+        uint16_t sp;
+        uint16_t dp;
+    } __attribute__((__packed__)) ipv4;
+
+    uint32_t src = f->src.addr_data32[0];
+    uint32_t dst = f->dst.addr_data32[0];
+    uint16_t sp = f->sp;
+    if (f->proto == IPPROTO_ICMP)
+        sp = f->icmp_s.type;
+    sp = htons(sp);
+    uint16_t dp = f->dp;
+    if (f->proto == IPPROTO_ICMP)
+        dp = f->icmp_d.type;
+    dp = htons(dp);
+
+    ipv4.seed = htons(seed);
+    if (ntohl(src) < ntohl(dst) || (src == dst && sp < dp)) {
+        ipv4.src = src;
+        ipv4.dst = dst;
+        ipv4.sp = sp;
+        ipv4.dp = dp;
+    } else {
+        ipv4.src = dst;
+        ipv4.dst = src;
+        ipv4.sp = dp;
+        ipv4.dp = sp;
+    }
+    ipv4.proto = f->proto;
+    ipv4.pad0 = 0;
+
+    uint8_t hash[20];
+    if (ComputeSHA1((const uint8_t *)&ipv4, sizeof(ipv4), hash, sizeof(hash)) == 1) {
+        unsigned char base64buf[64] = "1:";
+        unsigned long out_len = sizeof(base64buf) - 2;
+        if (Base64Encode(hash, sizeof(hash), base64buf+2, &out_len) == SC_BASE64_OK) {
+            json_object_set_new(js, "community_id", json_string((const char *)base64buf));
+        }
+    }
+}
+
+static inline bool FlowHashRawAddressIPv6LtU32(const uint32_t *a, const uint32_t *b)
+{
+    for (int i = 0; i < 4; i++) {
+        if (a[i] < b[i])
+            return true;
+        if (a[i] > b[i])
+            break;
+    }
+
+    return false;
+}
+
+static void CreateJSONCommunityFlowIdv6(json_t *js, const Flow *f,
+        const uint16_t seed)
+{
+    struct {
+        uint16_t seed;
+        uint32_t src[4];
+        uint32_t dst[4];
+        uint8_t proto;
+        uint8_t pad0;
+        uint16_t sp;
+        uint16_t dp;
+    } __attribute__((__packed__)) ipv6;
+
+    uint16_t sp = f->sp;
+    if (f->proto == IPPROTO_ICMPV6)
+        sp = f->icmp_s.type;
+    sp = htons(sp);
+    uint16_t dp = f->dp;
+    if (f->proto == IPPROTO_ICMPV6)
+        dp = f->icmp_d.type;
+    dp = htons(dp);
+
+    ipv6.seed = htons(seed);
+    if (FlowHashRawAddressIPv6LtU32(f->src.addr_data32, f->dst.addr_data32) ||
+            ((memcmp(&f->src, &f->dst, sizeof(f->src)) == 0) && sp < dp))
+    {
+        memcpy(&ipv6.src, &f->src.addr_data32, 16);
+        memcpy(&ipv6.dst, &f->dst.addr_data32, 16);
+        ipv6.sp = sp;
+        ipv6.dp = dp;
+    } else {
+        memcpy(&ipv6.src, &f->dst.addr_data32, 16);
+        memcpy(&ipv6.dst, &f->src.addr_data32, 16);
+        ipv6.sp = dp;
+        ipv6.dp = sp;
+    }
+    ipv6.proto = f->proto;
+    ipv6.pad0 = 0;
+
+    uint8_t hash[20];
+    if (ComputeSHA1((const uint8_t *)&ipv6, sizeof(ipv6), hash, sizeof(hash)) == 1) {
+        unsigned char base64buf[64] = "1:";
+        unsigned long out_len = sizeof(base64buf) - 2;
+        if (Base64Encode(hash, sizeof(hash), base64buf+2, &out_len) == SC_BASE64_OK) {
+            json_object_set_new(js, "community_id", json_string((const char *)base64buf));
+        }
+    }
+}
+
+static void CreateJSONCommunityFlowId(json_t *js, const Flow *f, const uint16_t seed)
+{
+    if (f->flags & FLOW_IPV4)
+        return CreateJSONCommunityFlowIdv4(js, f, seed);
+    else if (f->flags & FLOW_IPV6)
+        return CreateJSONCommunityFlowIdv6(js, f, seed);
+}
+
 void CreateJSONFlowId(json_t *js, const Flow *f)
 {
     if (f == NULL)
@@ -885,6 +1008,26 @@ OutputInitResult OutputJsonInitCtx(ConfNode *conf)
             json_ctx->cfg.include_metadata = true;
         }
 
+        /* See if we want to enable the community id */
+        const ConfNode *community_id = ConfNodeLookupChild(conf, "community-id");
+        if (community_id && community_id->val && ConfValIsTrue(community_id->val)) {
+            SCLogConfig("Enabling eve community_id logging.");
+            json_ctx->cfg.include_community_id = true;
+        } else {
+            json_ctx->cfg.include_community_id = false;
+        }
+        const char *cid_seed = ConfNodeLookupChildValue(conf, "community-id-seed");
+        if (cid_seed != NULL) {
+            if (ByteExtractStringUint16(&json_ctx->cfg.community_id_seed,
+                        10, 0, cid_seed) == -1)
+            {
+                SCLogError(SC_ERR_INVALID_ARGUMENT,
+                           "Failed to initialize JSON output, "
+                           "invalid community-id-seed: %s", cid_seed);
+                exit(EXIT_FAILURE);
+            }
+        }
+
         /* Do we have a global eve xff configuration? */
         const ConfNode *xff = ConfNodeLookupChild(conf, "xff");
         if (xff != NULL) {

src/output-json.h

@@ -66,6 +66,8 @@ TmEcode JsonLogThreadDeinit(ThreadVars *t, void *data);
 
 typedef struct OutputJsonCommonSettings_ {
     bool include_metadata;
+    bool include_community_id;
+    uint16_t community_id_seed;
 } OutputJsonCommonSettings;
 
 /*

suricata.yaml.in

@@ -102,6 +102,19 @@ outputs:
 
       pcap-file: false
 
+      # Community Flow ID
+      # Adds a 'community_id' field to EVE records. These are meant to give
+      # a records a predictable flow id that can be used to match records to
+      # output of other tools such as Bro.
+      #
+      # Takes a 'seed' that needs to be same across sensors and tools
+      # to make the id less predictable.
+
+      # enable/disable the community id feature.
+      community-id: false
+      # Seed value for the ID output. Valid values are 0-65535.
+      community-id-seed: 0
+
       # HTTP X-Forwarded-For support by adding an extra field or overwriting
       # the source or destination IP address (depending on flow direction)
       # with the one reported in the X-Forwarded-For HTTP header. This is