Merge a0af50f188 into 9a37bb659b

doc/userguide/configuration/suricata-yaml.rst

@@ -3188,6 +3188,25 @@ headers are encapsulated in the same number of headers.
 Advanced Options
 ----------------
 
+entropy
+~~~~~~~
+
+When a rule causes an entropy value to be calculated for a flow, output for the flow will include
+the calculated entropy value. The log output contains the sticky buffer name for which the
+entropy was calculated. Often, more context is needed and the configuration setting shown
+below will amend the sticky buffer name with the signature id and instance number from the rule
+that caused the entropy value calculation. The instance number changes for each ``entropy``
+keyword usage within a rule.  The default value is ``off``. We strongly recommend changing
+the value to ``on``.
+
+::
+
+    logging:
+        # Ensure that logged entropy values have unique names by appending the signature_id
+        # of the rule and other values where used
+        #entropy:
+            #make-unique: off
+
 stacktrace
 ~~~~~~~~~~
 Display diagnostic stacktraces when a signal unexpectedly terminates Suricata, e.g., such as

doc/userguide/rules/payload-keywords.rst

@@ -842,7 +842,7 @@ The following operators are available::
 This example matches if the `file.data` content for an HTTP transaction has
 a Shannon entropy value of 4 or higher::
 
-	alert http any any -> any any (msg:"entropy simple test"; file.data; entropy: value >= 4; sid:1;)
+    alert http any any -> any any (msg:"entropy simple test"; file.data; entropy: value >= 4; sid:1;)
 
 Logging
 ~~~~~~~
@@ -855,9 +855,28 @@ the ``metadata`` section of an output log. The follow is an example that shows
 the calculated entropy value with the buffer on which the value was computed::
 
      "metadata": {
-        "entropy": {
-          "file_data": 4.265743301617466
-        }
+        "entropy":[
+           {
+             "buffer":"file_data",
+             "value":4.150007324019584
+           }
+        ]
+      }
+
+When ``logging.entropy.make-unique`` is enabled, the format is modified to
+include the signature id, instance number (tracks which entropy keyword
+was used in a rule in case there are multiple usages), buffer and
+value::
+
+     "metadata": {
+        entropy": [
+          {
+            "sid": 1,
+            "buffer": "file_data",
+            "instance": 1,
+            "value": 4.137370175000773
+          }
+        ]
       }
 
 The events where entropy is logged will depend largely on how it's used within a

etc/schema.json

@@ -4012,14 +4012,32 @@
             "type": "object",
             "additionalProperties": false,
             "properties": {
-                "entropy": {
-                    "type": "object",
-                    "additionalProperties": true,
-                    "suricata": {
-                        "keywords": [
-                            "entropy"
-                        ]
-                    }
+            "entropy": {
+                "type": "array",
+                "items": {
+                    "anyOf": [
+                        {
+                            "type": "object",
+                            "properties": {
+                              "buffer": { "type": "string" },
+                              "instance": { "type": "integer" },
+                              "sid": { "type": "integer" },
+                              "value": { "type": "number" }
+                        },
+                        "required": ["sid", "buffer", "instance", "value"],
+                        "additionalProperties": false
+                      },
+                      {
+                        "type": "object",
+                        "properties": {
+                          "buffer": { "type": "string" },
+                          "value": { "type": "number" }
+                        },
+                        "required": ["buffer", "value"],
+                        "additionalProperties": false
+                      }
+                    ]
+                  }
                 },
                 "flowbits": {
                     "type": "array",

src/detect-entropy.c

@@ -28,8 +28,39 @@
 
 #include "rust.h"
 
+/*
+ * Base size required for the entropy var name:
+ * - 17 (tags)
+ *   10 (max sid)
+ *   5 (separators)
+ *   2 (entropy instance cnt)
+ */
+#define ENTROPY_VAR_NAME_BASE_LEN (17 + 10 + 5 + 2)
+#define ENTROPY_VAR_NAME_FORMAT   "sid:%d;buffer:%s;instance:%d"
+
+static int DetectEntropyRuleInstanceCount(Signature *s)
+{
+    int entropy_cnt = 1;
+    for (uint32_t x = 0; x < s->init_data->buffer_index; x++) {
+        for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) {
+            if (sm->type == DETECT_ENTROPY) {
+                ++entropy_cnt;
+            }
+        }
+    }
+
+    return entropy_cnt;
+}
+
 static int DetectEntropySetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg)
 {
+
+    int unique_name = 0;
+    SCConfGetBool("logging.entropy.make-unique", &unique_name);
+    if (unique_name) {
+        SCLogConfig("entropy values are marked with signature_id");
+    }
+
     DetectEntropyData *ded = SCDetectEntropyParse(arg);
     if (ded == NULL) {
         goto error;
@@ -42,17 +73,28 @@ static int DetectEntropySetup(DetectEngineCtx *de_ctx, Signature *s, const char
             goto error;
 
         sm_list = s->init_data->list;
-        const char *name;
+        const char *var_name_ptr;
         if (sm_list == DETECT_SM_LIST_BASE64_DATA) {
-            name = "base64_data";
+            var_name_ptr = "base64_data";
         } else {
-            name = DetectEngineBufferTypeGetNameById(de_ctx, sm_list);
-            if (name == NULL) {
+            var_name_ptr = DetectEngineBufferTypeGetNameById(de_ctx, sm_list);
+            if (var_name_ptr == NULL) {
                 DEBUG_VALIDATE_BUG_ON(1);
-                name = "unknown";
+                var_name_ptr = "unknown";
             }
         }
-        ded->fv_idx = VarNameStoreRegister(name, VAR_TYPE_FLOW_FLOAT);
+        if (unique_name) {
+            int entropy_cnt = DetectEntropyRuleInstanceCount(s);
+            SCLogDebug("There are a total of %d entropy usages in this rule", entropy_cnt);
+
+            char name_buf[ENTROPY_VAR_NAME_BASE_LEN + strlen(var_name_ptr)];
+            snprintf(name_buf, sizeof(name_buf), ENTROPY_VAR_NAME_FORMAT, s->id, var_name_ptr,
+                    entropy_cnt);
+
+            ded->fv_idx = VarNameStoreRegister(name_buf, VAR_TYPE_FLOW_FLOAT);
+        } else {
+            ded->fv_idx = VarNameStoreRegister(var_name_ptr, VAR_TYPE_FLOW_FLOAT);
+        }
     } else {
         ded->fv_idx = VarNameStoreRegister("content", VAR_TYPE_FLOW_FLOAT);
     }

src/output-json.c

@@ -275,11 +275,59 @@ static void EveAddFlowVars(const Flow *f, SCJsonBuilder *js_root, SCJsonBuilder
                 const char *varname = VarNameStoreLookupById(fv->idx, VAR_TYPE_FLOW_FLOAT);
                 if (varname) {
                     if (js_entropyvals == NULL) {
-                        js_entropyvals = SCJbNewObject();
+                        js_entropyvals = SCJbNewArray();
                         if (js_entropyvals == NULL)
                             break;
                     }
-                    SCJbSetFloat(js_entropyvals, varname, fv->data.fv_float.value);
+                    /* Transform a variable name like "sid:39133;buffer:file_data;instance:1"
+                     * into a set of pairs like:
+                     * "sid:39133;buffer:file_data;instance:1"
+                     */
+                    char *tmp_var = SCStrdup(varname);
+                    if (tmp_var == NULL) {
+                        continue;
+                    }
+                    char *saveptr;
+                    char *tok = strtok_r(tmp_var, ";", &saveptr);
+                    if (tok == NULL) {
+                        SCFree(tmp_var);
+                        continue;
+                    }
+                    SCJbStartObject(js_entropyvals);
+                    while (tok) {
+                        char *colon = strchr(tok, ':');
+                        if (colon) {
+                            *colon = '\0';
+                            const char *key = tok;
+                            const char *val = colon + 1;
+                            bool is_num = true;
+                            for (const char *p = val; *p; ++p) {
+                                if (!isdigit((unsigned char)*p)) {
+                                    is_num = false;
+                                    SCJbSetString(js_entropyvals, key, val);
+                                    break;
+                                }
+                            }
+                            if (is_num) {
+                                uint32_t value;
+                                if (StringParseU32RangeCheck(&value, 10, 0, val, 0, UINT32_MAX) >
+                                        0) {
+                                    SCJbSetUint(js_entropyvals, key, value);
+                                } else {
+                                    SCLogDebug("Invalid value from entropy varname for key %s; "
+                                               "value \"%s\"",
+                                            key, val);
+                                    break;
+                                }
+                            }
+                        } else {
+                            SCJbSetString(js_entropyvals, "buffer", tok);
+                        }
+                        tok = strtok_r(NULL, ";", &saveptr);
+                    }
+                    SCFree(tmp_var);
+                    SCJbSetFloat(js_entropyvals, "value", fv->data.fv_float.value);
+                    SCJbClose(js_entropyvals);
                 }
 
             } else if (fv->datatype == FLOWVAR_TYPE_INT) {

suricata.yaml.in

@@ -621,6 +621,15 @@ logging:
   # This value is overridden by the SC_LOG_LEVEL env var.
   default-log-level: notice
 
+  # Ensure that logged entropy values have unique names by including additional
+  # information in the logged output name: signature id, instance
+  # When off: file_data
+  # When on: sid:nnn;buffer:file_data:instance:1
+  # The default value is off; it is strongly recommended to change this
+  # value to on
+  entropy:
+    make-unique: on
+
   # The default output format.  Optional parameter, should default to
   # something reasonable if not provided.  Can be overridden in an
   # output section.  You can leave this out to get the default.