From 65f6d24e340e2a4db8b984406c39c5570b9db64b Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Tue, 8 Jul 2025 09:27:58 -0400 Subject: [PATCH 1/2] detect/entropy: Unique flowvar names Issue: 7814 Support unique flow variable names for entropy-related values. Since this is a breaking change from Suricata 8, unique names are configurable (default value off) with this setting logging: entropy: make-unique: on When `make-unique` is on, entropy-related flow variable names are structured as: sid:;buffer:;instance: Instance is a small integer value corresponding to the number of entropy values in the rule. The first entropy usage will create a flow variable with an instance value of 1; the second will use 2, and so on. This is necessary to disambiguate the flow variable names. When `make-unique` is off (default = off), this format is used: --- etc/schema.json | 34 +++++++++++++++++++++------- src/detect-entropy.c | 54 +++++++++++++++++++++++++++++++++++++++----- src/output-json.c | 52 ++++++++++++++++++++++++++++++++++++++++-- suricata.yaml.in | 9 ++++++++ 4 files changed, 133 insertions(+), 16 deletions(-) diff --git a/etc/schema.json b/etc/schema.json index 03e8b37402..bb90c3fb08 100644 --- a/etc/schema.json +++ b/etc/schema.json @@ -3449,14 +3449,32 @@ "type": "object", "additionalProperties": false, "properties": { - "entropy": { - "type": "object", - "additionalProperties": true, - "suricata": { - "keywords": [ - "entropy" - ] - } + "entropy": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "properties": { + "buffer": { "type": "string" }, + "instance": { "type": "integer" }, + "sid": { "type": "integer" }, + "value": { "type": "number" } + }, + "required": ["sid", "buffer", "instance", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "buffer": { "type": "string" }, + "value": { "type": "number" } + }, + "required": ["buffer", "value"], + "additionalProperties": false + } + ] + } }, "flowbits": { "type": "array", diff --git a/src/detect-entropy.c b/src/detect-entropy.c index fee8307a0e..2281388ae7 100644 --- a/src/detect-entropy.c +++ b/src/detect-entropy.c @@ -28,8 +28,39 @@ #include "rust.h" +/* + * Base size required for the entropy var name: + * - 17 (tags) + * 10 (max sid) + * 5 (separators) + * 2 (entropy instance cnt) + */ +#define ENTROPY_VAR_NAME_BASE_LEN (17 + 10 + 5 + 2) +#define ENTROPY_VAR_NAME_FORMAT "sid:%d;buffer:%s;instance:%d" + +static int DetectEntropyRuleInstanceCount(Signature *s) +{ + int entropy_cnt = 1; + for (uint32_t x = 0; x < s->init_data->buffer_index; x++) { + for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_ENTROPY) { + ++entropy_cnt; + } + } + } + + return entropy_cnt; +} + static int DetectEntropySetup(DetectEngineCtx *de_ctx, Signature *s, const char *arg) { + + int unique_name = 0; + SCConfGetBool("logging.entropy.make-unique", &unique_name); + if (unique_name) { + SCLogConfig("entropy values are marked with signature_id"); + } + DetectEntropyData *ded = SCDetectEntropyParse(arg); if (ded == NULL) { goto error; @@ -42,17 +73,28 @@ static int DetectEntropySetup(DetectEngineCtx *de_ctx, Signature *s, const char goto error; sm_list = s->init_data->list; - const char *name; + const char *var_name_ptr; if (sm_list == DETECT_SM_LIST_BASE64_DATA) { - name = "base64_data"; + var_name_ptr = "base64_data"; } else { - name = DetectEngineBufferTypeGetNameById(de_ctx, sm_list); - if (name == NULL) { + var_name_ptr = DetectEngineBufferTypeGetNameById(de_ctx, sm_list); + if (var_name_ptr == NULL) { DEBUG_VALIDATE_BUG_ON(1); - name = "unknown"; + var_name_ptr = "unknown"; } } - ded->fv_idx = VarNameStoreRegister(name, VAR_TYPE_FLOW_FLOAT); + if (unique_name) { + int entropy_cnt = DetectEntropyRuleInstanceCount(s); + SCLogDebug("There are a total of %d entropy usages in this rule", entropy_cnt); + + char name_buf[ENTROPY_VAR_NAME_BASE_LEN + strlen(var_name_ptr)]; + snprintf(name_buf, sizeof(name_buf), ENTROPY_VAR_NAME_FORMAT, s->id, var_name_ptr, + entropy_cnt); + + ded->fv_idx = VarNameStoreRegister(name_buf, VAR_TYPE_FLOW_FLOAT); + } else { + ded->fv_idx = VarNameStoreRegister(var_name_ptr, VAR_TYPE_FLOW_FLOAT); + } } else { ded->fv_idx = VarNameStoreRegister("content", VAR_TYPE_FLOW_FLOAT); } diff --git a/src/output-json.c b/src/output-json.c index f496df192f..5556fdb00a 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -279,11 +279,59 @@ static void EveAddFlowVars(const Flow *f, SCJsonBuilder *js_root, SCJsonBuilder const char *varname = VarNameStoreLookupById(fv->idx, VAR_TYPE_FLOW_FLOAT); if (varname) { if (js_entropyvals == NULL) { - js_entropyvals = SCJbNewObject(); + js_entropyvals = SCJbNewArray(); if (js_entropyvals == NULL) break; } - SCJbSetFloat(js_entropyvals, varname, fv->data.fv_float.value); + /* Transform a variable name like "sid:39133;buffer:file_data;instance:1" + * into a set of pairs like: + * "sid:39133;buffer:file_data;instance:1" + */ + char *tmp_var = SCStrdup(varname); + if (tmp_var == NULL) { + continue; + } + char *saveptr; + char *tok = strtok_r(tmp_var, ";", &saveptr); + if (tok == NULL) { + SCFree(tmp_var); + continue; + } + SCJbStartObject(js_entropyvals); + while (tok) { + char *colon = strchr(tok, ':'); + if (colon) { + *colon = '\0'; + const char *key = tok; + const char *val = colon + 1; + bool is_num = true; + for (const char *p = val; *p; ++p) { + if (!isdigit((unsigned char)*p)) { + is_num = false; + SCJbSetString(js_entropyvals, key, val); + break; + } + } + if (is_num) { + uint32_t value; + if (StringParseU32RangeCheck(&value, 10, 0, val, 0, UINT32_MAX) > + 0) { + SCJbSetUint(js_entropyvals, key, value); + } else { + SCLogDebug("Invalid value from entropy varname for key %s; " + "value \"%s\"", + key, val); + break; + } + } + } else { + SCJbSetString(js_entropyvals, "buffer", tok); + } + tok = strtok_r(NULL, ";", &saveptr); + } + SCFree(tmp_var); + SCJbSetFloat(js_entropyvals, "value", fv->data.fv_float.value); + SCJbClose(js_entropyvals); } } else if (fv->datatype == FLOWVAR_TYPE_INT) { diff --git a/suricata.yaml.in b/suricata.yaml.in index 2f937f21d2..26bb3ef085 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -608,6 +608,15 @@ logging: # This value is overridden by the SC_LOG_LEVEL env var. default-log-level: notice + # Ensure that logged entropy values have unique names by including additional + # information in the logged output name: signature id, instance + # When off: file_data + # When on: sid:nnn;buffer:file_data:instance:1 + # The default value is off; it is strongly recommended to change this + # value to on + entropy: + make-unique: on + # The default output format. Optional parameter, should default to # something reasonable if not provided. Can be overridden in an # output section. You can leave this out to get the default. From a0af50f188ede828f29c08ccdc7f3c12d7be665c Mon Sep 17 00:00:00 2001 From: Jeff Lucovsky Date: Sat, 12 Jul 2025 09:11:27 -0400 Subject: [PATCH 2/2] doc/entropy: Describe unique names conf var Discuss the new configuration variable that provides unique names for entropy values in logging output: logging.entropy.make-unique Issue: 7814 --- doc/userguide/configuration/suricata-yaml.rst | 29 +++++++++++++++---- doc/userguide/rules/payload-keywords.rst | 27 ++++++++++++++--- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/doc/userguide/configuration/suricata-yaml.rst b/doc/userguide/configuration/suricata-yaml.rst index ef2a55baca..3d29b7aab9 100644 --- a/doc/userguide/configuration/suricata-yaml.rst +++ b/doc/userguide/configuration/suricata-yaml.rst @@ -540,13 +540,13 @@ In multi mode the filename takes a few special variables: - %n representing the thread number - %i representing the thread id - %t representing the timestamp (secs or secs.usecs based on 'ts-format') - + Example: filename: pcap.%n.%t .. note:: It is possible to use directories but the directories are not created by Suricata. For example ``filename: pcaps/%n/log.%s`` will log into the pre-existing ``pcaps`` directory and per thread sub directories. - + .. note:: that the limit and max-files settings are enforced per thread. So the size limit using 8 threads with 1000mb files and 2000 files is about 16TiB. @@ -2151,8 +2151,8 @@ A logging line exists of two parts. First it displays meta information i: suricata: This is Suricata version 7.0.2 RELEASE running in USER mode -(Here the part until the second `:` is the meta info, -"This is Suricata version 7.0.2 RELEASE running in USER mode" is the actual +(Here the part until the second `:` is the meta info, +"This is Suricata version 7.0.2 RELEASE running in USER mode" is the actual message.) It is possible to determine which information will be displayed in @@ -3063,6 +3063,25 @@ headers are encapsulated in the same number of headers. Advanced Options ---------------- +entropy +~~~~~~~ + +When a rule causes an entropy value to be calculated for a flow, output for the flow will include +the calculated entropy value. The log output contains the sticky buffer name for which the +entropy was calculated. Often, more context is needed and the configuration setting shown +below will amend the sticky buffer name with the signature id and instance number from the rule +that caused the entropy value calculation. The instance number changes for each ``entropy`` +keyword usage within a rule. The default value is ``off``. We strongly recommend changing +the value to ``on``. + +:: + + logging: + # Ensure that logged entropy values have unique names by appending the signature_id + # of the rule and other values where used + #entropy: + #make-unique: off + stacktrace ~~~~~~~~~~ Display diagnostic stacktraces when a signal unexpectedly terminates Suricata, e.g., such as @@ -3121,7 +3140,7 @@ Suricata 7.0 default: #allow-rules: true # Upper bound of allocations by a Lua rule before it will fail - #max-bytes: 500000 + #max-bytes: 500000 # Upper bound of lua instructions by a Lua rule before it will fail #max-instructions: 500000 diff --git a/doc/userguide/rules/payload-keywords.rst b/doc/userguide/rules/payload-keywords.rst index 8e9d8f71fa..c245f91ada 100644 --- a/doc/userguide/rules/payload-keywords.rst +++ b/doc/userguide/rules/payload-keywords.rst @@ -731,7 +731,7 @@ The following operators are available:: This example matches if the `file.data` content for an HTTP transaction has a Shannon entropy value of 4 or higher:: - alert http any any -> any any (msg:"entropy simple test"; file.data; entropy: value >= 4; sid:1;) + alert http any any -> any any (msg:"entropy simple test"; file.data; entropy: value >= 4; sid:1;) Logging ~~~~~~~ @@ -744,9 +744,28 @@ the ``metadata`` section of an output log. The follow is an example that shows the calculated entropy value with the buffer on which the value was computed:: "metadata": { - "entropy": { - "file_data": 4.265743301617466 - } + "entropy":[ + { + "buffer":"file_data", + "value":4.150007324019584 + } + ] + } + +When ``logging.entropy.make-unique`` is enabled, the format is modified to +include the signature id, instance number (tracks which entropy keyword +was used in a rule in case there are multiple usages), buffer and +value:: + + "metadata": { + entropy": [ + { + "sid": 1, + "buffer": "file_data", + "instance": 1, + "value": 4.137370175000773 + } + ] } The events where entropy is logged will depend largely on how it's used within a