diff --git a/rust/Cargo.lock.in b/rust/Cargo.lock.in index 46925776f5..a9019d4d70 100644 --- a/rust/Cargo.lock.in +++ b/rust/Cargo.lock.in @@ -416,6 +416,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "endian-type" version = "0.1.2" @@ -491,6 +497,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.0.35" @@ -502,6 +514,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -639,6 +657,15 @@ dependencies = [ "polyval", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -647,7 +674,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -702,7 +729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", ] [[package]] @@ -724,6 +751,15 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -803,7 +839,7 @@ version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" dependencies = [ - "hashbrown", + "hashbrown 0.16.1", ] [[package]] @@ -1082,6 +1118,18 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", + "serde", +] + [[package]] name = "phf" version = "0.10.1" @@ -1557,6 +1605,7 @@ dependencies = [ "hkdf", "humantime", "ipsec-parser", + "itertools", "kerberos-parser", "lazy_static", "ldap-parser", @@ -1571,6 +1620,7 @@ dependencies = [ "num", "num-derive", "num-traits 0.2.19", + "petgraph", "psl", "regex", "sawp", diff --git a/rust/Cargo.toml.in b/rust/Cargo.toml.in index 10014e215c..028c572005 100644 --- a/rust/Cargo.toml.in +++ b/rust/Cargo.toml.in @@ -85,6 +85,9 @@ lru = "~0.16.3" der-parser = { version = "~9.0.0", default-features = false } kerberos-parser = { version = "~0.8.0", default-features = false } +petgraph = "~0.8.2" +itertools = "~0.14.0" + sawp-modbus = "~0.13.1" sawp-pop3 = "~0.13.1" sawp = "~0.13.1" diff --git a/rust/src/utils/flowbits_resolver.rs b/rust/src/utils/flowbits_resolver.rs new file mode 100644 index 0000000000..c87b137a6a --- /dev/null +++ b/rust/src/utils/flowbits_resolver.rs @@ -0,0 +1,343 @@ +/* Copyright (C) 2026 Open Information Security Foundation +* + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +// Author: Shivani Bhardwaj + +use crate::jsonbuilder::*; +use petgraph::algo::{is_cyclic_directed, tarjan_scc, toposort}; +use petgraph::graph::{GraphError, NodeIndex}; +use petgraph::stable_graph::StableDiGraph; +use petgraph::visit::EdgeRef; +use petgraph::Direction; +use std::collections::{HashMap, HashSet}; +use std::os::raw::c_void; + +/// Special Graph Node storing flowbit or signature +#[derive(Debug, Copy, Clone)] +struct SCGNode { + iid: u32, + sid: u32, + nidx: NodeIndex, /* Graph's internal node index */ +} + +/// Edge object to determine a genuine cycle breaker +#[derive(Debug, Copy, Clone)] +struct SCGEdge { + cmd: u8, + fb_id: u32, +} + +#[derive(Debug)] +struct FlowbitSidStore { + graph: StableDiGraph, + iid_map: HashMap, + edge_set: HashSet<(u32, u32, u8, u32)>, +} + +/// Function to create an empty directed Graph +#[no_mangle] +pub unsafe extern "C" fn SCCreateDirectedGraph() -> *mut c_void { + // StableDiGraph is the ideal choice here for there is removal of + // nodes in the line later and this type of graph guarantees to + // not re-use any existing node indices + let fb_sid_store = FlowbitSidStore { + graph: StableDiGraph::new(), + iid_map: HashMap::new(), + edge_set: HashSet::new(), + }; + + /* Make an opaque pointer for C as nothing is changed there */ + return Box::into_raw(Box::new(fb_sid_store)) as *mut c_void; +} + +/// Drop the directed Graph. Called from C. +#[no_mangle] +pub unsafe extern "C" fn SCFreeDirectedGraph(store: *mut c_void) { + let _ = Box::from_raw(store as *mut FlowbitSidStore); +} + +#[no_mangle] +pub unsafe extern "C" fn SCGetOrCreateNodeGraph(graph: *mut c_void, iid: u32, sid: u32) -> i64 { + let g = &mut *(graph as *mut FlowbitSidStore); + + let Some(node_idx) = get_or_create_node(g, iid, sid) else { + SCLogError!("Error adding node; Graph is at full capacity"); + return -2; + }; + + node_idx.index() as i64 +} + +/// Function to get or create a node and add an appropriate directed +/// edge based on its type +#[no_mangle] +pub unsafe extern "C" fn SCCreateNodeEdgeDirectedGraph( + fss_void: *mut c_void, from: u32, to: u32, cmd: u8, fb_id: u32, +) -> i64 { + let fss = &mut *(fss_void as *mut FlowbitSidStore); + + let from_idx = NodeIndex::from(from); + let to_idx = NodeIndex::from(to); + + if !fss.edge_set.insert((from, to, cmd, fb_id)) { + SCLogDebug!( + "Skipped duplicate edge from {:?} -> {:?} for flowbit {:?} with command: {:?}", + from, + to, + fb_id, + cmd + ); + return 0; + } + + match fss + .graph + .try_add_edge(from_idx, to_idx, SCGEdge { cmd, fb_id }) + { + /* edge from a flowbit setter to a flowbit reader */ + Ok(_) => { + SCLogDebug!( + "Created an edge from {:?} -> {:?} for flowbit {:?} with command: {:?}", + from, + to, + fb_id, + cmd + ); + } + Err(GraphError::EdgeIxLimit) => { + SCLogError!("Error adding edge; Graph is at full capacity"); + return -2; + } + Err(GraphError::NodeOutBounds) => { + SCLogError!("Error adding edge; node does not exist"); + return -2; + } + Err(_) => { + SCLogError!("Error adding edge to the Graph"); + return -2; + } + } + + return 0; +} + +fn log_graph( + js: &mut JsonBuilder, graph: &mut StableDiGraph, +) -> Result<(), JsonError> { + SCLogDebug!("Starting the logging.."); + for node in graph.node_weights() { + SCLogDebug!("{:?}", node.nidx.index()); + js.open_object(&node.sid.to_string())?; + js.open_array("in")?; + for edge in graph.edges_directed(node.nidx, Direction::Incoming) { + js.start_object()?; + js.set_uint("id", edge.source().index() as u64)?; + js.set_uint("weight", edge.weight().cmd as u64)?; + js.set_uint("sid", graph[edge.source()].sid as u64)?; + js.close()?; + } + js.close()?; + js.open_array("out")?; + for edge in graph.edges_directed(node.nidx, Direction::Outgoing) { + js.start_object()?; + js.set_uint("id", edge.target().index() as u64)?; + js.set_uint("weight", edge.weight().cmd as u64)?; + js.set_uint("sid", graph[edge.target()].sid as u64)?; + js.close()?; + } + js.close()?; + js.close()?; + } + Ok(()) +} + +#[no_mangle] +pub unsafe extern "C" fn SCDebugLogFlowbitGraph( + jsb: &mut JsonBuilder, fss_void: *mut c_void, +) -> bool { + let fss = &mut *(fss_void as *mut FlowbitSidStore); + log_graph(jsb, &mut fss.graph).is_ok() +} + +fn check_cycle_update_graph( + graph: &mut StableDiGraph, max_cycle_resolution: u8, +) -> i8 { + let max_stack_size = max_cycle_resolution as usize; + + for i in 0..=max_stack_size { + /* Check graph for any cycles */ + if !is_cyclic_directed(&*graph) { + SCLogDebug!("no cycles after {} tries", i); + return 0; + } + + SCLogDebug!("Found a cycle in i {}. Checking if it's valid..", i); + if i == max_stack_size { + break; + } + + if !try_resolve_one_cycle(graph) { + /* If we can't resolve any cycle, we're stuck */ + SCLogError!("Unable to resolve cycles after {} tries", i); + return -1; + } + } + + SCLogError!( + "Maximum tries ({}) reached while trying to resolve cycles", + max_stack_size + ); + return -1; +} + +fn try_resolve_one_cycle(graph: &mut StableDiGraph) -> bool { + let sccs = tarjan_scc(&*graph); + let mut edge_map: HashMap = HashMap::new(); + + /* Find the first multi-node SCC */ + for scc in sccs { + if scc.len() == 1 { + let self_loop_edges: Vec<_> = graph + .edges(scc[0]) + .filter(|edge| edge.target() == scc[0]) + .map(|edge| edge.id()) + .collect(); + + if let Some(edge) = self_loop_edges.into_iter().next() { + graph.remove_edge(edge); + return true; + } + continue; + } + + SCLogDebug!("Current scc: {:?}", scc); + let scc_set: HashSet<_> = scc.iter().copied().collect(); + let edge_indices: Vec<_> = graph.edge_indices().collect(); + + for edge_idx in edge_indices { + let Some((_src, tgt)) = graph.edge_endpoints(edge_idx) else { + continue; + }; + if !(scc_set.contains(&_src) && scc_set.contains(&tgt)) { + continue; + } + let SCGEdge { cmd, fb_id } = graph[edge_idx]; + let same_bit_setters = graph + .edges_directed(tgt, Direction::Incoming) + .filter(|e| e.weight().fb_id == fb_id) + .count(); + if same_bit_setters > 1 { + graph.remove_edge(edge_idx); + return true; + } + /* store cycle edges to inspect their commands (weights) later */ + edge_map.insert(edge_idx, cmd); + } + /* Find if the cycle causing edges are made up of differing weights */ + let distinct_weights = edge_map.values().copied().collect::>().len(); + debug_validate_bug_on!(distinct_weights == 0); + if distinct_weights > 1 { + /* Find and remove the edge with highest weight (lowest priority) */ + if let Some((cur_e, _)) = edge_map.iter().max_by_key(|(_, &weight)| weight) { + graph.remove_edge(*cur_e); + return true; + } + } else { + /* Valid cycle with same weights -- can't resolve */ + let sids: Vec<_> = scc_set.into_iter().map(|a| graph[a].sid).collect(); + SCLogError!( + "Cyclic dependency found between flowbits from signatures: {:?}", + sids + ); + return false; + } + break; + } + + /* couldn't resolve */ + false +} + +/// Wrapper function to resolve flowbit dependencies +#[no_mangle] +pub unsafe extern "C" fn SCResolveFlowbitDependencies( + fss_void: *mut c_void, sorted_iid_list: *mut u32, sorted_iid_list_len: u32, + max_cycle_resolution: u8, +) -> i8 { + SCLogDebug!("Attempting to resolve flowbit dependencies"); + let fss = &mut *(fss_void as *mut FlowbitSidStore); + let r = check_cycle_update_graph(&mut fss.graph, max_cycle_resolution); + if r == -1 { + SCLogError!("Couldn't do anything to fix the graph. Retreating.."); + return -1; + } + + debug_validate_bug_on!(fss.graph.node_count() == 0); + + let sorted_iid_list = + std::slice::from_raw_parts_mut(&mut *sorted_iid_list, sorted_iid_list_len as usize); + + /* No need for all the extra work if there's just one node */ + if fss.graph.node_count() == 1 { + debug_validate_bug_on!(sorted_iid_list_len != 1); + sorted_iid_list[0] = fss.graph[NodeIndex::from(0)].iid; + return 0; + } + + /* At this point, it must be a DAG, so perform a topological sort to find + * out the correct order of signatures */ + return toposort_dag(&fss.graph, sorted_iid_list); +} + +fn get_or_create_node(fss: &mut FlowbitSidStore, iid: u32, sid: u32) -> Option { + if let Some(&nidx) = fss.iid_map.get(&iid) { + return Some(nidx); /* O(1) */ + } + let nd = SCGNode { + iid, + sid, + nidx: NodeIndex::from(u32::MAX), + }; + if let Ok(idx) = fss.graph.try_add_node(nd) { + /* O(1) */ + fss.graph[idx].nidx = idx; + fss.iid_map.insert(iid, idx); + SCLogDebug!("Created node: {:?}", fss.graph[idx]); + return Some(idx); + } + + None +} + +/// Produce a topological ordering of the DAG +fn toposort_dag(graph: &StableDiGraph, sorted_iid_list: &mut [u32]) -> i8 { + match toposort(graph, None) { + Ok(order) => { + debug_validate_bug_on!(order.len() != sorted_iid_list.len()); + for (i, idx) in order.into_iter().enumerate() { + SCLogDebug!("[{:?}]: {:?}", i, graph[idx]); + sorted_iid_list[i] = graph[idx].iid; + } + 0 + } + Err(_) => { + /* Unreachable in practice: cycles are resolved before this point */ + SCLogError!("Graph still contains a cycle; cannot produce an order"); + -1 + } + } +} diff --git a/rust/src/utils/mod.rs b/rust/src/utils/mod.rs index 542aa7791f..36aea88ec4 100644 --- a/rust/src/utils/mod.rs +++ b/rust/src/utils/mod.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2024 Open Information Security Foundation +/* Copyright (C) 2024-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -17,4 +17,5 @@ pub mod base64; pub mod datalink; +pub mod flowbits_resolver; pub mod ipaddr; diff --git a/src/detect-engine-sigorder.c b/src/detect-engine-sigorder.c index e233d28f0a..e931d4f54b 100644 --- a/src/detect-engine-sigorder.c +++ b/src/detect-engine-sigorder.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2024 Open Information Security Foundation +/* Copyright (C) 2007-2026 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -36,9 +36,12 @@ #include "util-unittest-helper.h" #include "util-debug.h" #include "util-action.h" +#include "util-conf.h" #include "action-globals.h" #include "flow-util.h" #include "util-validate.h" +#include "suricata.h" +#include "rust.h" #define DETECT_FLOWVAR_NOT_USED 1 #define DETECT_FLOWVAR_TYPE_READ 2 @@ -65,6 +68,9 @@ #define DETECT_XBITS_TYPE_SET_READ 3 #define DETECT_XBITS_TYPE_SET 4 +extern bool rule_engine_analysis_set; +SCMutex g_flowbits_graph_dump_write_m = SCMUTEX_INITIALIZER; + /** * \brief Different kinds of helper data that can be used by the signature * ordering module. Used by the "user" field in SCSigSignatureWrapper @@ -534,6 +540,16 @@ static int SCSigLessThan(SCSigSignatureWrapper *sw1, { SCSigOrderFunc *funcs = cmp_func_list; + // Special handling for flowbits, do not touch the order if the comparison fn + // calculates two signatures to be equal + if ((ActionOrderVal(sw1->sig->action) == ActionOrderVal(sw2->sig->action)) && + !(sw1->sig->init_data->firewall_rule || sw2->sig->init_data->firewall_rule) && + (sw1->user[DETECT_SIGORDER_FLOWBITS] > DETECT_FLOWBITS_NOT_USED) && + (sw1->user[DETECT_SIGORDER_FLOWBITS] == DETECT_FLOWBITS_TYPE_SET_READ) && + (sw1->user[DETECT_SIGORDER_FLOWBITS] == sw2->user[DETECT_SIGORDER_FLOWBITS])) { + return 1; + } + while (funcs != NULL) { int delta = funcs->SWCompare(sw1, sw2); if (delta > 0) @@ -543,6 +559,7 @@ static int SCSigLessThan(SCSigSignatureWrapper *sw1, funcs = funcs->next; } + // They are equal, so use sid as the final decider. return sw1->sig->id < sw2->sig->id; } @@ -554,38 +571,50 @@ static SCSigSignatureWrapper *SCSigOrder(SCSigSignatureWrapper *sw, { DEBUG_VALIDATE_BUG_ON(sw == NULL); + if (sw->next == NULL) { + /* only one element in the list */ + return sw; + } + SCSigSignatureWrapper *subA = NULL; SCSigSignatureWrapper *subB = NULL; - SCSigSignatureWrapper *first; - SCSigSignatureWrapper *second; SCSigSignatureWrapper *result = NULL; SCSigSignatureWrapper *last = NULL; SCSigSignatureWrapper *new = NULL; - /* Divide input list into two sub-lists. */ - while (sw != NULL) { - first = sw; - sw = sw->next; - /* Push the first element onto sub-list A */ - first->next = subA; - subA = first; - - if (sw == NULL) - break; - second = sw; - sw = sw->next; - /* Push the second element onto sub-list B */ - second->next = subB; - subB = second; - } - if (subB == NULL) { - /* Only zero or one element on the list. */ - return subA; + /* Divide input list into two sub-lists using Tortoise and Hare algorithm */ + SCSigSignatureWrapper *slow = sw; + SCSigSignatureWrapper *fast = sw; /* 2x faster pointer */ + SCSigSignatureWrapper *prev = NULL; + + while (fast != NULL && fast->next != NULL) { + prev = slow; + slow = slow->next; + fast = fast->next->next; } + + prev->next = NULL; /* Cut the first half */ + subA = sw; /* First half */ + subB = slow; /* Second half */ + + SCLogDebug("subA is: %d; subB is: %d", subA->sig->id, subB->sig->id); + DEBUG_VALIDATE_BUG_ON(subA == NULL); + DEBUG_VALIDATE_BUG_ON(subB == NULL); - /* Now sort each list */ +#ifdef DEBUG + SCLogDebug("subA:"); + for (SCSigSignatureWrapper *sigw = subA; sigw != NULL; sigw = sigw->next) { + SCLogDebug("sig_id: %d", sigw->sig->id); + } +#endif subA = SCSigOrder(subA, cmp_func_list); +#ifdef DEBUG + SCLogDebug("subB:"); + for (SCSigSignatureWrapper *sigw = subB; sigw != NULL; sigw = sigw->next) { + SCLogDebug("sig_id: %d", sigw->sig->id); + } +#endif subB = SCSigOrder(subB, cmp_func_list); DEBUG_VALIDATE_BUG_ON(subA == NULL); DEBUG_VALIDATE_BUG_ON(subB == NULL); @@ -618,6 +647,233 @@ static SCSigSignatureWrapper *SCSigOrder(SCSigSignatureWrapper *sw, return result; } +static inline void SCLogFlowbitsGraph(void *graph) +{ + SCJsonBuilder *jb = SCJbNewObject(); + if (jb == NULL) + return; + + SCDebugLogFlowbitGraph(jb, graph); + SCJbClose(jb); + const char *filename = "flowbits_dependency_graph.json"; + const char *log_dir = SCConfigGetLogDirectory(); + char log_path[PATH_MAX] = ""; + snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename); + + SCMutexLock(&g_flowbits_graph_dump_write_m); + FILE *fp = fopen(log_path, "w"); + if (fp != NULL) { + fwrite(SCJbPtr(jb), SCJbLen(jb), 1, fp); + fprintf(fp, "\n"); + fclose(fp); + } + SCMutexUnlock(&g_flowbits_graph_dump_write_m); + SCJbFree(jb); +} + +static int CreateGraphFromFlowbitAnalyzer( + void *graph, SCSigSignatureWrapper *sw, uint32_t max_fb_id) +{ + SCSigSignatureWrapper *tmp = sw; + if (max_fb_id == 0) + return 0; + + struct FBAnalyzer fba = { .array = NULL, .array_size = 0 }; + const uint32_t array_size = max_fb_id + 1; + struct FBAnalyze *array = SCCalloc(array_size, sizeof(struct FBAnalyze)); + if (array == NULL) { + SCLogError("Unable to allocate flowbit analyze array"); + return -1; + } + fba.array = array; + fba.array_size = array_size; + + SCLogDebug( + "fb analyzer array size: %" PRIu64, (uint64_t)(array_size * sizeof(struct FBAnalyze))); + + /* fill flowbit array, updating counters per sig */ + while (tmp != NULL) { + const Signature *s = tmp->sig; + + int64_t node = SCGetOrCreateNodeGraph(graph, s->iid, s->id); + if (node < 0) { + FatalErrorOnInit("Couldn't create or get a node in the graph for sid: %d", s->id); + } + SCLogDebug("added node: %ld", node); + + int r = DetectFlowbitsAnalyzeSignature(s, &fba); + if (r < 0) { + FBAnalyzerFree(&fba); + return -1; + } + tmp = tmp->next; + } + + // varnamestore ids start at 1 + for (uint32_t x = 1; x < array_size; x++) { + for (uint32_t i = 0; i < fba.array[x].isset_iids_idx; i++) { + SCLogDebug("Getting or creating node for sid: %d", fba.array[x].isset_iids[i].sid); + int64_t to = SCGetOrCreateNodeGraph( + graph, fba.array[x].isset_iids[i].iid, fba.array[x].isset_iids[i].sid); + if (to < 0) { + FatalErrorOnInit("Couldn't create or get a node in the graph for sid: %d", + fba.array[x].isset_iids[i].sid); + } + SCLogDebug("added node: %ld", to); + for (uint32_t y = 0; y < fba.array[x].set_iids_idx; y++) { + SCLogDebug("Getting or creating node for iid: %d", fba.array[x].set_iids[y].sid); + int64_t from = SCGetOrCreateNodeGraph( + graph, fba.array[x].set_iids[y].iid, fba.array[x].set_iids[y].sid); + if (from < 0) { + FatalErrorOnInit("Couldn't create or get a node in the graph for sid: %d", + fba.array[x].set_iids[y].sid); + } + SCLogDebug("added node: %ld", from); + if (SCCreateNodeEdgeDirectedGraph( + graph, (uint32_t)from, (uint32_t)to, DETECT_FLOWBITS_CMD_SET, x) < 0) { + if (rule_engine_analysis_set) { + SCLogFlowbitsGraph(graph); + } + FatalErrorOnInit( + "Flowbits signatures that are unsatisfiable at runtime found: %d, %d", + fba.array[x].isset_iids[i].sid, fba.array[x].set_iids[y].sid); + } + } + } + } + + for (uint32_t x = 1; x < array_size; x++) { + for (uint32_t i = 0; i < fba.array[x].isnotset_iids_idx; i++) { + int64_t to = SCGetOrCreateNodeGraph( + graph, fba.array[x].isnotset_iids[i].iid, fba.array[x].isnotset_iids[i].sid); + if (to < 0) { + FatalErrorOnInit("Couldn't create or get a node in the graph for sid: %d", + fba.array[x].isnotset_iids[i].sid); + } + SCLogDebug("added node: %ld", to); + for (uint32_t y = 0; y < fba.array[x].unset_iids_idx; y++) { + int64_t from = SCGetOrCreateNodeGraph( + graph, fba.array[x].unset_iids[y].iid, fba.array[x].unset_iids[y].sid); + if (from < 0) { + FatalErrorOnInit("Couldn't create or get a node in the graph for sid: %d", + fba.array[x].unset_iids[y].sid); + } + SCLogDebug("added node: %ld", from); + if (SCCreateNodeEdgeDirectedGraph(graph, (uint32_t)from, (uint32_t)to, + DETECT_FLOWBITS_CMD_UNSET, x) < 0) { + if (rule_engine_analysis_set) { + SCLogFlowbitsGraph(graph); + } + FatalErrorOnInit( + "Flowbits signatures that are unsatisfiable at runtime found: %d, %d", + fba.array[x].isnotset_iids[i].sid, fba.array[x].unset_iids[y].sid); + } + } + } + } + + FBAnalyzerFree(&fba); + return 0; +} + +/** + * \brief Function to Resolve dependencies among flowbits + * + * \param arg_sw Signature Wrapper containing all flowbits of SET_READ type + * + * \return SCSigSignatureWrapper list post dependency resolution + */ +static SCSigSignatureWrapper *SCSigResolveFlowbitDependencies( + SCSigSignatureWrapper *head, uint32_t max_fb_id, uint8_t max_cycle_resolution) +{ + uint32_t sig_cnt = 0; + uint32_t *sorted_iids = NULL; + SCSigSignatureWrapper *tmp = NULL; + + SCLogInfo("Signatures with flowbits found"); + + if (head == NULL) { + return NULL; + } + if (head->next == NULL) { + return head; + } + tmp = head; + while (tmp != NULL) { + ++sig_cnt; + /* The dependency-resolved order overrides any explicit priority for + * SET_READ flowbit rules. Warn once per such rule that set a priority. */ + if (tmp->user[DETECT_SIGORDER_FLOWBITS] == DETECT_FLOWBITS_TYPE_SET_READ && + (tmp->sig->init_data->init_flags & SIG_FLAG_INIT_PRIO_EXPLICIT)) { + SCLogWarning("sid %u: explicitly set priority is overridden by flowbits " + "dependency resolution ordering", + tmp->sig->id); + } + tmp = tmp->next; + } + SCLogDebug("sig_cnt: %d", sig_cnt); + + DEBUG_VALIDATE_BUG_ON(max_fb_id == 0); + DEBUG_VALIDATE_BUG_ON(sig_cnt == 0); + void *graph = SCCreateDirectedGraph(); + if (CreateGraphFromFlowbitAnalyzer(graph, head, max_fb_id) < 0) { + FatalErrorOnInit("Could not create Flowbit Analyzer Graph"); + } + + sorted_iids = SCCalloc(sig_cnt, sizeof(uint32_t)); + if (sorted_iids == NULL) { + goto error; + } + if (rule_engine_analysis_set) { + SCLogFlowbitsGraph(graph); + } + int ret = SCResolveFlowbitDependencies(graph, sorted_iids, sig_cnt, max_cycle_resolution); + if (ret < 0) { + goto error; + } + + SCSigSignatureWrapper *fin = NULL; + tmp = NULL; + + for (uint32_t i = 0; i < sig_cnt; i++) { + SCSigSignatureWrapper *prev = NULL; + SCSigSignatureWrapper *cur = head; + + while (cur != NULL) { + if (sorted_iids[i] == cur->sig->iid) { + if (prev == NULL) { + head = cur->next; + } else { + prev->next = cur->next; + } + + cur->next = NULL; + if (tmp != NULL) { + tmp->next = cur; + } else { + fin = cur; + } + tmp = cur; + break; + } + prev = cur; + cur = cur->next; + } + } + + SCFree(sorted_iids); /* No longer needed */ + SCFreeDirectedGraph(graph); + return fin; + +error: + SCLogError("Error resolving flowbit dependencies"); + if (sorted_iids != NULL) { + SCFree(sorted_iids); + } + SCFreeDirectedGraph(graph); + return NULL; +} + /** * \brief Orders an incoming Signature based on its action * @@ -808,12 +1064,17 @@ int SCSigOrderSignatures(DetectEngineCtx *de_ctx) int retval = 0; SCLogDebug("ordering signatures in memory"); SCSigSignatureWrapper *sigw = NULL; - SCSigSignatureWrapper *td_sigw_list = NULL; /* unified td list */ + + SCSigSignatureWrapper *td_sigw_list_start = NULL; /* unified td list start */ + SCSigSignatureWrapper *td_sigw_list_end = NULL; /* unified td list end */ + + SCSigSignatureWrapper *fb_sigw_list_start = NULL; /* flowbits list start */ SCSigSignatureWrapper *fw_pf_sigw_list = NULL; /* hook: packet_filter */ SCSigSignatureWrapper *fw_af_sigw_list = NULL; /* hook: app_filter */ Signature *sig = de_ctx->sig_list; + while (sig != NULL) { sigw = SCSigAllocSignatureWrapper(sig); if (sigw == NULL) { @@ -833,11 +1094,26 @@ int SCSigOrderSignatures(DetectEngineCtx *de_ctx) fw_af_sigw_list = sigw; } } else { - sigw->next = td_sigw_list; - td_sigw_list = sigw; + /* Flowbit specific handling. Store the three types of flowbit signatures + * separately for an easy merge of the lists by priority later on */ + if (sigw->user[DETECT_SIGORDER_FLOWBITS] > DETECT_FLOWBITS_NOT_USED) { + sigw->next = fb_sigw_list_start; + fb_sigw_list_start = sigw; + } else { + sigw->next = td_sigw_list_start; + if (!td_sigw_list_end) + td_sigw_list_end = sigw; + td_sigw_list_start = sigw; + } } sig = sig->next; } +#if DEBUG + SCLogDebug("Initial list:"); + for (sigw = fb_sigw_list_start; sigw != NULL; sigw = sigw->next) { + SCLogDebug("sig id: %d, iid: %d", sigw->sig->id, sigw->sig->iid); + } +#endif /* despite having Append in the name, the new Sig/Rule funcs actually prepend with some special * logic around bidir sigs. So to respect the firewall rule order, we sort this part of the list @@ -850,10 +1126,41 @@ int SCSigOrderSignatures(DetectEngineCtx *de_ctx) SCSigOrderFunc OrderFn = { .SWCompare = SCSigOrderByAppFirewall, .next = NULL }; fw_af_sigw_list = SCSigOrder(fw_af_sigw_list, &OrderFn); } - if (td_sigw_list) { - /* Sort the list */ - td_sigw_list = SCSigOrder(td_sigw_list, de_ctx->sc_sig_order_funcs); + if (fb_sigw_list_start) { + /* Resolve any complex dependencies, if possible, or roll back to the original ruleset */ + SCSigSignatureWrapper *tmp = SCSigResolveFlowbitDependencies( + fb_sigw_list_start, de_ctx->max_fb_id, de_ctx->max_flowbits_cycle_resolution); + if (tmp == NULL) { + SC_ATOMIC_EXTERN(unsigned int, engine_stage); + if (SC_ATOMIC_GET(engine_stage) == SURICATA_INIT) { + FatalErrorOnInit( + "Flowbits have circular dependencies that cannot be met at runtime"); + } else { + SCLogWarning("Applying the existing ruleset as flowbit dependencies could not be " + "resolved"); + } + } else { + fb_sigw_list_start = tmp; + } } + +#if DEBUG + SCLogDebug("Right BEFORE SigOrder:"); + for (sigw = fb_sigw_list_start; sigw != NULL; sigw = sigw->next) { + SCLogDebug("sig id: %d", sigw->sig->id); + } +#endif + + if (td_sigw_list_end) { + td_sigw_list_end->next = fb_sigw_list_start; + } else if (fb_sigw_list_start) { + DEBUG_VALIDATE_BUG_ON(td_sigw_list_start != NULL); + td_sigw_list_start = fb_sigw_list_start; + } + if (td_sigw_list_start) { + td_sigw_list_start = SCSigOrder(td_sigw_list_start, de_ctx->sc_sig_order_funcs); + } + /* Recreate the sig list in order */ de_ctx->sig_list = NULL; @@ -893,7 +1200,7 @@ cleanup: SCFree(sigw_to_free); } /* threat detect list for hook app_td */ - for (sigw = td_sigw_list; sigw != NULL;) { + for (sigw = td_sigw_list_start; sigw != NULL;) { sigw->sig->next = NULL; if (de_ctx->sig_list == NULL) { /* First entry on the list */ diff --git a/src/detect-engine.c b/src/detect-engine.c index f8ce5d9cf3..8f43a1990f 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -94,6 +94,7 @@ #define DETECT_ENGINE_DEFAULT_INSPECTION_RECURSION_LIMIT 3000 +#define DEFAULT_MAX_FLOWBITS_CYCLE_RESOLUTION_ATTEMPTS 100 #define DEFAULT_MAX_FLOWBITS_PER_SIGNATURE 8 static int DetectEngineCtxLoadConf(DetectEngineCtx *); @@ -2519,6 +2520,27 @@ static void DetectEngineLoadFlowbitSettings(DetectEngineCtx *de_ctx) } } SCLogConfig("Setting flowbits.max-per-signature to %d", de_ctx->max_flowbits); + + de_ctx->max_flowbits_cycle_resolution = DEFAULT_MAX_FLOWBITS_CYCLE_RESOLUTION_ATTEMPTS; + char cycle_varname[128] = "detect.flowbits.max-cycle-resolution"; + if (strlen(de_ctx->config_prefix) > 0) { + snprintf(cycle_varname, sizeof(cycle_varname), "%s.detect.flowbits.max-cycle-resolution", + de_ctx->config_prefix); + } + if (SCConfGet(cycle_varname, &str) == 1) { + uint8_t val = 0; + int ret = StringParseUint8(&val, 10, 0, str); + if (ret > 0) { + /* 0 is a valid setting: cycles are reported without any attempt to + * break them */ + de_ctx->max_flowbits_cycle_resolution = val; + } else { + SCLogWarning( + "Invalid setting for flowbits.max-cycle-resolution, resetting to the default"); + } + } + SCLogConfig( + "Setting flowbits.max-cycle-resolution to %d", de_ctx->max_flowbits_cycle_resolution); } /** \internal diff --git a/src/detect.h b/src/detect.h index 605b54693a..d270fe80fc 100644 --- a/src/detect.h +++ b/src/detect.h @@ -993,91 +993,35 @@ typedef uint8_t (*SCDetectRateFilterFunc)(const Packet *p, uint32_t sid, uint32_ /** \brief main detection engine ctx */ typedef struct DetectEngineCtx_ { - bool failure_fatal; - uint8_t flags; /**< only DE_QUIET */ - uint8_t mpm_matcher; /**< mpm matcher this ctx uses */ - uint8_t max_flowbits; /**< maximum number of flowbits per signature */ - uint32_t tenant_id; - MpmConfig *mpm_cfg; - uint8_t spm_matcher; /**< spm matcher this ctx uses */ - Signature *sig_list; - uint32_t sig_cnt; - - /* version of the srep data */ - uint32_t srep_version; /* reputation for netblocks */ SRepCIDRTree *srepCIDR_ctx; - Signature **sig_array; - uint32_t sig_array_len; /* size in array members */ - - uint32_t signum; /* used by the signature ordering module */ struct SCSigOrderFunc_ *sc_sig_order_funcs; - /* main sigs */ - DetectEngineLookupFlow flow_gh[FLOW_STATES]; - /** firewall policy table entry point */ struct DetectFirewallPolicies *fw_policies; /* init phase vars */ HashListTable *sgh_hash_table; - HashListTable *mpm_hash_table; HashListTable *pattern_hash_table; /* hash table used to cull out duplicate sigs */ HashListTable *dup_sig_hash_table; - DetectEngineIPOnlyCtx io_ctx; - - /* maximum recursion depth for content inspection */ - int inspection_recursion_limit; - - /* maximum number of times a tx will get logged for rules not using app-layer keywords */ - uint8_t guess_applayer_log_limit; - - /* force app-layer tx finding for alerts with signatures not having app-layer keywords */ - bool guess_applayer; - - /* registration id for per thread ctx for the filemagic/file.magic keywords */ - int filemagic_thread_ctx_id; - /* spm thread context prototype, built as spm matchers are constructed and * later used to construct thread context for each thread. */ SpmGlobalThreadCtx *spm_global_thread_ctx; - - /* Config options */ - - uint16_t max_uniq_toclient_groups; - uint16_t max_uniq_toserver_groups; - - /* max flowbit id that is used */ - uint32_t max_fb_id; - MpmCtxFactoryContainer *mpm_ctx_factory_container; /* array containing all sgh's in use so we can loop * through it in Stage4. */ struct SigGroupHead_ **sgh_array; - uint32_t sgh_array_cnt; - uint32_t sgh_array_size; - - int32_t sgh_mpm_context_proto_tcp_packet; - int32_t sgh_mpm_context_proto_udp_packet; - int32_t sgh_mpm_context_proto_other_packet; - int32_t sgh_mpm_context_stream; - - /* the max local id used amongst all sigs */ - int32_t byte_extract_max_local_id; - - /** version of the detect engine. The version is incremented on reloads */ - uint32_t version; /** sgh for signatures that match against invalid packets. In those cases * we can't lookup by proto, address, port as we don't have these */ @@ -1085,29 +1029,12 @@ typedef struct DetectEngineCtx_ { /** sgh for `alert ether` / `alert arp` etc. */ struct SigGroupHead_ *eth_non_ip_sgh; - - /* Maximum size of the buffer for decoded base64 data. */ - uint16_t base64_decode_max_len; - - /** Store rule file and line so that parsers can use them in errors. */ - int rule_line; const char *rule_file; const char *sigerror; - bool sigerror_silent; - bool sigerror_ok; - /** The rule errored out due to missing requirements. */ - bool sigerror_requires; - - /* specify the configuration for mpm context factory */ - uint8_t sgh_mpm_ctx_cnf; - - int keyword_id; /** hash list of keywords that need thread local ctxs */ HashListTable *keyword_hash; - DetectFileDataCfg *filedata_config; - #ifdef PROFILE_RULES struct SCProfileDetectCtx_ *profile_ctx; #endif @@ -1118,23 +1045,10 @@ typedef struct DetectEngineCtx_ { struct SCProfileSghDetectCtx_ *profile_sgh_ctx; uint32_t profile_match_logging_threshold; #endif - char config_prefix[64]; - - enum DetectEngineType type; - /** how many de_ctx' are referencing this */ - uint32_t ref_cnt; /** list in master: either active or freelist */ struct DetectEngineCtx_ *next; - - /** id of loader thread 'owning' this de_ctx */ - int loader_id; - - /** are we using just mpm or also other prefilters */ - enum DetectEnginePrefilterSetting prefilter_setting; - HashListTable *dport_hash_table; - DetectPort *tcp_priorityports; DetectPort *udp_priorityports; @@ -1148,29 +1062,17 @@ typedef struct DetectEngineCtx_ { * is in detect-engine.c::g_buffer_type_hash */ HashListTable *buffer_type_hash_name; HashListTable *buffer_type_hash_id; - uint32_t buffer_type_id; - - uint32_t app_mpms_list_cnt; DetectBufferMpmRegistry *app_mpms_list; + /* list with app inspect engines. Both the start-time registered ones and * the rule-time registered ones. */ DetectEngineAppInspectionEngine *app_inspect_engines; DetectEnginePktInspectionEngine *pkt_inspect_engines; DetectBufferMpmRegistry *pkt_mpms_list; - uint32_t pkt_mpms_list_cnt; DetectEngineFrameInspectionEngine *frame_inspect_engines; DetectBufferMpmRegistry *frame_mpms_list; - uint32_t frame_mpms_list_cnt; - - uint32_t prefilter_id; HashListTable *prefilter_hash_table; - /** time of last ruleset reload */ - struct timeval last_reload; - - /** signatures stats */ - SigFileLoaderStat sig_stat; - /* list of Fast Pattern registrations. Initially filled using a copy of * `g_fp_support_smlist_list`, then extended at rule loading time if needed */ SCFPSupportSMList *fp_support_smlist_list; @@ -1204,13 +1106,9 @@ typedef struct DetectEngineCtx_ { /* Track rule requirements for reporting after loading rules. */ SCDetectRequiresStatus *requirements; - /* number of signatures using filestore, limited as u16 */ - uint16_t filestore_cnt; - /* name store for non-prefilter engines. Used in profiling but * part of the API, so hash is always used. */ HashTable *non_pf_engine_names; - const char *firewall_rule_file_exclusive; /* user provided rate filter callbacks. */ @@ -1221,13 +1119,103 @@ typedef struct DetectEngineCtx_ { /* Hook for pre_stream engine if it is used. */ DetectPacketHookFunc PreStreamHook; - /** TCP pre_stream hook rule groups. One per direction. */ - struct SigGroupHead_ *pre_stream_sgh[2]; /* Hook for pre_flow engine if it is used. */ DetectPacketHookFunc PreFlowHook; + /** pre_flow hook rule groups. Before flow we don't know a direction yet. */ struct SigGroupHead_ *pre_flow_sgh; + + /** time of last ruleset reload */ + struct timeval last_reload; + + /** TCP pre_stream hook rule groups. One per direction. */ + struct SigGroupHead_ *pre_stream_sgh[2]; + + /** signatures stats */ + SigFileLoaderStat sig_stat; + DetectEngineIPOnlyCtx io_ctx; + + /* main sigs */ + DetectEngineLookupFlow flow_gh[FLOW_STATES]; + uint32_t tenant_id; + uint32_t sig_cnt; + + /* version of the srep data */ + uint32_t srep_version; + uint32_t sig_array_len; /* size in array members */ + uint32_t signum; + + /* maximum recursion depth for content inspection */ + int inspection_recursion_limit; + + /* registration id for per thread ctx for the filemagic/file.magic keywords */ + int filemagic_thread_ctx_id; + + /* max flowbit id that is used */ + uint32_t max_fb_id; + uint32_t sgh_array_cnt; + uint32_t sgh_array_size; + int32_t sgh_mpm_context_proto_tcp_packet; + int32_t sgh_mpm_context_proto_udp_packet; + int32_t sgh_mpm_context_proto_other_packet; + int32_t sgh_mpm_context_stream; + + /* the max local id used amongst all sigs */ + int32_t byte_extract_max_local_id; + + /** version of the detect engine. The version is incremented on reloads */ + uint32_t version; + + /** Store rule file and line so that parsers can use them in errors. */ + int rule_line; + int keyword_id; + enum DetectEngineType type; + + /** how many de_ctx' are referencing this */ + uint32_t ref_cnt; + + /** id of loader thread 'owning' this de_ctx */ + int loader_id; + + /** are we using just mpm or also other prefilters */ + enum DetectEnginePrefilterSetting prefilter_setting; + uint32_t buffer_type_id; + uint32_t app_mpms_list_cnt; + uint32_t pkt_mpms_list_cnt; + uint32_t frame_mpms_list_cnt; + uint32_t prefilter_id; + + /* Config options */ + uint16_t max_uniq_toclient_groups; + uint16_t max_uniq_toserver_groups; + + /* Maximum size of the buffer for decoded base64 data. */ + uint16_t base64_decode_max_len; + + /* number of signatures using filestore, limited as u16 */ + uint16_t filestore_cnt; + bool failure_fatal; + uint8_t flags; /**< only DE_QUIET */ + uint8_t mpm_matcher; /**< mpm matcher this ctx uses */ + uint8_t max_flowbits; /**< maximum number of flowbits per signature */ + uint8_t spm_matcher; /**< spm matcher this ctx uses */ + + /* maximum number of times a tx will get logged for rules not using app-layer keywords */ + uint8_t guess_applayer_log_limit; + + /* force app-layer tx finding for alerts with signatures not having app-layer keywords */ + bool guess_applayer; + uint8_t max_flowbits_cycle_resolution; + bool sigerror_silent; + bool sigerror_ok; + + /** The rule errored out due to missing requirements. */ + bool sigerror_requires; + + /* specify the configuration for mpm context factory */ + uint8_t sgh_mpm_ctx_cnf; + char config_prefix[64]; } DetectEngineCtx; /** diff --git a/suricata.yaml.in b/suricata.yaml.in index 49c1df696c..9110cd3330 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1906,6 +1906,12 @@ detect: #flowbits: # max-per-signature: 8 # maximum number of flowbits keyword usage per signature, minimum value allowed is 1 + # In order to determine the correct order of signatures when flowbits are present + # and may have inter dependencies, Suricata makes a dependency graph of the + # signatures with flowbits and attempts to resolve any cycles that can be + # fixed to give a plausible order of signatures. This value determines how many + # times Suricata will try to resolve cycles before it gives up. + #max-cycle-resolution: 100 # Select the multi pattern algorithm you want to run for scan/search the # in the engine.