diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index a9ad630fcf..ed58c5af5e 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -3068,6 +3068,79 @@ jobs: !suricata-verify/live/tests/**/output/sur if-no-files-found: ignore + ubuntu-xdp: + name: Ubuntu (xdp) + runs-on: ubuntu-latest + needs: [prepare-deps] + steps: + # Cache Rust stuff. + - name: Cache cargo registry + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 + with: + path: ~/.cargo/registry + key: cargo-registry + + - name: Determine number of CPUs + run: echo CPUS=$(nproc --all) >> $GITHUB_ENV + + - run: sudo apt update + - run: | + sudo apt -y install \ + autoconf \ + automake \ + build-essential \ + cmake \ + curl \ + git \ + hwloc \ + libhwloc-dev \ + jq \ + make \ + libpcre3 \ + libpcre3-dbg \ + libpcre3-dev \ + libpcre2-dev \ + libtool \ + libpcap-dev \ + libnet1-dev \ + libyaml-0-2 \ + libyaml-dev \ + libcap-ng-dev \ + libcap-ng0 \ + libjansson-dev \ + libjansson4 \ + libnuma-dev \ + liblz4-dev \ + libssl-dev \ + pkg-config \ + python3 \ + python3-yaml \ + tcpreplay \ + zlib1g \ + zlib1g-dev \ + clang \ + libxdp-dev + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 + - run: git config --global --add safe.directory /__w/suricata/suricata + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 + with: + name: prep + path: prep + - name: Install Rust + run: curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain $(grep rust-version rust/Cargo.toml.in|sed 's/\"//g'|awk '{print $3}') -y + - run: echo "$HOME/.cargo/bin" >> $GITHUB_PATH + - uses: ./.github/actions/install-cbindgen + - run: tar xf prep/suricata-verify.tar.gz + - run: ./autogen.sh + - run: CFLAGS="${DEFAULT_CFLAGS} -DAFPACKET_TEST_REPLAY" ./configure --enable-warnings --enable-unittests --enable-ebpf --enable-ebpf-build + - run: make -j ${{ env.CPUS }} + - run: make check + - name: Running suricata-verify + run: python3 ./suricata-verify/run.py -q --debug-failed + - run: ulimit -a + - name: Running suricata-verify live tests + run: sudo python3 ./suricata-verify/run.py --live --debug-failed + debian-12: name: Debian 12 (xdp) runs-on: ubuntu-latest diff --git a/src/flow-manager.c b/src/flow-manager.c index 1c04b6dbd6..32159ee2b1 100644 --- a/src/flow-manager.c +++ b/src/flow-manager.c @@ -359,7 +359,12 @@ static void FlowManagerHashRowTimeout(FlowManagerTimeoutThread *td, Flow *f, SCT * be modified when we have both the flow and hash row lock */ /* timeout logic goes here */ - if (!FlowManagerFlowTimeout(f, ts, next_ts, emergency)) { + bool bypass_shutdown = false; +#ifdef CAPTURE_OFFLOAD + bypass_shutdown = (f->flow_state == FLOW_STATE_CAPTURE_BYPASSED) && + (SC_ATOMIC_GET(flow_flags) & FLOW_SHUTDOWN); +#endif + if (!bypass_shutdown && !FlowManagerFlowTimeout(f, ts, next_ts, emergency)) { FLOWLOCK_UNLOCK(f); counters->flows_notimeout++; @@ -450,21 +455,25 @@ static uint32_t FlowTimeoutHash(FlowManagerTimeoutThread *td, SCTime_t ts, const #endif const uint32_t ts_secs = (uint32_t)SCTIME_SECS(ts); + const bool shutdown = (SC_ATOMIC_GET(flow_flags) & FLOW_SHUTDOWN); for (uint32_t idx = hash_min; idx < hash_max; idx+=BITS) { TYPE check_bits = 0; const uint32_t check = MIN(BITS, (hash_max - idx)); - for (uint32_t i = 0; i < check; i++) { - FlowBucket *fb = &flow_hash[idx+i]; - check_bits |= (TYPE)(SC_ATOMIC_LOAD_EXPLICIT( - fb->next_ts, SC_ATOMIC_MEMORY_ORDER_RELAXED) <= ts_secs) - << (TYPE)i; + if (!shutdown) { + for (uint32_t i = 0; i < check; i++) { + FlowBucket *fb = &flow_hash[idx + i]; + check_bits |= (TYPE)(SC_ATOMIC_LOAD_EXPLICIT(fb->next_ts, + SC_ATOMIC_MEMORY_ORDER_RELAXED) <= ts_secs) + << (TYPE)i; + } + if (check_bits == 0) + continue; } - if (check_bits == 0) - continue; - for (uint32_t i = 0; i < check; i++) { FlowBucket *fb = &flow_hash[idx+i]; - if ((check_bits & ((TYPE)1 << (TYPE)i)) != 0 && SC_ATOMIC_GET(fb->next_ts) <= ts_secs) { + if (((check_bits & ((TYPE)1 << (TYPE)i)) != 0 && + SC_ATOMIC_GET(fb->next_ts) <= ts_secs) || + shutdown) { FBLOCK_LOCK(fb); Flow *evicted = NULL; if (fb->evicted != NULL || fb->head != NULL) { @@ -974,6 +983,13 @@ static TmEcode FlowManager(ThreadVars *th_v, void *thread_data) } if (TmThreadsCheckFlag(th_v, THV_KILL)) { + SC_ATOMIC_OR(flow_flags, FLOW_SHUTDOWN); +#ifdef CAPTURE_OFFLOAD + /* Pass through all flows to gather counters from bypassed flows */ + FlowTimeoutCounters counters = { 0 }; + FlowTimeoutHash(&ftd->timeout, ts, ftd->min, ftd->max, &counters); + FlowCountersUpdate(th_v, ftd, &counters); +#endif /* CAPTURE_OFFLOAD */ StatsSyncCounters(&th_v->stats); break; } diff --git a/src/flow-private.h b/src/flow-private.h index b48e18fed6..9852760726 100644 --- a/src/flow-private.h +++ b/src/flow-private.h @@ -33,6 +33,9 @@ * flows for new flows and/or it's memcap limit it reached. In this state the * flow engine with evaluate flows with lower timeout settings. */ #define FLOW_EMERGENCY 0x01 +/** FlowManager is in shutdown stage. This results in a final pass of all + * flows in the flow table, during which statistics from flows are collected. */ +#define FLOW_SHUTDOWN 0x02 /* Flow Time out values */ #define FLOW_DEFAULT_NEW_TIMEOUT 30 diff --git a/src/runmode-af-packet.c b/src/runmode-af-packet.c index 440a345971..8aa1399aaf 100644 --- a/src/runmode-af-packet.c +++ b/src/runmode-af-packet.c @@ -593,7 +593,13 @@ static void *ParseAFPConfig(const char *iface) SCLogError("%s: XDP support is not built-in", iface); #endif } - +#ifdef AFPACKET_TEST_REPLAY + if ((SCConfGetChildValueIntWithDefault(if_root, if_default, "max-packets", &value)) == 1) { + aconf->max_packets = (uint32_t)value; + } else { + aconf->max_packets = 0; + } +#endif if ((SCConfGetChildValueIntWithDefault(if_root, if_default, "buffer-size", &value)) == 1) { aconf->buffer_size = (int)value; } else { diff --git a/src/source-af-packet.c b/src/source-af-packet.c index eb87c750e8..d8f9141b6c 100644 --- a/src/source-af-packet.c +++ b/src/source-af-packet.c @@ -353,7 +353,13 @@ typedef struct AFPThreadVars_ int ebpf_filter_fd; struct ebpf_timeout_config ebpf_t_config; #endif - +#ifdef AFPACKET_TEST_REPLAY + // Number of packets to read on afpacket interface before stopping Suricata + // This is meant for test purposes, especially to run suricata-verify against tcpreplay + // to test features that are not available when reading from a pcap (such as ebpf) + // Value 0 means no packet (and should thus not be used). + uint32_t max_packets; +#endif } AFPThreadVars; static TmEcode ReceiveAFPThreadInit(ThreadVars *, const void *, void **); @@ -1428,6 +1434,15 @@ TmEcode ReceiveAFPLoop(ThreadVars *tv, void *data, void *slot) } else if (r > 0) { StatsCounterIncr(&ptv->tv->stats, ptv->capture_afp_poll_data); r = AFPReadFunc(ptv); +#ifdef AFPACKET_TEST_REPLAY + // When we reached the configured maximum number of packets + // we tell Suricata to stop. + // This is useful for test purposes when tcpreplaying a pcap + // so we know we do not expect anymore packets. + if (ptv->max_packets > 0 && ptv->pkts >= ptv->max_packets) { + suricata_ctl_flags |= SURICATA_STOP; + } +#endif switch (r) { case AFP_READ_OK: /* Trigger one dump of stats every second */ @@ -2566,6 +2581,9 @@ TmEcode ReceiveAFPThreadInit(ThreadVars *tv, const void *initdata, void **data) ptv->promisc = afpconfig->promisc; ptv->checksum_mode = afpconfig->checksum_mode; ptv->bpf_filter = NULL; +#ifdef AFPACKET_TEST_REPLAY + ptv->max_packets = afpconfig->max_packets; +#endif ptv->threads = 1; #ifdef HAVE_PACKET_FANOUT diff --git a/src/source-af-packet.h b/src/source-af-packet.h index 8645bdcffc..3fbf3c068f 100644 --- a/src/source-af-packet.h +++ b/src/source-af-packet.h @@ -120,6 +120,9 @@ typedef struct AFPIfaceConfig_ const char *out_iface; #ifdef HAVE_PACKET_EBPF struct ebpf_timeout_config ebpf_t_config; +#endif +#ifdef AFPACKET_TEST_REPLAY + uint32_t max_packets; #endif SC_ATOMIC_DECLARE(unsigned int, ref); void (*DerefFunc)(void *); diff --git a/src/suricata.c b/src/suricata.c index e9a377421d..6f66d516ef 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -855,6 +855,9 @@ static void PrintBuildInfo(void) #ifdef HAVE_PACKET_EBPF strlcat(features, "EBPF ", sizeof(features)); #endif +#ifdef AFPACKET_TEST_REPLAY + strlcat(features, "AFPACKET_TEST_REPLAY ", sizeof(features)); +#endif #ifdef PROFILE_LOCKING strlcat(features, "PROFILE_LOCKING ", sizeof(features)); #endif