diff --git a/doc/userguide/capture-hardware/dpdk.rst b/doc/userguide/capture-hardware/dpdk.rst index 21f5e198a9..73b7ab4049 100644 --- a/doc/userguide/capture-hardware/dpdk.rst +++ b/doc/userguide/capture-hardware/dpdk.rst @@ -239,3 +239,44 @@ Encapsulation stripping Suricata supports stripping the hardware-offloaded encapsulation stripping on the supported NICs. Currently, VLAN encapsulation stripping is supported. VLAN encapsulation stripping can be enabled with `vlan-strip-offload`. + +.. _dpdk-segmented-mbufs: + +Segmented mbufs +--------------- + +A packet larger than one mbuf arrives as a chain of segmented mbufs. Suricata +sizes its mbufs from the configured ``mtu``. +Setting the MTU to the largest expected packet size is beneficial for +the performance so that no segmented mbufs occur. +Segmented mbufs still occur when a virtual PMD does not enforce +the MTU, or when the primary DPDK application uses small mbufs, so +avoiding mbuf segments is not always possible. + +Suricata merges the chain into one contiguous mbuf or, when that is not +possible, inspects a contiguous copy of the packet. + +In the copy modes (IPS/TAP), Suricata forwards a chain it cannot merge only +with ``tx-segmented-mbufs: true``. In IPS mode, the chain carries the +modifications made by Suricata (e.g. stream normalization or the ``replace`` +keyword). +To transmit segmented mbufs, Suricata enables the multi-segment TX offload +and disables the ``MBUF_FAST_FREE`` TX offload on the copy interfaces. On some +NICs, this selects a slower TX path, so enable ``tx-segmented-mbufs`` only when +Suricata drops segmented mbufs because it is disabled. +When an interface does not support multi-segment TX, Suricata attempts to +forward segmented mbufs anyway. +Both copy interfaces must have the same ``tx-segmented-mbufs`` setting. + +Suricata drops a chain it cannot merge, e.g. when: + +- in the copy modes, ``tx-segmented-mbufs`` is ``false`` (the default), counted + in ``capture.dpdk.segmented_drops``; +- the chain has more segments than the copy interface can transmit, counted in + ``capture.dpdk.segmented_drops``; +- the packet is larger than the maximum packet size Suricata can process + (about 64 KiB), counted in ``capture.dpdk.segmented_too_large``. + +In each worker thread, the first drop for each reason logs a warning that names +the reason. In IPS mode, these packets are also counted in ``ips.blocked`` and +``ips.drop_reason.capture_error``. diff --git a/doc/userguide/configuration/suricata-yaml.rst b/doc/userguide/configuration/suricata-yaml.rst index c0f6e493da..3e88339759 100644 --- a/doc/userguide/configuration/suricata-yaml.rst +++ b/doc/userguide/configuration/suricata-yaml.rst @@ -2378,6 +2378,7 @@ encapsulates 2 main subnodes, and those are eal-params and interfaces. tx-descriptors: auto copy-mode: none copy-iface: none # or PCIe address of the second interface + tx-segmented-mbufs: false The `DPDK arguments @@ -2414,8 +2415,9 @@ As a result, there are extra configuration options (how NICs can be configured) in the items (interfaces) of the ``dpdk.interfaces`` list. At the start of the configuration process, all NIC offloads are disabled to prevent any packet modification. According to the configuration, checksum -validation offload can be enabled to drop invalid packets. Other offloads can -not currently be enabled. +validation offload can be enabled to drop invalid packets. In the copy modes, +``tx-segmented-mbufs`` enables the multi-segment TX offload, see +:ref:`dpdk-segmented-mbufs`. Other offloads can not currently be enabled. Additionally, the list items in ``dpdk.interfaces`` contain DPDK specific settings such as ``mempool-size`` or ``rx-descriptors``. These settings adjust individual parameters of EAL. One of the entries in ``dpdk.interfaces`` is diff --git a/etc/schema.json b/etc/schema.json index 0b51377cb5..6de3a3c882 100644 --- a/etc/schema.json +++ b/etc/schema.json @@ -6991,6 +6991,16 @@ "type": "integer", "description": "Number of RX packet-buffer allocation failures reported by DPDK" + }, + "segmented_drops": { + "type": "integer", + "description": + "Number of packets received in segmented mbufs that were dropped without processing, excluding those counted in segmented_too_large" + }, + "segmented_too_large": { + "type": "integer", + "description": + "Number of packets received in segmented mbufs that were dropped as larger than the maximum packet size" } } }, @@ -8686,6 +8696,11 @@ "description": "Number of packets dropped due to applayer memcap" }, + "capture_error": { + "type": "integer", + "description": + "Number of packets the capture method dropped before inspection because it could not receive or forward them in full" + }, "decode_error": { "type": "integer", "description": diff --git a/src/decode.c b/src/decode.c index 9d7b2df244..d8620989b7 100644 --- a/src/decode.c +++ b/src/decode.c @@ -974,6 +974,8 @@ const char *PacketDropReasonToString(enum PacketDropReason r) return "threshold detection_filter"; case PKT_DROP_REASON_NFQ_ERROR: return "nfq error"; + case PKT_DROP_REASON_CAPTURE_ERROR: + return "capture error"; case PKT_DROP_REASON_INNER_PACKET: return "tunnel packet drop"; case PKT_DROP_REASON_FW_RULES: @@ -1030,6 +1032,8 @@ static const char *PacketDropReasonToJsonString(enum PacketDropReason r) return "ips.drop_reason.threshold_detection_filter"; case PKT_DROP_REASON_NFQ_ERROR: return "ips.drop_reason.nfq_error"; + case PKT_DROP_REASON_CAPTURE_ERROR: + return "ips.drop_reason.capture_error"; case PKT_DROP_REASON_INNER_PACKET: return "ips.drop_reason.tunnel_packet_drop"; case PKT_DROP_REASON_FW_RULES: diff --git a/src/decode.h b/src/decode.h index fb7b1466ef..8d14403d96 100644 --- a/src/decode.h +++ b/src/decode.h @@ -397,8 +397,9 @@ enum PacketDropReason { PKT_DROP_REASON_STREAM_MIDSTREAM, PKT_DROP_REASON_STREAM_REASSEMBLY, PKT_DROP_REASON_STREAM_URG, - PKT_DROP_REASON_NFQ_ERROR, /**< no nfq verdict, must be error */ - PKT_DROP_REASON_INNER_PACKET, /**< drop issued by inner (tunnel) packet */ + PKT_DROP_REASON_NFQ_ERROR, /**< no nfq verdict, must be error */ + PKT_DROP_REASON_CAPTURE_ERROR, /**< capture can't receive or forward the packet in full */ + PKT_DROP_REASON_INNER_PACKET, /**< drop issued by inner (tunnel) packet */ /** If more non-firewall drop reasons are added, make sure not to "break" PKT_DROP_REASON_NON_FW_MAX */ /* Limiter for non-firewall drop reasons. */ diff --git a/src/runmode-dpdk.c b/src/runmode-dpdk.c index 2b6d738b0a..cd5bc3b659 100644 --- a/src/runmode-dpdk.c +++ b/src/runmode-dpdk.c @@ -121,6 +121,7 @@ static void DPDKDerefConfig(void *conf); #define DPDK_CONFIG_DEFAULT_CHECKSUM_VALIDATION 1 #define DPDK_CONFIG_DEFAULT_CHECKSUM_VALIDATION_OFFLOAD 1 #define DPDK_CONFIG_DEFAULT_VLAN_STRIP 0 +#define DPDK_CONFIG_DEFAULT_TX_SEGMENTED_MBUFS 0 #define DPDK_CONFIG_DEFAULT_LINKUP_TIMEOUT 0 #define DPDK_CONFIG_DEFAULT_COPY_MODE "none" #define DPDK_CONFIG_DEFAULT_COPY_INTERFACE "none" @@ -134,6 +135,7 @@ DPDKIfaceConfigAttributes dpdk_yaml = { .checksum_checks_offload = "checksum-checks-offload", .mtu = "mtu", .vlan_strip_offload = "vlan-strip-offload", + .tx_segmented_mbufs = "tx-segmented-mbufs", .rss_hf = "rss-hash-functions", .linkup_timeout = "linkup-timeout", .mempool_size = "mempool-size", @@ -353,6 +355,7 @@ static void ConfigInit(DPDKIfaceConfig **iconf) FatalError("Could not allocate memory for DPDKIfaceConfig"); ptr->out_port_id = UINT16_MAX; // make sure no port is set + ptr->out_tx_seg_max = UINT16_MAX; SC_ATOMIC_INIT(ptr->ref); (void)SC_ATOMIC_ADD(ptr->ref, 1); ptr->DerefFunc = DPDKDerefConfig; @@ -828,6 +831,13 @@ static void ConfigSetVlanStrip(DPDKIfaceConfig *iconf, int entry_bool) SCReturn; } +static void ConfigSetTxSegmentedMbufs(DPDKIfaceConfig *iconf, int entry_bool) +{ + SCEnter(); + iconf->tx_segmented_mbufs = entry_bool; + SCReturn; +} + static int ConfigSetCopyIface(DPDKIfaceConfig *iconf, const char *entry_str) { SCEnter(); @@ -1051,6 +1061,14 @@ static int ConfigLoad(DPDKIfaceConfig *iconf, const char *iface) ConfigSetVlanStrip(iconf, entry_bool); } + retval = SCConfGetChildValueBoolWithDefault( + if_root, if_default, dpdk_yaml.tx_segmented_mbufs, &entry_bool); + if (retval != 1) { + ConfigSetTxSegmentedMbufs(iconf, DPDK_CONFIG_DEFAULT_TX_SEGMENTED_MBUFS); + } else { + ConfigSetTxSegmentedMbufs(iconf, entry_bool); + } + retval = SCConfGetChildValueIntWithDefault( if_root, if_default, dpdk_yaml.linkup_timeout, &entry_int) != 1 ? ConfigSetLinkupTimeout(iconf, DPDK_CONFIG_DEFAULT_LINKUP_TIMEOUT) @@ -1450,6 +1468,24 @@ static void PortConfSetVlanOffload(const DPDKIfaceConfig *iconf, } } +static void PortConfSetTxOffloads(const DPDKIfaceConfig *iconf, + const struct rte_eth_dev_info *dev_info, struct rte_eth_conf *port_conf) +{ + if (iconf->copy_mode != DPDK_COPY_MODE_NONE && iconf->tx_segmented_mbufs) { + // the copy interface forwards segmented mbufs as they are received + if (dev_info->tx_offload_capa & RTE_ETH_TX_OFFLOAD_MULTI_SEGS) { + port_conf->txmode.offloads |= RTE_ETH_TX_OFFLOAD_MULTI_SEGS; + } else { + SCLogWarning("%s: multi-segment TX not supported, segmented mbufs may be transmitted " + "incorrectly, consider setting tx-segmented-mbufs to false", + iconf->iface); + } + } else if (dev_info->tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { + // fast free requires single-segment mbufs + port_conf->txmode.offloads |= RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; + } +} + static void DeviceInitPortConf(const DPDKIfaceConfig *iconf, const struct rte_eth_dev_info *dev_info, struct rte_eth_conf *port_conf) { @@ -1473,9 +1509,8 @@ static void DeviceInitPortConf(const DPDKIfaceConfig *iconf, DeviceSetMTU(port_conf, iconf->mtu); PortConfSetVlanOffload(iconf, dev_info, port_conf); - if (dev_info->tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) { - port_conf->txmode.offloads |= RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; - } + // configure TX offloads + PortConfSetTxOffloads(iconf, dev_info, port_conf); } static int DeviceConfigureQueues(DPDKIfaceConfig *iconf, const struct rte_eth_dev_info *dev_info, @@ -1598,6 +1633,11 @@ static int DeviceValidateOutIfaceConfig(DPDKIfaceConfig *iconf) iconf->iface, out_iconf->iface); out_iconf->DerefFunc(out_iconf); SCReturnInt(-EINVAL); + } else if (iconf->tx_segmented_mbufs != out_iconf->tx_segmented_mbufs) { + SCLogError("%s: tx-segmented-mbufs settings of interfaces %s and %s are not equal", + iconf->iface, iconf->iface, out_iconf->iface); + out_iconf->DerefFunc(out_iconf); + SCReturnInt(-EINVAL); } else if (strcmp(iconf->iface, out_iconf->out_iface) != 0) { // check if the other iface has the current iface set as a copy iface SCLogError("%s: copy interface of %s is not set to %s", iconf->iface, out_iconf->iface, @@ -1634,6 +1674,18 @@ static int DeviceConfigureIPS(DPDKIfaceConfig *iconf) iconf->out_iface, out_port_socket_id); } + struct rte_eth_dev_info out_dev_info; + retval = rte_eth_dev_info_get(iconf->out_port_id, &out_dev_info); + if (retval < 0) { + SCLogError( + "%s: getting device info failed: %s", iconf->out_iface, rte_strerror(-retval)); + SCReturnInt(retval); + } + // some drivers do not report the limit + if (out_dev_info.tx_desc_lim.nb_mtu_seg_max != 0) { + iconf->out_tx_seg_max = out_dev_info.tx_desc_lim.nb_mtu_seg_max; + } + retval = DeviceValidateOutIfaceConfig(iconf); if (retval != 0) { // Error will be written out by the validation function diff --git a/src/runmode-dpdk.h b/src/runmode-dpdk.h index b91838915c..4545997338 100644 --- a/src/runmode-dpdk.h +++ b/src/runmode-dpdk.h @@ -32,6 +32,7 @@ typedef struct DPDKIfaceConfigAttributes_ { const char *checksum_checks_offload; const char *mtu; const char *vlan_strip_offload; + const char *tx_segmented_mbufs; const char *rss_hf; const char *linkup_timeout; const char *mempool_size; diff --git a/src/source-dpdk.c b/src/source-dpdk.c index 074b0964c7..3afc2cabac 100644 --- a/src/source-dpdk.c +++ b/src/source-dpdk.c @@ -122,11 +122,15 @@ typedef struct DPDKThreadVars_ { StatsCounterId capture_dpdk_rx_no_mbufs; StatsCounterId capture_dpdk_ierrors; StatsCounterId capture_dpdk_tx_errs; + StatsCounterId capture_dpdk_segmented_drops; + StatsCounterId capture_dpdk_segmented_too_large; unsigned int flags; uint16_t threads; /* for IPS */ DpdkCopyModeEnum copy_mode; uint16_t out_port_id; + uint16_t out_tx_seg_max; // max segments per packet the copy interface can transmit + bool tx_segmented_mbufs; // forward packets received in multiple mbufs /* Entry in the peers_list */ uint64_t bytes; @@ -307,6 +311,26 @@ static inline void DPDKDumpCounters(DPDKThreadVars *ptv) } } +/** + * \brief Copy the packet data back to the segmented mbuf it was copied from + * + * Suricata modifies the packet data in place (inline stream normalization or + * the replace keyword) without changing the packet length. + */ +static inline void DPDKSegmentedMbufWriteBack(Packet *p) +{ + const uint8_t *data = GET_PKT_DATA(p); + const uint32_t len = GET_PKT_LEN(p); + DEBUG_VALIDATE_BUG_ON(len != rte_pktmbuf_pkt_len(p->dpdk_v.mbuf)); + + uint32_t offset = 0; + for (struct rte_mbuf *seg = p->dpdk_v.mbuf; seg != NULL && offset < len; seg = seg->next) { + uint32_t seg_len = MIN(rte_pktmbuf_data_len(seg), len - offset); + memcpy(rte_pktmbuf_mtod(seg, uint8_t *), data + offset, seg_len); + offset += seg_len; + } +} + static void DPDKReleasePacket(Packet *p) { int retval; @@ -314,13 +338,19 @@ static void DPDKReleasePacket(Packet *p) where Ethernet header could not be set (and pseudo packet) When enabling promiscuous mode on Intel cards, 2 ICMPv6 packets are generated. These get into the infinite cycle between the NIC and the switch in some cases */ - if ((p->dpdk_v.copy_mode == DPDK_COPY_MODE_TAP || - (p->dpdk_v.copy_mode == DPDK_COPY_MODE_IPS && !PacketCheckAction(p, ACTION_DROP))) + if (p->dpdk_v.mbuf != NULL && + (p->dpdk_v.copy_mode == DPDK_COPY_MODE_TAP || + (p->dpdk_v.copy_mode == DPDK_COPY_MODE_IPS && + !PacketCheckAction(p, ACTION_DROP))) #if defined(RTE_LIBRTE_I40E_PMD) || defined(RTE_LIBRTE_IXGBE_PMD) || defined(RTE_LIBRTE_ICE_PMD) && !(PacketIsICMPv6(p) && PacketGetICMPv6(p)->type == 143) #endif ) { BUG_ON(PKT_IS_PSEUDOPKT(p)); + // packets of segmented mbufs are copies, modifications must reach the mbufs + if (unlikely((p->flags & PKT_STREAM_MODIFIED) && !(p->flags & PKT_ZERO_COPY))) { + DPDKSegmentedMbufWriteBack(p); + } retval = rte_eth_tx_burst(p->dpdk_v.out_port_id, p->dpdk_v.out_queue_id, &p->dpdk_v.mbuf, 1); // rte_eth_tx_burst can return only 0 (failure) or 1 (success) because we are only @@ -465,11 +495,12 @@ static inline void DPDKSegmentedMbufWarning(void) if (unlikely(!segmented_mbufs_warned)) { enum rte_proc_type_t eal_t = rte_eal_process_type(); if (eal_t == RTE_PROC_SECONDARY) { - SCLogWarning("Segmented mbufs detected! Suricata handles them but for better " - "performance, try to increase mbuf size in your primary application"); + SCLogWarning("Segmented mbufs detected! Suricata processes them when possible, but " + "for better performance, try to increase mbuf size in your primary " + "application"); } else { - SCLogWarning("Segmented mbufs detected! Suricata handles them but for better " - "performance, try to increase MTU in your suricata.yaml"); + SCLogWarning("Segmented mbufs detected! Suricata processes them when possible, but " + "for better performance, try to increase MTU in your suricata.yaml"); } segmented_mbufs_warned = true; } @@ -521,58 +552,175 @@ cleanup: SCFree(xstats_names); } +static inline void DPDKSegmentedMbufTxDisabledWarning(void) +{ + static thread_local bool disabled_warned = false; + if (unlikely(!disabled_warned)) { + SCLogWarning("Segmented mbufs cannot be forwarded with tx-segmented-mbufs disabled, " + "dropping such packets, see capture.dpdk.segmented_drops"); + disabled_warned = true; + } +} + static inline void DPDKSegmentedMbufTooLargeWarning(uint32_t pkt_len) { static thread_local bool size_warned = false; if (unlikely(!size_warned)) { - SCLogWarning("Segmented mbuf larger than max payload size (%u > %d), packet will be " - "truncated", + SCLogWarning("Segmented mbuf larger than max payload size (%u > %d), dropping such " + "packets, see capture.dpdk.segmented_too_large", pkt_len, MAX_PAYLOAD_SIZE); size_warned = true; } } +static inline void DPDKSegmentedMbufTooManySegmentsWarning(uint16_t nb_segs, uint16_t seg_max) +{ + static thread_local bool segs_warned = false; + if (unlikely(!segs_warned)) { + SCLogWarning("Segmented mbuf of %u segments exceeds the %u segments the copy interface " + "can transmit, dropping such packets, see capture.dpdk.segmented_drops", + nb_segs, seg_max); + segs_warned = true; + } +} + +static inline void DPDKSegmentedMbufMalformedWarning(void) +{ + static thread_local bool malformed_warned = false; + if (unlikely(!malformed_warned)) { + SCLogWarning("Segmented mbuf whose segments do not match its packet length, dropping " + "such packets, see capture.dpdk.segmented_drops"); + malformed_warned = true; + } +} + +static inline void DPDKSegmentedMbufCopyFailedWarning(void) +{ + static thread_local bool copy_warned = false; + if (unlikely(!copy_warned)) { + SCLogWarning("Failed to copy a segmented mbuf into the packet buffer, dropping such " + "packets, see capture.dpdk.segmented_drops"); + copy_warned = true; + } +} + /** - * \brief Handle segmented (chained) mbufs by linearizing them + * \brief Check that the segments of a chained mbuf add up to its packet length + */ +static inline bool DPDKSegmentedMbufIsValid(const struct rte_mbuf *mbuf) +{ + const struct rte_mbuf *seg = mbuf; + uint32_t len = 0; + for (uint16_t i = 0; i < mbuf->nb_segs; i++) { + if (seg == NULL) { + return false; + } + len += rte_pktmbuf_data_len(seg); + seg = seg->next; + } + return seg == NULL && len == rte_pktmbuf_pkt_len(mbuf); +} + +/** + * \brief Decide whether a segmented (chained) mbuf can be processed * - * First tries rte_pktmbuf_linearize() which copies all segment data into the - * first segment. If that fails (not enough tailroom), copies data into the - * packet's internal buffer. + * Drops malformed chains. Then tries rte_pktmbuf_linearize() which copies all + * segment data into the first segment. If that fails (not enough tailroom or a + * shared first segment), the data is later copied into the packet's buffer, + * which holds at most MAX_PAYLOAD_SIZE bytes. In the copy modes, the mbuf chain + * itself is forwarded, so tx-segmented-mbufs must be enabled and the chain must + * not have more segments than the copy interface can transmit. * - * \param p Pointer to the Packet structure + * \param ptv Pointer to the thread variables * \param mbuf Pointer to the DPDK mbuf - * \return 0 on success, -1 on failure + * \return true if the mbuf can be processed, false if it must be dropped */ -static inline int DPDKSegmentedMbufHandle(Packet *p, struct rte_mbuf *mbuf) +static inline bool DPDKSegmentedMbufAccept(DPDKThreadVars *ptv, struct rte_mbuf *mbuf) { - if (rte_pktmbuf_linearize(mbuf) == 0) { - PacketSetData(p, rte_pktmbuf_mtod(mbuf, uint8_t *), rte_pktmbuf_pkt_len(mbuf)); - return 0; + DPDKSegmentedMbufWarning(); + if (unlikely(!DPDKSegmentedMbufIsValid(mbuf))) { + DPDKSegmentedMbufMalformedWarning(); + StatsCounterIncr(&ptv->tv->stats, ptv->capture_dpdk_segmented_drops); + return false; } - uint32_t pkt_len = rte_pktmbuf_pkt_len(mbuf); - uint32_t copy_len = pkt_len; - if (unlikely(pkt_len > MAX_PAYLOAD_SIZE)) { - DPDKSegmentedMbufTooLargeWarning(pkt_len); - copy_len = MAX_PAYLOAD_SIZE; + // linearize writes into the first segment, which must not be shared + if (RTE_MBUF_DIRECT(mbuf) && rte_mbuf_refcnt_read(mbuf) == 1 && + rte_pktmbuf_linearize(mbuf) == 0) { + return true; } + if (unlikely(rte_pktmbuf_pkt_len(mbuf) > MAX_PAYLOAD_SIZE)) { + DPDKSegmentedMbufTooLargeWarning(rte_pktmbuf_pkt_len(mbuf)); + StatsCounterIncr(&ptv->tv->stats, ptv->capture_dpdk_segmented_too_large); + return false; + } + + if (ptv->copy_mode != DPDK_COPY_MODE_NONE) { + if (!ptv->tx_segmented_mbufs) { + DPDKSegmentedMbufTxDisabledWarning(); + StatsCounterIncr(&ptv->tv->stats, ptv->capture_dpdk_segmented_drops); + return false; + } + if (mbuf->nb_segs > ptv->out_tx_seg_max) { + DPDKSegmentedMbufTooManySegmentsWarning(mbuf->nb_segs, ptv->out_tx_seg_max); + StatsCounterIncr(&ptv->tv->stats, ptv->capture_dpdk_segmented_drops); + return false; + } + } + + return true; +} + +/** + * \brief Copy the data of a segmented (chained) mbuf into the packet's buffer + * + * \param p Pointer to the Packet structure + * \param mbuf Pointer to the DPDK mbuf + * \return 0 on success, -1 on failure + */ +static inline int DPDKSegmentedMbufCopy(Packet *p, struct rte_mbuf *mbuf) +{ uint32_t offset = 0; - for (struct rte_mbuf *seg = mbuf; seg != NULL && offset < copy_len; seg = seg->next) { + for (struct rte_mbuf *seg = mbuf; seg != NULL; seg = seg->next) { uint32_t seg_len = rte_pktmbuf_data_len(seg); - uint32_t to_copy = (offset + seg_len > copy_len) ? (copy_len - offset) : seg_len; - if (PacketCopyDataOffset(p, offset, rte_pktmbuf_mtod(seg, uint8_t *), to_copy) != 0) { - SCLogWarning("Failed to copy segmented mbuf data at offset %u", offset); + if (PacketCopyDataOffset(p, offset, rte_pktmbuf_mtod(seg, uint8_t *), seg_len) != 0) { return -1; } - offset += to_copy; + offset += seg_len; } - SET_PKT_LEN(p, copy_len); + SET_PKT_LEN(p, offset); return 0; } +/** + * \brief Set the data of a packet received in a segmented (chained) mbuf + * + * \param ptv Pointer to the thread variables + * \param p Pointer to the Packet structure + * \return true on success, false if the packet must be dropped + */ +static inline bool DPDKSegmentedMbufSetData(DPDKThreadVars *ptv, Packet *p) +{ + struct rte_mbuf *mbuf = p->dpdk_v.mbuf; + if (!DPDKSegmentedMbufAccept(ptv, mbuf)) { + return false; + } + + if (rte_pktmbuf_is_contiguous(mbuf)) { + // linearized + PacketSetData(p, rte_pktmbuf_mtod(mbuf, uint8_t *), rte_pktmbuf_pkt_len(mbuf)); + } else if (DPDKSegmentedMbufCopy(p, mbuf) != 0) { + DPDKSegmentedMbufCopyFailedWarning(); + StatsCounterIncr(&ptv->tv->stats, ptv->capture_dpdk_segmented_drops); + return false; + } + + return true; +} + static void HandleShutdown(DPDKThreadVars *ptv) { SCLogDebug("Stopping Suricata!"); @@ -648,12 +796,14 @@ static TmEcode ReceiveDPDKLoop(ThreadVars *tv, void *data, void *slot) if (likely(rte_pktmbuf_is_contiguous(p->dpdk_v.mbuf))) { PacketSetData(p, rte_pktmbuf_mtod(p->dpdk_v.mbuf, uint8_t *), rte_pktmbuf_pkt_len(p->dpdk_v.mbuf)); - } else { - DPDKSegmentedMbufWarning(); - if (DPDKSegmentedMbufHandle(p, p->dpdk_v.mbuf) != 0) { - TmqhOutputPacketpool(ptv->tv, p); - continue; - } + } else if (!DPDKSegmentedMbufSetData(ptv, p)) { + // drop the uninspected packet, freeing the mbuf also keeps TAP mode, which + // ignores verdicts, from forwarding it + PacketDrop(p, ACTION_DROP, PKT_DROP_REASON_CAPTURE_ERROR); + rte_pktmbuf_free(p->dpdk_v.mbuf); + p->dpdk_v.mbuf = NULL; + TmqhOutputPacketpool(ptv->tv, p); + continue; } if (TmThreadsSlotProcessPkt(ptv->tv, ptv->slot, p) != TM_ECODE_OK) { @@ -712,14 +862,20 @@ static TmEcode ReceiveDPDKThreadInit(ThreadVars *tv, const void *initdata, void ptv->capture_dpdk_imissed = StatsRegisterCounter("capture.dpdk.imissed", &ptv->tv->stats); ptv->capture_dpdk_rx_no_mbufs = StatsRegisterCounter("capture.dpdk.no_mbufs", &ptv->tv->stats); ptv->capture_dpdk_ierrors = StatsRegisterCounter("capture.dpdk.ierrors", &ptv->tv->stats); + ptv->capture_dpdk_segmented_drops = + StatsRegisterCounter("capture.dpdk.segmented_drops", &ptv->tv->stats); + ptv->capture_dpdk_segmented_too_large = + StatsRegisterCounter("capture.dpdk.segmented_too_large", &ptv->tv->stats); ptv->copy_mode = dpdk_config->copy_mode; ptv->checksum_mode = dpdk_config->checksum_mode; + ptv->tx_segmented_mbufs = dpdk_config->tx_segmented_mbufs; ptv->threads = dpdk_config->threads; ptv->intr_enabled = (dpdk_config->flags & DPDK_IRQ_MODE) != 0; ptv->port_id = dpdk_config->port_id; ptv->out_port_id = dpdk_config->out_port_id; + ptv->out_tx_seg_max = dpdk_config->out_tx_seg_max; ptv->port_socket_id = dpdk_config->socket_id; thread_numa = GetNumaNode(); diff --git a/src/source-dpdk.h b/src/source-dpdk.h index 4748601de9..98923ca9b1 100644 --- a/src/source-dpdk.h +++ b/src/source-dpdk.h @@ -60,6 +60,8 @@ typedef struct DPDKIfaceConfig_ { DpdkCopyModeEnum copy_mode; const char *out_iface; uint16_t out_port_id; + uint16_t out_tx_seg_max; // max segments per packet the copy interface can transmit + bool tx_segmented_mbufs; // forward packets received in multiple mbufs in the copy modes /* DPDK flags */ uint32_t flags; ChecksumValidationMode checksum_mode; diff --git a/src/util-dpdk-common.h b/src/util-dpdk-common.h index bda8345ec4..9be5079e5f 100644 --- a/src/util-dpdk-common.h +++ b/src/util-dpdk-common.h @@ -46,6 +46,7 @@ #if RTE_VERSION < RTE_VERSION_NUM(21, 11, 0, 0) #define RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE DEV_TX_OFFLOAD_MBUF_FAST_FREE +#define RTE_ETH_TX_OFFLOAD_MULTI_SEGS DEV_TX_OFFLOAD_MULTI_SEGS #define RTE_ETH_RX_OFFLOAD_CHECKSUM DEV_RX_OFFLOAD_CHECKSUM diff --git a/suricata.yaml.in b/suricata.yaml.in index 49c1df696c..7705b547c7 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -843,6 +843,9 @@ dpdk: # - ips: the same as tap mode but it also drops packets that are flagged by rules to be dropped copy-mode: none copy-iface: none # or PCIe address of the second interface + # tap/ips: forward segmented mbufs that cannot be merged into one mbuf + # instead of dropping them; this can select a slower TX path on some NICs + tx-segmented-mbufs: false - interface: default threads: auto @@ -860,6 +863,7 @@ dpdk: tx-descriptors: auto copy-mode: none copy-iface: none + tx-segmented-mbufs: false # Cross platform libpcap capture support