/* Copyright (C) 2007-2025 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free * Software Foundation. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * version 2 along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA * 02110-1301, USA. */ /** * \file * * \author Victor Julien * * Basic detection engine */ #include "suricata-common.h" #include "suricata.h" #include "decode.h" #include "packet.h" #include "flow.h" #include "stream-tcp.h" #include "app-layer.h" #include "app-layer-parser.h" #include "app-layer-frames.h" #include "detect.h" #include "detect-parse.h" #include "detect-dsize.h" #include "detect-engine.h" #include "detect-engine-build.h" #include "detect-engine-frame.h" #include "detect-engine-profile.h" #include "detect-engine-alert.h" #include "detect-engine-siggroup.h" #include "detect-engine-address.h" #include "detect-engine-proto.h" #include "detect-engine-port.h" #include "detect-engine-mpm.h" #include "detect-engine-iponly.h" #include "detect-engine-threshold.h" #include "detect-engine-prefilter.h" #include "detect-engine-state.h" #include "detect-engine-analyzer.h" #include "detect-engine-payload.h" #include "detect-engine-event.h" #include "detect-filestore.h" #include "detect-flowvar.h" #include "detect-replace.h" #include "util-validate.h" #include "util-detect.h" #include "util-profiling.h" #include "action-globals.h" typedef struct DetectRunScratchpad { const AppProto alproto; const uint8_t flow_flags; /* flow/state flags: STREAM_* */ const bool app_decoder_events; const enum DetectFirewallPacketPolicies fw_pkt_policy; const SigGroupHead *sgh; } DetectRunScratchpad; /* prototypes */ static DetectRunScratchpad DetectRunSetup(const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *const p, Flow *const pflow, const enum DetectFirewallPacketPolicies fw_pkt_policy); static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Flow * const pflow, Packet * const p); static inline void DetectRunGetRuleGroup(const DetectEngineCtx *de_ctx, Packet * const p, Flow * const pflow, DetectRunScratchpad *scratch); static inline void DetectRunPrefilterPkt(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p, DetectRunScratchpad *scratch); static inline uint8_t DetectRulePacketRules(ThreadVars *const tv, const DetectEngineCtx *const de_ctx, DetectEngineThreadCtx *const det_ctx, Packet *const p, Flow *const pflow, const DetectRunScratchpad *scratch); static void DetectRunTx(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, DetectRunScratchpad *scratch); static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, DetectRunScratchpad *scratch); static inline void DetectRunPostRules(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *const p, Flow *const pflow, DetectRunScratchpad *scratch); static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx, Packet *p, Flow * const pflow); static inline void DetectRunAppendDefaultAccept(DetectEngineThreadCtx *det_ctx, Packet *p); /** \internal */ static void DetectRun(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p) { SCEnter(); SCLogDebug("pcap_cnt %" PRIu64 " direction %s pkt_src %s", PcapPacketCntGet(p), p->flow ? (FlowGetPacketDirection(p->flow, p) == TOSERVER ? "toserver" : "toclient") : "noflow", PktSrcToString(p->pkt_src)); /* Load the Packet's flow early, even though it might not be needed. * Mark as a constant pointer, although the flow itself can change. */ Flow * const pflow = p->flow; DetectRunScratchpad scratch = DetectRunSetup(de_ctx, det_ctx, p, pflow, DETECT_FIREWALL_POLICY_PACKET_FILTER); /* run the IPonly engine */ DetectRunInspectIPOnly(th_v, de_ctx, det_ctx, pflow, p); /* get our rule group */ DetectRunGetRuleGroup(de_ctx, p, pflow, &scratch); /* if we didn't get a sig group head, we * have nothing to do.... */ if (scratch.sgh == NULL) { if (!EngineModeIsFirewall()) { SCLogDebug("no sgh for this packet, nothing to match against"); goto end; } SCLogDebug( "packet %" PRIu64 ": no sgh, need to apply default policies", PcapPacketCntGet(p)); } else { /* run the prefilters for packets */ DetectRunPrefilterPkt(th_v, de_ctx, det_ctx, p, &scratch); } PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES); /* inspect the rules against the packet */ const uint8_t pkt_policy = DetectRulePacketRules(th_v, de_ctx, det_ctx, p, pflow, &scratch); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_RULES); SCLogDebug("packet %" PRIu64 ": pkt_policy %02x (p->action %02x)", PcapPacketCntGet(p), pkt_policy, p->action); /* Only FW rules will already have set the action, IDS rules go through PacketAlertFinalize * * If rules told us to drop or accept:packet/accept:flow, we skip app_filter and app_td. * * accept:hook won't have set the pkt_policy, so we simply continue. * * TODO what about app state progression, cleanup and such? */ if (pkt_policy & (ACTION_DROP | ACTION_ACCEPT)) { goto end; } /* run tx/state inspection. Don't call for ICMP error msgs. */ if (pflow && pflow->alstate && likely(pflow->proto == p->proto)) { if (p->proto == IPPROTO_TCP) { if ((p->flags & PKT_STREAM_EST) == 0) { SCLogDebug("packet %" PRIu64 ": skip tcp non-established", PcapPacketCntGet(p)); if (EngineModeIsFirewall()) { SCLogDebug("default accept: no PKT_STREAM_EST"); DetectRunAppendDefaultAccept(det_ctx, p); } goto end; } const TcpSession *ssn = p->flow->protoctx; bool setting_nopayload = p->flow->alparser && SCAppLayerParserStateIssetFlag( p->flow->alparser, APP_LAYER_PARSER_NO_INSPECTION) && !(p->flags & PKT_NOPAYLOAD_INSPECTION); // we may be right after disabling app-layer (ssh) if (ssn && ((ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) == 0 || setting_nopayload)) { // PACKET_PROFILING_DETECT_START(p, PROF_DETECT_TX); DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch); // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_TX); } // no update to transactions if (!PKT_IS_PSEUDOPKT(p) && p->app_update_direction == 0 && ((PKT_IS_TOSERVER(p) && (p->flow->flags & FLOW_TS_APP_UPDATED) == 0) || (PKT_IS_TOCLIENT(p) && (p->flow->flags & FLOW_TC_APP_UPDATED) == 0))) { SCLogDebug("packet %" PRIu64 ": no app-layer update", PcapPacketCntGet(p)); if (EngineModeIsFirewall()) { SCLogDebug("default accept: no app update"); DetectRunAppendDefaultAccept(det_ctx, p); } goto end; } } else if (p->proto == IPPROTO_UDP) { DetectRunFrames(th_v, de_ctx, det_ctx, p, pflow, &scratch); } PACKET_PROFILING_DETECT_START(p, PROF_DETECT_TX); DetectRunTx(th_v, de_ctx, det_ctx, p, pflow, &scratch); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_TX); /* see if we need to increment the inspect_id and reset the de_state */ PACKET_PROFILING_DETECT_START(p, PROF_DETECT_TX_UPDATE); AppLayerParserSetTransactionInspectId( pflow, pflow->alparser, pflow->alstate, scratch.flow_flags, (scratch.sgh == NULL)); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_TX_UPDATE); } else { SCLogDebug("packet %" PRIu64 ": no flow / app-layer", PcapPacketCntGet(p)); if (EngineModeIsFirewall()) { SCLogDebug("default accept: no flow/app"); DetectRunAppendDefaultAccept(det_ctx, p); } } end: DetectRunPostRules(th_v, de_ctx, det_ctx, p, pflow, &scratch); DetectRunCleanup(det_ctx, p, pflow); SCReturn; } /** \internal */ static void DetectRunPacketHook(ThreadVars *th_v, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p, enum DetectFirewallPacketPolicies fw_pkt_policy) { SCEnter(); SCLogDebug("pcap_cnt %" PRIu64 " direction %s pkt_src %s", PcapPacketCntGet(p), p->flow ? (FlowGetPacketDirection(p->flow, p) == TOSERVER ? "toserver" : "toclient") : "noflow", PktSrcToString(p->pkt_src)); /* Load the Packet's flow early, even though it might not be needed. * Mark as a constant pointer, although the flow itself can change. */ Flow *const pflow = p->flow; DetectRunScratchpad scratch = DetectRunSetup(de_ctx, det_ctx, p, pflow, fw_pkt_policy); scratch.sgh = sgh; /* if we didn't get a sig group head, we * have nothing to do.... */ if (scratch.sgh == NULL) { SCLogDebug("no sgh for this packet, nothing to match against"); goto end; } /* run the prefilters for packets */ DetectRunPrefilterPkt(th_v, de_ctx, det_ctx, p, &scratch); // PACKET_PROFILING_DETECT_START(p, PROF_DETECT_RULES); // TODO /* inspect the rules against the packet */ const uint8_t pkt_policy = DetectRulePacketRules(th_v, de_ctx, det_ctx, p, pflow, &scratch); // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_RULES); if (pkt_policy & (ACTION_DROP | ACTION_ACCEPT)) { goto end; } end: DetectRunPostRules(th_v, de_ctx, det_ctx, p, pflow, &scratch); DetectRunCleanup(det_ctx, p, pflow); SCReturn; } static void DetectRunPostMatch(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s) { /* run the packet match functions */ const SigMatchData *smd = s->sm_arrays[DETECT_SM_LIST_POSTMATCH]; if (smd != NULL) { KEYWORD_PROFILING_SET_LIST(det_ctx, DETECT_SM_LIST_POSTMATCH); SCLogDebug("running match functions, sm %p", smd); while (1) { KEYWORD_PROFILING_START; (void)sigmatch_table[smd->type].Match(det_ctx, p, s, smd->ctx); KEYWORD_PROFILING_END(det_ctx, smd->type, 1); if (smd->is_last) break; smd++; } } } /** * \brief Get the SigGroupHead for a packet. * * \param de_ctx detection engine context * \param p packet * * \retval sgh the SigGroupHead or NULL if non applies to the packet */ const SigGroupHead *SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx, const Packet *p) { SCEnter(); SigGroupHead *sgh = NULL; /* use ethernet non-IP sgh if we're ethernet but have no (valid) IP layer on top of it. */ if (PacketIsEthernet(p) && p->proto == 0 && de_ctx->eth_non_ip_sgh != NULL) { SCLogDebug("using eth_non_ip_sgh %p", de_ctx->eth_non_ip_sgh); SCReturnPtr(de_ctx->eth_non_ip_sgh, "SigGroupHead"); } /* if the packet proto is 0 (not set), we're inspecting it against * the decoder events sgh we have. */ if (p->proto == 0 && p->events.cnt > 0) { SCReturnPtr(de_ctx->decoder_event_sgh, "SigGroupHead"); } else if (p->proto == 0) { if (!(PacketIsIPv4(p) || PacketIsIPv6(p))) { /* not IP, so nothing to do */ SCReturnPtr(NULL, "SigGroupHead"); } } /* select the flow_gh */ const int dir = (p->flowflags & FLOW_PKT_TOCLIENT) == 0; int proto = PacketGetIPProto(p); if (proto == IPPROTO_TCP) { DetectPort *list = de_ctx->flow_gh[dir].tcp; SCLogDebug("tcp toserver %p, tcp toclient %p: going to use %p", de_ctx->flow_gh[1].tcp, de_ctx->flow_gh[0].tcp, de_ctx->flow_gh[dir].tcp); const uint16_t port = dir ? p->dp : p->sp; SCLogDebug("tcp port %u -> %u:%u", port, p->sp, p->dp); DetectPort *sghport = DetectPortLookupGroup(list, port); if (sghport != NULL) sgh = sghport->sh; SCLogDebug("TCP list %p, port %u, direction %s, sghport %p, sgh %p", list, port, dir ? "toserver" : "toclient", sghport, sgh); } else if (proto == IPPROTO_UDP) { DetectPort *list = de_ctx->flow_gh[dir].udp; uint16_t port = dir ? p->dp : p->sp; DetectPort *sghport = DetectPortLookupGroup(list, port); if (sghport != NULL) sgh = sghport->sh; SCLogDebug("UDP list %p, port %u, direction %s, sghport %p, sgh %p", list, port, dir ? "toserver" : "toclient", sghport, sgh); } else { sgh = de_ctx->flow_gh[dir].sgh[proto]; } SCReturnPtr(sgh, "SigGroupHead"); } static inline void DetectPrefilterCopyDeDup( const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx) { SigIntId *pf_ptr = det_ctx->pmq.rule_id_array; uint32_t final_cnt = det_ctx->pmq.rule_id_array_cnt; Signature **sig_array = de_ctx->sig_array; Signature **match_array = det_ctx->match_array; SigIntId previous_id = (SigIntId)-1; while (final_cnt-- > 0) { SigIntId id = *pf_ptr++; Signature *s = sig_array[id]; /* As the prefilter list can contain duplicates, check for that here. */ if (likely(id != previous_id)) { *match_array++ = s; previous_id = id; } } det_ctx->match_array_cnt = (uint32_t)(match_array - det_ctx->match_array); DEBUG_VALIDATE_BUG_ON(det_ctx->pmq.rule_id_array_cnt < det_ctx->match_array_cnt); PMQ_RESET(&det_ctx->pmq); } /** \internal * \brief update flow's file tracking flags based on the detection engine * A set of flags is prepared that is sent to the File API. The File API may reject one or more based on the global force settings. */ static inline void DetectPostInspectFileFlagsUpdate(Flow *f, const SigGroupHead *sgh, uint8_t direction) { uint16_t flow_file_flags = FLOWFILE_INIT; if (sgh == NULL) { SCLogDebug("requesting disabling all file features for flow"); flow_file_flags = FLOWFILE_NONE; } else { if (sgh->filestore_cnt == 0) { SCLogDebug("requesting disabling filestore for flow"); flow_file_flags |= (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC); } #ifdef HAVE_MAGIC if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMAGIC)) { SCLogDebug("requesting disabling magic for flow"); flow_file_flags |= (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC); } #endif if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILEMD5)) { SCLogDebug("requesting disabling md5 for flow"); flow_file_flags |= (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC); } if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA1)) { SCLogDebug("requesting disabling sha1 for flow"); flow_file_flags |= (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC); } if (!(sgh->flags & SIG_GROUP_HEAD_HAVEFILESHA256)) { SCLogDebug("requesting disabling sha256 for flow"); flow_file_flags |= (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC); } } if (flow_file_flags != 0) { FileUpdateFlowFileFlags(f, flow_file_flags, direction); } } static inline void DetectRunPostGetFirstRuleGroup(const Packet *p, Flow *pflow, const SigGroupHead *sgh) { if ((p->flowflags & FLOW_PKT_TOSERVER) && !(pflow->flags & FLOW_SGH_TOSERVER)) { /* first time we see this toserver sgh, store it */ pflow->sgh_toserver = sgh; pflow->flags |= FLOW_SGH_TOSERVER; if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) { if (pflow->protoctx != NULL) { TcpSession *ssn = pflow->protoctx; SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.client"); ssn->client.flags |= STREAMTCP_STREAM_FLAG_DISABLE_RAW; } } DetectPostInspectFileFlagsUpdate(pflow, pflow->sgh_toserver, STREAM_TOSERVER); } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && !(pflow->flags & FLOW_SGH_TOCLIENT)) { pflow->sgh_toclient = sgh; pflow->flags |= FLOW_SGH_TOCLIENT; if (p->proto == IPPROTO_TCP && (sgh == NULL || !(sgh->flags & SIG_GROUP_HEAD_HAVERAWSTREAM))) { if (pflow->protoctx != NULL) { TcpSession *ssn = pflow->protoctx; SCLogDebug("STREAMTCP_STREAM_FLAG_DISABLE_RAW ssn.server"); ssn->server.flags |= STREAMTCP_STREAM_FLAG_DISABLE_RAW; } } DetectPostInspectFileFlagsUpdate(pflow, pflow->sgh_toclient, STREAM_TOCLIENT); } } static inline void DetectRunGetRuleGroup( const DetectEngineCtx *de_ctx, Packet * const p, Flow * const pflow, DetectRunScratchpad *scratch) { const SigGroupHead *sgh = NULL; if (pflow) { bool use_flow_sgh = false; /* Get the stored sgh from the flow (if any). Make sure we're not using * the sgh for icmp error packets part of the same stream. */ if (PacketGetIPProto(p) == pflow->proto) { /* filter out icmp */ PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH); if ((p->flowflags & FLOW_PKT_TOSERVER) && (pflow->flags & FLOW_SGH_TOSERVER)) { sgh = pflow->sgh_toserver; SCLogDebug("sgh = pflow->sgh_toserver; => %p", sgh); use_flow_sgh = true; } else if ((p->flowflags & FLOW_PKT_TOCLIENT) && (pflow->flags & FLOW_SGH_TOCLIENT)) { sgh = pflow->sgh_toclient; SCLogDebug("sgh = pflow->sgh_toclient; => %p", sgh); use_flow_sgh = true; } PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH); } if (!(use_flow_sgh)) { PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH); sgh = SigMatchSignaturesGetSgh(de_ctx, p); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH); /* HACK: prevent the wrong sgh (or NULL) from being stored in the * flow's sgh pointers */ if (PacketIsICMPv4(p) && ICMPV4_DEST_UNREACH_IS_VALID(p)) { ; /* no-op */ } else { /* store the found sgh (or NULL) in the flow to save us * from looking it up again for the next packet. * Also run other tasks */ DetectRunPostGetFirstRuleGroup(p, pflow, sgh); } } } else { /* p->flags & PKT_HAS_FLOW */ /* no flow */ PACKET_PROFILING_DETECT_START(p, PROF_DETECT_GETSGH); sgh = SigMatchSignaturesGetSgh(de_ctx, p); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_GETSGH); } scratch->sgh = sgh; } static void DetectRunInspectIPOnly(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Flow * const pflow, Packet * const p) { if (pflow) { if (p->flowflags & (FLOW_PKT_TOSERVER_FIRST | FLOW_PKT_TOCLIENT_FIRST)) { SCLogDebug("testing against \"ip-only\" signatures"); PACKET_PROFILING_DETECT_START(p, PROF_DETECT_IPONLY); IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, p); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_IPONLY); } } else { /* p->flags & PKT_HAS_FLOW */ /* no flow */ /* Even without flow we should match the packet src/dst */ PACKET_PROFILING_DETECT_START(p, PROF_DETECT_IPONLY); IPOnlyMatchPacket(tv, de_ctx, det_ctx, &de_ctx->io_ctx, p); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_IPONLY); } } /** \internal * \brief inspect the rule header: protocol, ports, etc * \retval bool false if no match, true if match */ static inline bool DetectRunInspectRuleHeader( const Packet *p, const Flow *f, const Signature *s, const uint32_t sflags) { /* check if this signature has a requirement for flowvars of some type * and if so, if we actually have any in the flow. If not, the sig * can't match and we skip it. */ if ((p->flags & PKT_HAS_FLOW) && (sflags & SIG_FLAG_REQUIRE_FLOWVAR)) { DEBUG_VALIDATE_BUG_ON(f == NULL); /* no flowvars? skip this sig */ const bool fv = f->flowvar != NULL; if (!fv) { SCLogDebug("skipping sig as the flow has no flowvars and sig " "has SIG_FLAG_REQUIRE_FLOWVAR flag set."); return false; } } if (!(s->proto == NULL)) { const uint8_t s_proto_flags = s->proto->flags; /* TODO does it make sense to move these flags to s->flags? */ if ((s_proto_flags & DETECT_PROTO_IPV4) && !PacketIsIPv4(p)) { SCLogDebug("ip version didn't match"); return false; } if ((s_proto_flags & DETECT_PROTO_IPV6) && !PacketIsIPv6(p)) { SCLogDebug("ip version didn't match"); return false; } if (DetectProtoContainsProto(s->proto, PacketGetIPProto(p)) == 0) { SCLogDebug("proto didn't match"); if (PacketIsEthernet(p) && (s_proto_flags & (DETECT_PROTO_ETHERNET | DETECT_PROTO_ARP))) { SCLogDebug("checking ether/arp protocol"); if ((s_proto_flags & DETECT_PROTO_ARP) && !PacketIsARP(p)) { return false; } SCLogDebug("checking eth protocol: match!"); } else { return false; } } } /* check the source & dst port in the sig */ if (p->proto == IPPROTO_TCP || p->proto == IPPROTO_UDP || p->proto == IPPROTO_SCTP) { if (!(sflags & SIG_FLAG_DP_ANY)) { if (p->flags & PKT_IS_FRAGMENT) return false; const DetectPort *dport = DetectPortLookupGroup(s->dp, p->dp); if (dport == NULL) { SCLogDebug("dport didn't match."); return false; } } if (!(sflags & SIG_FLAG_SP_ANY)) { if (p->flags & PKT_IS_FRAGMENT) return false; const DetectPort *sport = DetectPortLookupGroup(s->sp, p->sp); if (sport == NULL) { SCLogDebug("sport didn't match."); return false; } } } else if ((sflags & (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) != (SIG_FLAG_DP_ANY|SIG_FLAG_SP_ANY)) { SCLogDebug("port-less protocol and sig needs ports"); return false; } /* check the destination address */ if (!(sflags & SIG_FLAG_DST_ANY)) { if (PacketIsIPv4(p)) { if (DetectAddressMatchIPv4(s->addr_dst_match4, s->addr_dst_match4_cnt, &p->dst) == 0) return false; } else if (PacketIsIPv6(p)) { if (DetectAddressMatchIPv6(s->addr_dst_match6, s->addr_dst_match6_cnt, &p->dst) == 0) return false; } } /* check the source address */ if (!(sflags & SIG_FLAG_SRC_ANY)) { if (PacketIsIPv4(p)) { if (DetectAddressMatchIPv4(s->addr_src_match4, s->addr_src_match4_cnt, &p->src) == 0) return false; } else if (PacketIsIPv6(p)) { if (DetectAddressMatchIPv6(s->addr_src_match6, s->addr_src_match6_cnt, &p->src) == 0) return false; } } return true; } /** \internal * \brief run packet/stream prefilter engines */ static inline void DetectRunPrefilterPkt(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p, DetectRunScratchpad *scratch) { /* create our prefilter mask */ PacketCreateMask(p, &p->sig_mask, scratch->alproto, scratch->app_decoder_events); /* run the prefilter engines */ Prefilter(det_ctx, scratch->sgh, p, scratch->flow_flags, p->sig_mask); /* create match list if we have non-pf and/or pf */ if (det_ctx->pmq.rule_id_array_cnt) { #ifdef PROFILING if (tv) { StatsCounterAvgAddI64( &tv->stats, det_ctx->counter_mpm_list, (int64_t)det_ctx->pmq.rule_id_array_cnt); } #endif PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_SORT2); DetectPrefilterCopyDeDup(de_ctx, det_ctx); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_SORT2); } } /** \internal * \brief check if the tx whose id is given is the only one * live transaction for the flow in the given direction * * \param f flow * \param txid transaction id * \param dir direction * * \retval bool true if we are sure this tx is the only one live in said direction */ static bool IsOnlyTxInDirection(Flow *f, uint64_t txid, uint8_t dir) { uint64_t tx_cnt = AppLayerParserGetTxCnt(f, f->alstate); if (tx_cnt == txid + 1) { // only live tx return true; } if (tx_cnt == txid + 2) { // 2 live txs, one after us void *tx = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, txid + 1); if (tx) { AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, tx); // test if the other tx is unidirectional in the other way if ((dir == STREAM_TOSERVER && (txd->flags & APP_LAYER_TX_SKIP_INSPECT_TS)) || (dir == STREAM_TOCLIENT && (txd->flags & APP_LAYER_TX_SKIP_INSPECT_TC))) { return true; } } } return false; } static int SortHelper(const void *a, const void *b) { const Signature *sa = *(const Signature **)a; const Signature *sb = *(const Signature **)b; if (sa->iid == sb->iid) return 0; return sa->iid > sb->iid ? 1 : -1; } static inline bool SkipFwRules(const Packet *p) { if (p->flow != NULL) { return (p->flow->flags & FLOW_ACTION_ACCEPT) != 0; } return false; } /** * \internal * \brief apply packet default policy * \param[in] de_ctx detect engine, for looking up the policy * \param[in] policy policy to apply * \param[in] p packet to apply policy to * \param[in] final see if we need to apply accept:hook to the packet * \retval action action to immediately apply, accept:hook will not set this unless final is true * * If this is run from the post-match final check, we need to apply a * packet:filter accept:hook to the packet as well. */ static uint8_t DetectRunApplyPacketPolicy(const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const enum DetectFirewallPacketPolicies policy, Packet *p, const bool final) { DEBUG_VALIDATE_BUG_ON(de_ctx->fw_policies == NULL); const struct DetectFirewallPolicy *pol = &de_ctx->fw_policies->pkt[policy]; if (pol->action & ACTION_DROP) { SCLogDebug("packet %" PRIu64 ": drop PKT_DROP_REASON_FW_DEFAULT_PACKET_POLICY", PcapPacketCntGet(p)); PacketDrop(p, pol->action, PKT_DROP_REASON_FW_DEFAULT_PACKET_POLICY); if (p->flow && pol->action_scope == ACTION_SCOPE_FLOW) { FlowSetDropAction(p->flow, FLOW_ACTION_BY_FIREWALL); SCLogDebug("packet %" PRIu64 ": drop scope flow", PcapPacketCntGet(p)); } } else if (pol->action & ACTION_ACCEPT) { SCLogDebug("packet %" PRIu64 ": accept", PcapPacketCntGet(p)); if (pol->action_scope == ACTION_SCOPE_PACKET) { p->action |= pol->action; SCLogDebug("packet %" PRIu64 ": accept scope packet", PcapPacketCntGet(p)); } else if (pol->action_scope == ACTION_SCOPE_HOOK) { SCLogDebug("packet %" PRIu64 ": accept scope hook", PcapPacketCntGet(p)); if (final) { p->action |= pol->action; SCLogDebug("packet %" PRIu64 ": accept scope hook upgraded to packet", PcapPacketCntGet(p)); } } else if (pol->action_scope == ACTION_SCOPE_FLOW) { p->action |= pol->action; SCLogDebug("packet %" PRIu64 ": accept scope flow", PcapPacketCntGet(p)); if (p->flow) { p->flow->flags |= FLOW_ACTION_ACCEPT; } } else { /* should be unreachable */ DEBUG_VALIDATE_BUG_ON(1); } } else { /* should be unreachable */ DEBUG_VALIDATE_BUG_ON(1); } Signature *s = de_ctx->fw_policies->pkt_policy_signatures[policy]; if (s != NULL) { AlertQueueAppendPacket(det_ctx, s, p, PACKET_ALERT_FLAG_APPLY_ACTION_TO_PACKET); } return p->action; } /** \internal * \brief helper for appending a packet alert * Tries to find (guess) a TX to add to the alert. */ static void DetectRulePacketAppendAlert(const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, const Signature *s, Packet *p, Flow *f, const uint8_t alert_flags_in) { DEBUG_VALIDATE_BUG_ON(alert_flags_in & PACKET_ALERT_FLAG_TX); if (f && f->alstate) { const uint8_t dir = (p->flowflags & FLOW_PKT_TOCLIENT) ? STREAM_TOCLIENT : STREAM_TOSERVER; const uint64_t tx_id = AppLayerParserGetTransactionInspectId(f->alparser, dir); if ((s->alproto != ALPROTO_UNKNOWN && f->proto == IPPROTO_UDP) || (de_ctx->guess_applayer && IsOnlyTxInDirection(f, tx_id, dir))) { // if there is a UDP specific app-layer signature, // or only one live transaction // try to use the good tx for the packet direction void *tx_ptr = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, tx_id); AppLayerTxData *txd = tx_ptr ? AppLayerParserGetTxData(f->proto, f->alproto, tx_ptr) : NULL; if (txd && txd->guessed_applayer_logged < de_ctx->guess_applayer_log_limit) { uint8_t alert_flags = alert_flags_in; if (f->proto != IPPROTO_UDP) { alert_flags |= PACKET_ALERT_FLAG_TX_GUESSED; } txd->guessed_applayer_logged++; AlertQueueAppendAppTxFromPacket(det_ctx, s, p, tx_id, txd->tx_type, alert_flags); return; } } } AlertQueueAppendPacket(det_ctx, s, p, alert_flags_in); } static inline uint8_t DetectRulePacketRules(ThreadVars *const tv, const DetectEngineCtx *const de_ctx, DetectEngineThreadCtx *const det_ctx, Packet *const p, Flow *const pflow, const DetectRunScratchpad *scratch) { uint8_t action = 0; bool fw_verdict = false; const bool have_fw_rules = EngineModeIsFirewall(); const Signature *next_s = NULL; /* inspect the sigs against the packet */ /* Prefetch the next signature. */ SigIntId match_cnt = det_ctx->match_array_cnt; #ifdef PROFILING if (tv) { StatsCounterAvgAddI64(&tv->stats, det_ctx->counter_match_list, (int64_t)match_cnt); } #endif Signature **match_array = det_ctx->match_array; SGH_PROFILING_RECORD(det_ctx, scratch->sgh); #ifdef PROFILING if (match_cnt >= de_ctx->profile_match_logging_threshold) RulesDumpMatchArray(det_ctx, scratch->sgh, p); #endif bool skip_fw = SkipFwRules(p); uint32_t sflags, next_sflags = 0; if (match_cnt) { next_s = *match_array++; next_sflags = next_s->flags; } while (match_cnt--) { RULE_PROFILING_START(p); bool break_out_of_packet_filter = false; uint8_t alert_flags = 0; #ifdef PROFILE_RULES bool smatch = false; /* signature match */ #endif const Signature *s = next_s; sflags = next_sflags; if (match_cnt) { next_s = *match_array++; next_sflags = next_s->flags; } SCLogDebug("packet %" PRIu64 ": inspecting signature id %" PRIu32 "", PcapPacketCntGet(p), s->id); /* if we accept:hook'd the `packet_filter` hook, we skip the rest of the firewall rules. */ if (s->flags & SIG_FLAG_FIREWALL) { if (skip_fw) { SCLogDebug("skipping firewall rule %u", s->id); goto next; } } else if (have_fw_rules) { /* fw mode, we skip anything after the fw rules if: * - flow pass is set * - packet pass (e.g. exception policy) */ if (p->flags & PKT_NOPACKET_INSPECTION || (pflow != NULL && pflow->flags & (FLOW_ACTION_PASS))) { SCLogDebug("skipping firewall rule %u", s->id); break_out_of_packet_filter = true; goto next; } } if (s->app_inspect != NULL) { goto next; // handle sig in DetectRunTx } if (s->frame_inspect != NULL) { goto next; // handle sig in DetectRunFrame } /* skip pkt sigs for flow end packets */ if ((p->flags & PKT_PSEUDO_STREAM_END) != 0 && s->type == SIG_TYPE_PKT) goto next; /* don't run mask check for stateful rules. * There we depend on prefilter */ if ((s->mask & p->sig_mask) != s->mask) { SCLogDebug("mask mismatch %x & %x != %x", s->mask, p->sig_mask, s->mask); goto next; } if (SigDsizePrefilter(p, s, sflags)) goto next; /* if the sig has alproto and the session as well they should match */ if (likely(sflags & SIG_FLAG_APPLAYER)) { if (s->alproto != ALPROTO_UNKNOWN && !AppProtoEquals(s->alproto, scratch->alproto)) { SCLogDebug("alproto mismatch"); goto next; } } if (DetectRunInspectRuleHeader(p, pflow, s, sflags) == false) { goto next; } if (!DetectEnginePktInspectionRun(tv, det_ctx, s, pflow, p, &alert_flags)) { goto next; } #ifdef PROFILE_RULES smatch = true; #endif DetectRunPostMatch(tv, det_ctx, p, s); DetectRulePacketAppendAlert(de_ctx, det_ctx, s, p, pflow, alert_flags); if (det_ctx->post_rule_work_queue.len > 0) { /* run post match prefilter engines on work queue */ PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, pflow); if (det_ctx->pmq.rule_id_array_cnt > 0) { /* undo "prefetch" */ match_array--; SCLogDebug("sig_array_len %u det_ctx->pmq.rule_id_array_cnt %u", de_ctx->sig_array_len, det_ctx->pmq.rule_id_array_cnt); const Signature **r = det_ctx->replace; for (uint32_t x = 0; x < match_cnt; x++) { *r++ = match_array[x]; SCLogDebug("appended %u", match_array[x]->id); } /* append the prefilter results, then sort it */ for (uint32_t x = 0; x < det_ctx->pmq.rule_id_array_cnt; x++) { SCLogDebug("adding iid %u", det_ctx->pmq.rule_id_array[x]); Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[x]]; SCLogDebug("adding id %u", ts->id); if (ts->app_inspect == NULL) { *r++ = ts; match_cnt++; } } if (match_cnt > 1) { qsort(det_ctx->replace, match_cnt, sizeof(Signature *), SortHelper); } /* rewrite match_array to include the new additions, and deduplicate * while at it. */ Signature **m = match_array; Signature *last_sig = NULL; uint32_t skipped = 0; for (uint32_t x = 0; x < match_cnt; x++) { /* de-duplicate */ if (last_sig == *m) { skipped++; continue; } last_sig = *m; *m++ = (Signature *)det_ctx->replace[x]; } match_cnt -= skipped; /* prefetch next */ next_s = *match_array++; next_sflags = next_s->flags; SCLogDebug("%u rules added", det_ctx->pmq.rule_id_array_cnt); det_ctx->post_rule_work_queue.len = 0; PMQ_RESET(&det_ctx->pmq); } } /* firewall logic in the packet:filter table: * 1. firewall rules preceed the packet:td rules in the list * 2. if no rule issues an accept, we drop * 3. drop is immediate * 4. accept: * - hook: skip rest of fw rules, inspect packet:td rules * - packet: immediate accept, no packet:td or app:* inspect * - flow: as packet, but applied to all future packets in the * flow as well */ if (s->flags & SIG_FLAG_FIREWALL) { if (s->action & (ACTION_ACCEPT)) { fw_verdict = true; enum ActionScope as = s->action_scope; if (as == ACTION_SCOPE_HOOK) { /* accept:hook: jump to first TD. Implemented as: * skip until the first TD rule. * Don't update action as we're just continuing to the next hook. */ skip_fw = true; } else if (as == ACTION_SCOPE_PACKET) { /* accept:packet: break loop, return accept */ action |= s->action; skip_fw = true; } else if (as == ACTION_SCOPE_FLOW) { /* accept:flow: break loop, return accept */ action |= s->action; skip_fw = true; /* set immediately, as we're in hook "packet_filter" */ if (pflow) { pflow->flags |= FLOW_ACTION_ACCEPT; } } } else if (s->action & ACTION_DROP) { /* apply a drop immediately here */ fw_verdict = true; action |= s->action; break_out_of_packet_filter = true; } } next: DetectVarProcessList(det_ctx, pflow, p); DetectReplaceFree(det_ctx); RULE_PROFILING_END(det_ctx, s, smatch, p); /* fw drop means we're done here */ if (break_out_of_packet_filter) break; continue; } /* if no rule told us to accept, and no rule explicitly dropped, we invoke the default drop * policy */ if (have_fw_rules) { if (skip_fw || fw_verdict) { /* apply fw action */ p->action |= action; } else { DEBUG_VALIDATE_BUG_ON(action & ACTION_DROP); /* non-final call as we may have to consider app-layer still */ action |= DetectRunApplyPacketPolicy(de_ctx, det_ctx, scratch->fw_pkt_policy, p, false); } } return action; } /** \internal * \param fw_pkt_policy policy to apply to packet rules */ static DetectRunScratchpad DetectRunSetup(const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *const p, Flow *const pflow, const enum DetectFirewallPacketPolicies fw_pkt_policy) { AppProto alproto = ALPROTO_UNKNOWN; uint8_t flow_flags = 0; /* flow/state flags */ bool app_decoder_events = false; PACKET_PROFILING_DETECT_START(p, PROF_DETECT_SETUP); #ifdef UNITTESTS if (RunmodeIsUnittests()) { p->alerts.cnt = 0; p->alerts.discarded = 0; p->alerts.suppressed = 0; } #endif det_ctx->filestore_cnt = 0; det_ctx->base64_decoded_len = 0; det_ctx->raw_stream_progress = 0; det_ctx->match_array_cnt = 0; det_ctx->json_content_len = 0; det_ctx->alert_queue_size = 0; p->alerts.drop.action = 0; p->alerts.firewall_discarded = 0; #ifdef DEBUG if (p->flags & PKT_STREAM_ADD) { det_ctx->pkt_stream_add_cnt++; } #endif /* grab the protocol state we will detect on */ if (p->flags & PKT_HAS_FLOW) { DEBUG_VALIDATE_BUG_ON(pflow == NULL); if (p->flowflags & FLOW_PKT_TOSERVER) { flow_flags = STREAM_TOSERVER; SCLogDebug("flag STREAM_TOSERVER set"); } else if (p->flowflags & FLOW_PKT_TOCLIENT) { flow_flags = STREAM_TOCLIENT; SCLogDebug("flag STREAM_TOCLIENT set"); } SCLogDebug("p->flowflags 0x%02x", p->flowflags); if (p->flags & PKT_PSEUDO_STREAM_END) { flow_flags |= STREAM_EOF; SCLogDebug("STREAM_EOF set"); } /* store tenant_id in the flow so that we can use it * for creating pseudo packets */ if (p->tenant_id > 0 && pflow->tenant_id == 0) { pflow->tenant_id = p->tenant_id; } /* live ruleswap check for flow updates */ if (pflow->de_ctx_version == 0) { /* first time this flow is inspected, set id */ pflow->de_ctx_version = de_ctx->version; } else if (pflow->de_ctx_version != de_ctx->version) { /* first time we inspect flow with this de_ctx, reset */ pflow->flags &= ~FLOW_SGH_TOSERVER; pflow->flags &= ~FLOW_SGH_TOCLIENT; pflow->sgh_toserver = NULL; pflow->sgh_toclient = NULL; pflow->de_ctx_version = de_ctx->version; SCGenericVarFree(pflow->flowvar); pflow->flowvar = NULL; DetectEngineStateResetTxs(pflow); } /* Retrieve the app layer state and protocol and the tcp reassembled * stream chunks. */ if ((p->proto == IPPROTO_TCP && (p->flags & PKT_STREAM_EST)) || (p->proto == IPPROTO_UDP) || (p->proto == IPPROTO_SCTP && (p->flowflags & FLOW_PKT_ESTABLISHED))) { /* update flow flags with knowledge on disruptions */ flow_flags = FlowGetDisruptionFlags(pflow, flow_flags); alproto = SCFlowGetAppProtocol(pflow); if (p->proto == IPPROTO_TCP && pflow->protoctx && StreamReassembleRawHasDataReady(pflow->protoctx, p)) { p->flags |= PKT_DETECT_HAS_STREAMDATA; } SCLogDebug("alproto %u", alproto); } else { SCLogDebug("packet doesn't have established flag set (proto %d)", p->proto); } app_decoder_events = AppLayerParserHasDecoderEvents(pflow->alparser); } DetectRunScratchpad pad = { alproto, flow_flags, app_decoder_events, fw_pkt_policy, NULL }; PACKET_PROFILING_DETECT_END(p, PROF_DETECT_SETUP); return pad; } static inline void DetectRunPostRules(ThreadVars *tv, const DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *const p, Flow *const pflow, DetectRunScratchpad *scratch) { /* so now let's iterate the alerts and remove the ones after a pass rule * matched (if any). This is done inside PacketAlertFinalize() */ /* PR: installed "tag" keywords are handled after the threshold inspection */ PACKET_PROFILING_DETECT_START(p, PROF_DETECT_ALERT); PacketAlertFinalize(de_ctx, det_ctx, p); if (p->alerts.cnt > 0) { StatsCounterAddI64(&tv->stats, det_ctx->counter_alerts, (uint64_t)p->alerts.cnt); } if (p->alerts.discarded > 0) { StatsCounterAddI64( &tv->stats, det_ctx->counter_alerts_overflow, (uint64_t)p->alerts.discarded); } if (p->alerts.firewall_discarded > 0) { StatsCounterAddI64(&tv->stats, det_ctx->counter_firewall_discarded_alerts, (uint64_t)p->alerts.firewall_discarded); } if (p->alerts.suppressed > 0) { StatsCounterAddI64( &tv->stats, det_ctx->counter_alerts_suppressed, (uint64_t)p->alerts.suppressed); } PACKET_PROFILING_DETECT_END(p, PROF_DETECT_ALERT); /* firewall: "fail" closed if we don't have an ACCEPT. This can happen * if there was no rule group. */ // TODO review packet src types here if (EngineModeIsFirewall() && ((p->action & (ACTION_ACCEPT | ACTION_DROP)) == 0) && p->pkt_src == PKT_SRC_WIRE) { SCLogDebug("packet %" PRIu64 ": default action as no verdict set %02x (pkt %s)", PcapPacketCntGet(p), p->action, PktSrcToString(p->pkt_src)); (void)DetectRunApplyPacketPolicy(de_ctx, det_ctx, scratch->fw_pkt_policy, p, true); DEBUG_VALIDATE_BUG_ON((p->action & (ACTION_DROP | ACTION_ACCEPT)) == 0); } } static void DetectRunCleanup(DetectEngineThreadCtx *det_ctx, Packet *p, Flow * const pflow) { PACKET_PROFILING_DETECT_START(p, PROF_DETECT_CLEANUP); InspectionBufferClean(det_ctx); if (pflow != NULL) { /* update inspected tracker for raw reassembly */ if (p->proto == IPPROTO_TCP && pflow->protoctx != NULL && (p->flags & PKT_DETECT_HAS_STREAMDATA)) { StreamReassembleRawUpdateProgress(pflow->protoctx, p, det_ctx->raw_stream_progress); } } PACKET_PROFILING_DETECT_END(p, PROF_DETECT_CLEANUP); SCReturn; } enum DetectTxFirewallFlowControl { DETECT_TX_FW_FC_OK = 0, /**< continue to next rule */ DETECT_TX_FW_FC_SKIP = 1, /**< skip this rule, continue to next */ DETECT_TX_FW_FC_BREAK = 2, /**< break rule loop */ }; void RuleMatchCandidateTxArrayInit(DetectEngineThreadCtx *det_ctx, uint32_t size) { DEBUG_VALIDATE_BUG_ON(det_ctx->tx_candidates); det_ctx->tx_candidates = SCCalloc(size, sizeof(RuleMatchCandidateTx)); if (det_ctx->tx_candidates == NULL) { FatalError("failed to allocate %" PRIu64 " bytes", (uint64_t)(size * sizeof(RuleMatchCandidateTx))); } det_ctx->tx_candidates_size = size; SCLogDebug("array initialized to %u elements (%"PRIu64" bytes)", size, (uint64_t)(size * sizeof(RuleMatchCandidateTx))); } void RuleMatchCandidateTxArrayFree(DetectEngineThreadCtx *det_ctx) { SCFree(det_ctx->tx_candidates); det_ctx->tx_candidates_size = 0; } /* if size >= cur_space */ static inline bool RuleMatchCandidateTxArrayHasSpace(const DetectEngineThreadCtx *det_ctx, const uint32_t need) { if (det_ctx->tx_candidates_size >= need) return 1; return 0; } /* realloc */ static int RuleMatchCandidateTxArrayExpand(DetectEngineThreadCtx *det_ctx, const uint32_t needed) { const uint32_t old_size = det_ctx->tx_candidates_size; uint32_t new_size = needed; void *ptmp = SCRealloc(det_ctx->tx_candidates, (new_size * sizeof(RuleMatchCandidateTx))); if (ptmp == NULL) { FatalError("failed to expand to %" PRIu64 " bytes", (uint64_t)(new_size * sizeof(RuleMatchCandidateTx))); // TODO can this be handled more gracefully? } det_ctx->tx_candidates = ptmp; det_ctx->tx_candidates_size = new_size; SCLogDebug("array expanded from %u to %u elements (%"PRIu64" bytes -> %"PRIu64" bytes)", old_size, new_size, (uint64_t)(old_size * sizeof(RuleMatchCandidateTx)), (uint64_t)(new_size * sizeof(RuleMatchCandidateTx))); (void)old_size; return 1; } /** \internal * \brief sort helper for sorting match candidates by id: ascending * * The id field is set from Signature::num, so we sort the candidates to match the signature * sort order (ascending), where candidates that have flags go first. */ static int DetectRunTxSortHelper(const void *a, const void *b) { const RuleMatchCandidateTx *s0 = a; const RuleMatchCandidateTx *s1 = b; if (s1->id == s0->id) { if (s1->flags && !s0->flags) return 1; else if (!s1->flags && s0->flags) return -1; return 0; } else return s0->id > s1->id ? 1 : -1; } #if 0 #define TRACE_SID_TXS(sid,txs,...) \ do { \ char _trace_buf[2048]; \ snprintf(_trace_buf, sizeof(_trace_buf), __VA_ARGS__); \ SCLogNotice("%p/%"PRIu64"/%u: %s", txs->tx_ptr, txs->tx_id, sid, _trace_buf); \ } while(0) #else #define TRACE_SID_TXS(sid,txs,...) #endif /** \internal * \brief get correct transaction pointer * * Gets an encapsulated DNS transaction in the DOH2 case. * * Returns NULL is the TX is not to be inspected by this engine. */ void *DetectGetInnerTx(void *tx_ptr, AppProto alproto, AppProto engine_alproto, uint8_t flow_flags) { SCLogDebug("pre: tx_ptr %p flow::alproto %s engine::alproto %s", tx_ptr, AppProtoToString(alproto), AppProtoToString(engine_alproto)); if (unlikely(alproto == ALPROTO_DOH2)) { switch (engine_alproto) { case ALPROTO_DOH2: /* need to get the dns tx pointer */ tx_ptr = SCDoH2GetDnsTx(tx_ptr, flow_flags); break; case ALPROTO_HTTP2: case ALPROTO_UNKNOWN: /* tx_ptr is untouched, so use outer (HTTP/2) layer */ break; default: /* any other protocol is a mismatch with DOH2 */ tx_ptr = NULL; break; } } else if (engine_alproto != alproto && engine_alproto != ALPROTO_UNKNOWN) { /* incompatible engine->alproto with flow alproto */ tx_ptr = NULL; } return tx_ptr; } /** \internal * \brief inspect a rule against a transaction * * Inspect a rule. New detection or continued stateful * detection. * * \param stored_flags pointer to stored flags or NULL. * If stored_flags is set it means we're continuing * inspection from an earlier run. * * \retval 1 sig matched * \retval 0 partial incomplete match * \retval -1 failed to match * \retval -2 no match yet, but not final: the MPM engine will revisit the * tx as it progresses. The caller must not apply a hook default * policy for it: that could decide the packet before the revisit * can turn it into a match. */ static int DetectRunTxInspectRule(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, const uint8_t in_flow_flags, // direction, EOF, etc void *alstate, DetectTransaction *tx, const Signature *s, uint32_t *stored_flags, RuleMatchCandidateTx *can, DetectRunScratchpad *scratch) { const uint8_t flow_flags = in_flow_flags; const int direction = (flow_flags & STREAM_TOSERVER) ? 0 : 1; uint32_t inspect_flags = stored_flags ? *stored_flags : 0; int total_matches = 0; uint16_t file_no_match = 0; bool mpm_before_progress = false; // is mpm engine before progress? bool mpm_in_progress = false; // is mpm engine in a buffer we will revisit? TRACE_SID_TXS(s->id, tx, "starting %s", direction ? "toclient" : "toserver"); /* for a new inspection we inspect pkt header and packet matches */ if (likely(stored_flags == NULL)) { TRACE_SID_TXS(s->id, tx, "first inspect, run packet matches"); /* the firewall LTE coverage build already ran the same header check for * this candidate, so skip repeating it */ if (!can->fw_lte_header_ok && DetectRunInspectRuleHeader(p, f, s, s->flags) == false) { TRACE_SID_TXS(s->id, tx, "DetectRunInspectRuleHeader() no match"); return -1; } if (!DetectEnginePktInspectionRun(tv, det_ctx, s, f, p, NULL)) { TRACE_SID_TXS(s->id, tx, "DetectEnginePktInspectionRun no match"); return -1; } /* stream mpm and negated mpm sigs can end up here with wrong proto */ if (!(AppProtoEquals(s->alproto, f->alproto) || s->alproto == ALPROTO_UNKNOWN)) { TRACE_SID_TXS(s->id, tx, "alproto mismatch"); return -1; } } else { TRACE_SID_TXS(s->id, tx, "continue, inspect_flags %x", inspect_flags); } const DetectEngineAppInspectionEngine *engine = s->app_inspect; do { TRACE_SID_TXS(s->id, tx, "engine %p inspect_flags %x", engine, inspect_flags); // also if it is not the same direction, but // this is a transactional signature, and we are toclient if (!(inspect_flags & BIT_U32(engine->id)) && (direction == engine->dir || ((s->flags & SIG_FLAG_TXBOTHDIR) && direction == 1))) { DEBUG_VALIDATE_BUG_ON( AppLayerParserSupportsSubStates(engine->alproto) && engine->sub_state == 0); DEBUG_VALIDATE_BUG_ON( !AppLayerParserSupportsSubStates(engine->alproto) && engine->sub_state != 0); if (engine->alproto != ALPROTO_UNKNOWN && // app-layer-events is registered for each // proto this way tx->tx_type != engine->sub_state) { TRACE_SID_TXS(s->id, tx, "skip because engine alproto %s sub_state %u != tx_type %u (engine " "progress %u)", AppProtoToString(engine->alproto), engine->sub_state, tx->tx_type, engine->progress); engine = engine->next; continue; } TRACE_SID_TXS(s->id, tx, "inspecting engine alproto %s sub_state %u == tx_type %u (engine progress %u)", AppProtoToString(engine->alproto), engine->sub_state, tx->tx_type, engine->progress); void *tx_ptr = DetectGetInnerTx(tx->tx_ptr, f->alproto, engine->alproto, flow_flags); if (tx_ptr == NULL) { TRACE_SID_TXS(s->id, tx, "no tx_ptr after DetectGetInnerTx"); if (engine->alproto != ALPROTO_UNKNOWN) { TRACE_SID_TXS(s->id, tx, "no tx_ptr skip engine"); /* special case: file_data on 'alert tcp' will have engines * in the list that are not for us. */ engine = engine->next; continue; } else { tx_ptr = tx->tx_ptr; } } TRACE_SID_TXS(s->id, tx, "tx_ptr %p", tx_ptr); /* engines are sorted per progress, except that the one with * mpm/prefilter enabled is first */ if (tx->tx_progress < engine->progress) { SCLogDebug("tx progress %d < engine progress %d", tx->tx_progress, engine->progress); break; } if (engine->mpm) { if (tx->tx_progress > engine->progress) { TRACE_SID_TXS(s->id, tx, "engine->mpm: t->tx_progress %u > engine->progress %u, so set " "mpm_before_progress", tx->tx_progress, engine->progress); mpm_before_progress = true; } else if (tx->tx_progress == engine->progress) { TRACE_SID_TXS(s->id, tx, "engine->mpm: t->tx_progress %u == engine->progress %u, so set " "mpm_in_progress", tx->tx_progress, engine->progress); if ((p->flags & PKT_PSEUDO_DETECTLOG_FLUSH) == 0) { mpm_in_progress = true; } } } uint8_t engine_flags = flow_flags; if (direction != engine->dir) { engine_flags = flow_flags ^ (STREAM_TOCLIENT | STREAM_TOSERVER); } /* run callback: but bypass stream callback if we can */ uint8_t match; if (unlikely(engine->stream && can->stream_stored)) { match = can->stream_result; TRACE_SID_TXS(s->id, tx, "stream skipped, stored result %d used instead", match); } else if (engine->v2.Callback == NULL) { /* TODO is this the cleanest way to support a non-app sig on a app hook? */ if (tx->tx_progress > engine->progress) { mpm_before_progress = true; // TODO needs a new name now } /* we don't have to store a "hook" match, also don't want to keep any state to make * sure the hook gets invoked again until tx progress progresses. */ if ((s->flags & SIG_FLAG_FW_HOOK_LTE) == 0 && tx->tx_progress <= engine->progress) { return 1; // DETECT_ENGINE_INSPECT_SIG_MATCH; } /* if progress > engine progress, track state to avoid additional matches */ match = DETECT_ENGINE_INSPECT_SIG_MATCH; } else { KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list); DEBUG_VALIDATE_BUG_ON(engine->v2.Callback == NULL); match = engine->v2.Callback( de_ctx, det_ctx, engine, s, f, engine_flags, alstate, tx_ptr, tx->tx_id); TRACE_SID_TXS(s->id, tx, "engine %p match %d", engine, match); if (engine->stream) { can->stream_stored = true; can->stream_result = match; TRACE_SID_TXS(s->id, tx, "stream ran, store result %d for next tx (if any)", match); } } if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) { inspect_flags |= BIT_U32(engine->id); engine = engine->next; total_matches++; continue; } else if (match == DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_FILES) { /* if the file engine matched, but indicated more * files are still in progress, we don't set inspect * flags as these would end inspection for this tx */ engine = engine->next; total_matches++; continue; } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) { inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; inspect_flags |= BIT_U32(engine->id); } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES) { inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; inspect_flags |= BIT_U32(engine->id); file_no_match = 1; } /* implied DETECT_ENGINE_INSPECT_SIG_NO_MATCH */ if (engine->mpm && mpm_before_progress) { inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; inspect_flags |= BIT_U32(engine->id); } break; } else if (!(inspect_flags & BIT_U32(engine->id)) && s->flags & SIG_FLAG_TXBOTHDIR && direction != engine->dir) { TRACE_SID_TXS(s->id, tx, "handle bidir engine"); // for transactional rules, the engines on the opposite direction // are ordered by progress on the different side // so we have a two mixed-up lists, and we skip the elements if (direction == 0 && engine->next == NULL) { // do not match yet on request only break; } engine = engine->next; continue; } engine = engine->next; } while (engine != NULL); TRACE_SID_TXS(s->id, tx, "inspect_flags %x, total_matches %u, engine %p", inspect_flags, total_matches, engine); bool full_match = false; if (engine == NULL && total_matches) { inspect_flags |= DE_STATE_FLAG_FULL_INSPECT; TRACE_SID_TXS(s->id, tx, "MATCH"); full_match = true; } if (stored_flags) { *stored_flags = inspect_flags; TRACE_SID_TXS(s->id, tx, "continue inspect flags %08x", inspect_flags); } else { // store... or? If tx is done we might not want to come back to this tx // also... if mpmid tracking is enabled, we won't do a sig again for this tx... TRACE_SID_TXS(s->id, tx, "start inspect flags %08x", inspect_flags); if (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) { if (file_no_match) { /* if we have a mismatch on a file sig, we need to keep state. * We may get another file on the same tx (for http and smtp * at least), so for a new file we need to re-eval the sig. * Thoughts / TODO: * - not for some protos that have 1 file per tx (e.g. nfs) * - maybe we only need this for file sigs that mix with * other matches? E.g. 'POST + filename', is different than * just 'filename'. */ DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s, inspect_flags, flow_flags, file_no_match); } } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) && mpm_before_progress) { TRACE_SID_TXS(s->id, tx, "no need to store match sig, " "mpm won't trigger for it anymore"); if (inspect_flags & DE_STATE_FLAG_FILE_INSPECT) { TRACE_SID_TXS(s->id, tx, "except that for new files, " "we may have to revisit anyway"); DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s, inspect_flags, flow_flags, file_no_match); } } else if ((inspect_flags & DE_STATE_FLAG_FULL_INSPECT) == 0 && mpm_in_progress) { TRACE_SID_TXS(s->id, tx, "no need to store no-match sig, " "mpm will revisit it"); return -2; /* no match, but not final: mpm will revisit it */ } else if (inspect_flags != 0 || file_no_match != 0) { TRACE_SID_TXS(s->id, tx, "storing state: flags %08x", inspect_flags); DetectRunStoreStateTx(scratch->sgh, f, tx->tx_ptr, tx->tx_id, s, inspect_flags, flow_flags, file_no_match); } else { if (inspect_flags == 0) { TRACE_SID_TXS(s->id, tx, "no match: inspect_flags %08x", inspect_flags); return -1; } } } if (full_match) { return 1; /* can't be a partial match if we're at the end state */ } else if ((inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) == 0 && tx->tx_progress < tx->tx_end_state) { return 0; } else { return -1; } } #define NO_TX \ { \ NULL, 0, NULL, NULL, 0, 0, 0, 0, false, 0, \ } /** \internal * \brief get a DetectTransaction object * \retval struct filled with relevant info or all nulls/0s */ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alproto, const uint64_t tx_id, void *tx_ptr, const uint8_t flow_flags) { AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx_ptr); const uint8_t tx_progress = (uint8_t)AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags); DEBUG_VALIDATE_BUG_ON(tx_progress >= APP_LAYER_MAX_PROGRESS); const uint8_t e_tx_end_state = AppLayerParserGetTxEndState(ipproto, alproto, tx_ptr, flow_flags); DEBUG_VALIDATE_BUG_ON(e_tx_end_state >= APP_LAYER_MAX_PROGRESS); bool updated = (flow_flags & STREAM_TOSERVER) ? txd->updated_ts : txd->updated_tc; if (!updated && tx_progress < e_tx_end_state && ((flow_flags & STREAM_EOF) == 0)) { DetectTransaction no_tx = NO_TX; return no_tx; } const uint8_t inspected_flag = (flow_flags & STREAM_TOSERVER) ? APP_LAYER_TX_INSPECTED_TS : APP_LAYER_TX_INSPECTED_TC; if (unlikely(txd->flags & inspected_flag)) { SCLogDebug("%" PRIu64 " tx already fully inspected for %s. Flags %02x", tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", txd->flags); DetectTransaction no_tx = NO_TX; return no_tx; } const uint8_t skip_flag = (flow_flags & STREAM_TOSERVER) ? APP_LAYER_TX_SKIP_INSPECT_TS : APP_LAYER_TX_SKIP_INSPECT_TC; if (unlikely(txd->flags & skip_flag)) { SCLogDebug("%" PRIu64 " tx should not be inspected in direction %s. Flags %02x", tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", txd->flags); DetectTransaction no_tx = NO_TX; return no_tx; } if (txd->tx_type != 0) { SCLogDebug("using tx_type %u", txd->tx_type); } const uint8_t detect_progress = (flow_flags & STREAM_TOSERVER) ? txd->detect_progress_ts : txd->detect_progress_tc; const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1; DetectEngineState *tx_de_state = txd->de_state; DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL; DetectTransaction tx = { .tx_ptr = tx_ptr, .tx_id = tx_id, .tx_data_ptr = (struct AppLayerTxData *)txd, .de_state = tx_dir_state, .detect_progress = detect_progress, .detect_progress_orig = detect_progress, .tx_progress = (uint8_t)tx_progress, .tx_end_state = e_tx_end_state, .is_last = false, .tx_type = txd->tx_type, }; return tx; } static inline void StoreDetectProgress( DetectTransaction *tx, const uint8_t flow_flags, const uint8_t progress) { AppLayerTxData *txd = (AppLayerTxData *)tx->tx_data_ptr; if (flow_flags & STREAM_TOSERVER) { txd->detect_progress_ts = progress; } else { txd->detect_progress_tc = progress; } } // Merge 'state' rules from the regular prefilter // updates array_idx on the way static inline void RuleMatchCandidateMergeStateRules( DetectEngineThreadCtx *det_ctx, uint32_t *array_idx, bool *fw_lte) { // Now, we will merge 2 sorted lists : // the one in det_ctx->tx_candidates // and the one in det_ctx->match_array // For match_array, we take only the relevant elements where s->app_inspect != NULL // Basically, we iterate at the same time over the 2 lists // comparing and taking an element from either. // Trick is to do so in place in det_ctx->tx_candidates, // so as to minimize the number of moves in det_ctx->tx_candidates. // For this, the algorithm traverses the lists in reverse order. // Otherwise, if the first element of match_array was to be put before // all tx_candidates, we would need to shift all tx_candidates // Retain the number of elements sorted in tx_candidates before merge uint32_t j = *array_idx; // First loop only counting the number of elements to add for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) { const Signature *s = det_ctx->match_array[i]; if (s->app_inspect != NULL) { if (s->flags & SIG_FLAG_FW_HOOK_LTE) { *fw_lte = true; } (*array_idx)++; } } // Future number of elements in tx_candidates after merge uint32_t k = *array_idx; if (k == j) { // no new element from match_array to merge in tx_candidates return; } // variable i is for all elements of match_array (even not relevant ones) // variable j is for elements of tx_candidates before merge // variable k is for elements of tx_candidates after merge for (uint32_t i = det_ctx->match_array_cnt; i > 0;) { const Signature *s = det_ctx->match_array[i - 1]; if (s->app_inspect == NULL) { // no relevant element, get the next one from match_array i--; continue; } // we have one element from match_array to merge in tx_candidates k--; if (j > 0) { // j > 0 means there is still at least one element in tx_candidates to merge const RuleMatchCandidateTx *s0 = &det_ctx->tx_candidates[j - 1]; if (s->iid <= s0->id) { // get next element from previous tx_candidates j--; // take the element from tx_candidates before merge det_ctx->tx_candidates[k].s = det_ctx->tx_candidates[j].s; det_ctx->tx_candidates[k].id = det_ctx->tx_candidates[j].id; det_ctx->tx_candidates[k].fw_lte_counted = det_ctx->tx_candidates[j].fw_lte_counted; det_ctx->tx_candidates[k].fw_lte_header_ok = det_ctx->tx_candidates[j].fw_lte_header_ok; det_ctx->tx_candidates[k].flags = det_ctx->tx_candidates[j].flags; det_ctx->tx_candidates[k].stream_reset = det_ctx->tx_candidates[j].stream_reset; continue; } } // otherwise // get next element from match_array i--; // take the element from match_array det_ctx->tx_candidates[k].s = s; det_ctx->tx_candidates[k].id = s->iid; det_ctx->tx_candidates[k].fw_lte_counted = false; det_ctx->tx_candidates[k].fw_lte_header_ok = false; det_ctx->tx_candidates[k].flags = NULL; det_ctx->tx_candidates[k].stream_reset = 0; } // Even if k > 0 or j > 0, the loop is over. (Note that j == k now) // The remaining elements in tx_candidates up to k were already sorted // and come before any other element later in the list } struct DetectFirewallAppTxState { bool fw_skip_app_filter; bool skip_fw_hook; uint8_t skip_before_progress; bool fw_last_for_progress; bool fw_next_progress_missing; bool last_fw_rule; /**< processing the last fw rule, so we need to eval all hooks after it. */ /** det_ctx->fw_lte_cover holds the counts for this walk. The counts answer * "does another pending rule cover hook H": per hook, how many pending LTE * rules cover it (their own hook and every prior hook). Progress values * stay under APP_LAYER_MAX_PROGRESS. */ bool fw_lte_cover_active; }; /** \internal * \brief initialize the per-thread LTE hook coverage for this walk. * * The counts live in the thread ctx so IDS traffic does not pay to clear them. * They are cleared lazily, on the first LTE rule of the walk. */ static inline void DetectFwEnsureLteCoverage( DetectEngineThreadCtx *det_ctx, struct DetectFirewallAppTxState *fw_state) { if (!fw_state->fw_lte_cover_active) { memset(det_ctx->fw_lte_cover, 0, sizeof(det_ctx->fw_lte_cover)); fw_state->fw_lte_cover_active = true; } } /** \internal * \brief build the per hook coverage of the pending LTE rules. * * Counts, per hook, how many pending LTE rules cover it (their own hook and * every prior hook). The candidate list is sorted, so duplicates - the state * store and the per-state registrations can both add a rule - are adjacent. */ static void DetectFwBuildLteCoverage(DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, struct DetectFirewallAppTxState *fw_state, const uint32_t array_idx) { DetectFwEnsureLteCoverage(det_ctx, fw_state); for (uint32_t i = 0; i < array_idx; i++) { const Signature *s = det_ctx->tx_candidates[i].s; /* only firewall rules implement the per-state policy: a threat * detection rule using the flags & SIG_FLAG_FW_HOOK_LTE) || !(s->flags & SIG_FLAG_FIREWALL)) { continue; } RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i]; can->fw_lte_counted = false; const uint32_t *flags = can->flags; if (flags == NULL) { if (!DetectRunInspectRuleHeader(p, f, s, s->flags)) { /* an out of scope rule never covered anything */ continue; } /* the walk inspects this candidate with the same arguments, so it * can skip repeating the header check */ can->fw_lte_header_ok = true; } if (flags != NULL && (*flags & (DE_STATE_FLAG_FULL_INSPECT | DE_STATE_FLAG_SIG_CANT_MATCH)) != 0) { /* already resolved in an earlier pass */ continue; } /* duplicates of the same signature (state store + per-state * registrations) are adjacent: the candidate list is sorted by id with * the flagged entry first */ DEBUG_VALIDATE_BUG_ON( i > 0 && det_ctx->tx_candidates[i].id < det_ctx->tx_candidates[i - 1].id); if (i > 0 && det_ctx->tx_candidates[i - 1].s == s) { continue; } if (unlikely(s->app_progress_hook >= APP_LAYER_MAX_PROGRESS)) { /* defensive: parse rejects this; skip it so the arrays stay in * bounds and the rule applies its own default policy */ DEBUG_VALIDATE_BUG_ON(1); continue; } can->fw_lte_counted = true; for (uint16_t h = 0; h <= s->app_progress_hook; h++) { det_ctx->fw_lte_cover[h]++; } } } /** \internal * \brief check if a pending LTE rule other than the current one covers its hook. */ static inline bool DetectFwOtherLteCoversHook(DetectEngineThreadCtx *det_ctx, const struct DetectFirewallAppTxState *fw_state, const Signature *s, const bool counted) { if (!fw_state->fw_lte_cover_active) { /* no LTE candidate in this walk, so nothing covers the hook */ return false; } if (unlikely(s->app_progress_hook >= APP_LAYER_MAX_PROGRESS)) { DEBUG_VALIDATE_BUG_ON(1); return false; } const uint32_t covered = det_ctx->fw_lte_cover[s->app_progress_hook]; if (s->flags & SIG_FLAG_FW_HOOK_LTE) { /* an uncounted candidate (header mismatch, already retired) carries * no own count in the coverage */ return covered > (counted ? 1 : 0); } return covered > 0; } /** \internal * \brief count an LTE rule appended mid-walk into the hook coverage. * * Appended candidates are the tail of the list, so duplicates cannot be * detected by adjacency like in DetectFwBuildLteCoverage(). */ static inline void DetectFwCountAppendedLteRule(DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, struct DetectFirewallAppTxState *fw_state, const Signature *s, const uint32_t can_idx) { if (!(s->flags & SIG_FLAG_FW_HOOK_LTE) || !(s->flags & SIG_FLAG_FIREWALL) || s->app_progress_hook >= APP_LAYER_MAX_PROGRESS || !DetectRunInspectRuleHeader(p, f, s, s->flags)) { return; } for (uint32_t k = 0; k < can_idx; k++) { if (det_ctx->tx_candidates[k].s == s && det_ctx->tx_candidates[k].fw_lte_counted) { return; } } DetectFwEnsureLteCoverage(det_ctx, fw_state); for (uint16_t h = 0; h <= s->app_progress_hook; h++) { det_ctx->fw_lte_cover[h]++; } det_ctx->tx_candidates[can_idx].fw_lte_counted = true; det_ctx->tx_candidates[can_idx].fw_lte_header_ok = true; } /** \internal * \brief remove an LTE rule that resolved definitively from the hook coverage. */ static void DetectFwRetireLteRule(DetectEngineThreadCtx *det_ctx, RuleMatchCandidateTx *can) { const Signature *s = can->s; if (unlikely(s->app_progress_hook >= APP_LAYER_MAX_PROGRESS)) { DEBUG_VALIDATE_BUG_ON(1); return; } for (uint16_t h = 0; h <= s->app_progress_hook; h++) { /* the caller only retires counted candidates: a zero here means the * accounting is broken, not a legitimate case */ DEBUG_VALIDATE_BUG_ON(det_ctx->fw_lte_cover[h] == 0); if (det_ctx->fw_lte_cover[h] > 0) { det_ctx->fw_lte_cover[h]--; } } /* the candidate no longer contributes to the counts */ can->fw_lte_counted = false; } static inline void DetectRunAppendDefaultAppPolicyAlert(DetectEngineThreadCtx *det_ctx, Packet *p, const bool apply_to_packet, const DetectTransaction *tx, const struct DetectFirewallAppPolicy *ap) { if (EngineModeIsFirewall()) { const Signature *s = ap->alert_signature; BUG_ON(s == NULL); uint8_t alert_flags = apply_to_packet ? PACKET_ALERT_FLAG_APPLY_ACTION_TO_PACKET : 0; AlertQueueAppendAppTxFromPacket(det_ctx, s, p, tx->tx_id, tx->tx_type, alert_flags); } } /** \internal * \brief apply default policy * \param p packet to apply policy to * \param alproto app proto * \param progress hook / progress value to apply the policy to * * \note alproto and progress are unused right now, will be used * to look up configurable default policies later */ static struct DetectFirewallPolicy DetectFirewallApplyDefaultAppPolicy( DetectEngineThreadCtx *det_ctx, const struct DetectFirewallPolicies *policies, const DetectTransaction *tx, Packet *p, const AppProto alproto, const uint8_t direction, const uint8_t progress) { const uint8_t dir_flags = direction & (STREAM_TOSERVER | STREAM_TOCLIENT); SCLogDebug("packet %" PRIu64 ": tx type %u", PcapPacketCntGet(p), tx->tx_type); const struct DetectFirewallPolicy drop_policy = { .action = ACTION_DROP, .action_scope = ACTION_SCOPE_FLOW }; const struct DetectFirewallAppPolicy lookup = { .alproto = alproto, .sub_state = tx->tx_type, .progress = progress, .direction = dir_flags }; const struct DetectFirewallPolicy *policy = NULL; const struct DetectFirewallAppPolicy *ap = HashTableLookup(policies->app_policies, (void *)&lookup, 0); /* table should be fully populated, so this should not be able to fail. * However as it continues to confuse tooling, at a fallback. */ DEBUG_VALIDATE_BUG_ON(ap == NULL); if (likely(ap != NULL)) { policy = &ap->policy; } else { policy = &drop_policy; } if (policy->action & ACTION_DROP) { SCLogDebug("dropping packet PKT_DROP_REASON_FW_DEFAULT_APP_POLICY"); PacketDrop(p, policy->action, PKT_DROP_REASON_FW_DEFAULT_APP_POLICY); if (policy->action_scope == ACTION_SCOPE_FLOW) { SCLogDebug("dropping flow"); FlowSetDropAction(p->flow, FLOW_ACTION_BY_FIREWALL); } if (policy->action & ACTION_ALERT) { DetectRunAppendDefaultAppPolicyAlert(det_ctx, p, true, tx, ap); } } else if (policy->action & ACTION_ACCEPT) { /* should the accept be applied to the packet? * ACTION_SCOPE_FLOW: yes * ACTION_SCOPE_TX: only if last_tx * ACTION_SCOPE_HOOK: only if last_tx and hook is highest available hook */ const bool last_hook = progress == tx->tx_progress; bool apply_to_packet = false; switch (policy->action_scope) { case ACTION_SCOPE_FLOW: p->flow->flags |= FLOW_ACTION_ACCEPT; apply_to_packet = true; break; case ACTION_SCOPE_TX: tx->tx_data_ptr->flags |= APP_LAYER_TX_ACCEPT; apply_to_packet = tx->is_last; break; case ACTION_SCOPE_HOOK: apply_to_packet = tx->is_last && last_hook; break; default: /* should be unreachable */ DEBUG_VALIDATE_BUG_ON(1); break; } SCLogDebug("packet %" PRIu64 " hook %u default policy ACCEPT, apply_to_packet:%s", PcapPacketCntGet(p), progress, BOOL2STR(apply_to_packet)); if (policy->action & ACTION_ALERT) { SCLogDebug("policy alert, do the append"); DetectRunAppendDefaultAppPolicyAlert(det_ctx, p, apply_to_packet, tx, ap); } else if (apply_to_packet) { SCLogDebug("default accept: last_tx"); DetectRunAppendDefaultAccept(det_ctx, p); } } else { /* should be unreachable */ DEBUG_VALIDATE_BUG_ON(1); } return *policy; } /** \internal * \brief run default policies for hook(s) * * For a range of hooks look up the policy and apply it. * * \param is_last is this tx the last we have? Used to check if an action needs to be applied to * the packet. * * \retval DETECT_TX_FW_FC_BREAK rest of rules shouldn't be inspected * \retval DETECT_TX_FW_FC_SKIP skip current firewall rule * \retval DETECT_TX_FW_FC_OK no action needed */ static enum DetectTxFirewallFlowControl DetectFirewallApplyDefaultPolicies( DetectEngineThreadCtx *det_ctx, const struct DetectFirewallPolicies *policies, DetectTransaction *tx, Packet *p, const AppProto alproto, const uint8_t direction, const uint8_t start_hook, const uint8_t end_hook) { DEBUG_VALIDATE_BUG_ON(start_hook > end_hook); const bool need_verdict = tx->is_last && (end_hook == tx->tx_end_state || end_hook == tx->tx_progress); SCLogDebug("need_verdict:%s is_last:%s end_hook:%u tx->tx_end_state:%u tx->progress: %u", BOOL2STR(need_verdict), BOOL2STR(tx->is_last), end_hook, tx->tx_end_state, tx->tx_progress); for (uint8_t hook = start_hook; hook <= end_hook; hook++) { const bool apply_to_packet = tx->is_last && (hook == tx->tx_end_state || hook == tx->tx_progress); SCLogDebug("%" PRIu64 ": %s default policy for hook %u, apply_to_packet %s", PcapPacketCntGet(p), direction & STREAM_TOSERVER ? "toserver" : "toclient", hook, BOOL2STR(apply_to_packet)); const struct DetectFirewallPolicy policy = DetectFirewallApplyDefaultAppPolicy( det_ctx, policies, tx, p, alproto, direction, hook); SCLogDebug("fw: hook:%u policy:%02x apply_to_packet:%s", hook, policy.action, BOOL2STR(apply_to_packet)); if (policy.action & ACTION_DROP) { SCLogDebug("fw: action %02x", policy.action); return DETECT_TX_FW_FC_BREAK; } else if (policy.action & ACTION_ACCEPT) { SCLogDebug("fw: accept hook %u action %02x", hook, policy.action); /* accepting flow, so skip rest of the fw rules */ if (policy.action_scope == ACTION_SCOPE_FLOW) { SCLogDebug("fw: accept flow"); return DETECT_TX_FW_FC_SKIP; /* accepting flow, so skip rest of the fw rules for this tx */ } else if (policy.action_scope == ACTION_SCOPE_TX) { return DETECT_TX_FW_FC_SKIP; } else if (policy.action_scope == ACTION_SCOPE_HOOK) { /* we're done */ if (apply_to_packet) { return DETECT_TX_FW_FC_SKIP; } } else { DEBUG_VALIDATE_BUG_ON(1); } } else { DEBUG_VALIDATE_BUG_ON(1); } } /* if the tx progress is at end_hook state, it means the rule that called us cannot * match: it's app_progress_hook is not yet available. In this can we need to set * a default accept. This happens if we got only `accept:hook` policies. */ if (need_verdict) { SCLogDebug("default accept: last tx and progress at end_hook %u", end_hook); DetectRunAppendDefaultAccept(det_ctx, p); /* break as we can't match the calling signature */ return DETECT_TX_FW_FC_BREAK; } return DETECT_TX_FW_FC_OK; } /** \internal * \brief run pre-rule inspection firewall policy checks * * Check for: * - check if we're in accept:tx mode * - check for missing accept hooks * - * * \retval DETECT_TX_FW_FC_OK no action needed * \retval DETECT_TX_FW_FC_BREAK rest of rules shouldn't be inspected * \retval DETECT_TX_FW_FC_SKIP skip this rule */ static enum DetectTxFirewallFlowControl DetectRunTxPreCheckFirewallPolicy( DetectEngineThreadCtx *det_ctx, Packet *p, DetectTransaction *tx, const uint8_t direction, const Signature *s, const uint32_t can_idx, const bool lte_counted, struct DetectFirewallAppTxState *fw_state) { SCLogDebug("packet %" PRIu64 ": running pre-checks before sid %u", PcapPacketCntGet(p), s->id); /* enforce skip app filter. If a prior rule caused a fw_skip_app_filter set, we will skip * each fw rule from now. Non-FW rules will just be inspected. Non-FW need to hit this * path to help put into effect the FLOW_ACTION_ACCEPT/APP_LAYER_TX_ACCEPT and call * the default policy enforcement. */ if (fw_state->fw_skip_app_filter) { if ((s->flags & SIG_FLAG_FIREWALL) != 0) { return DETECT_TX_FW_FC_SKIP; } else { return DETECT_TX_FW_FC_OK; } } if (p->flow->flags & FLOW_ACTION_ACCEPT) { fw_state->fw_skip_app_filter = true; SCLogDebug("default accept due to flow accept"); DetectRunAppendDefaultAccept(det_ctx, p); if (s->flags & SIG_FLAG_FIREWALL) { return DETECT_TX_FW_FC_SKIP; } } /* skip fw rules if we're in accept:tx mode */ if (tx->tx_data_ptr->flags & APP_LAYER_TX_ACCEPT) { /* append a blank accept:packet action for the APP_LAYER_TX_ACCEPT, * if this is the last tx */ fw_state->fw_skip_app_filter = true; const bool accept_tx_applies_to_packet = tx->is_last; if (accept_tx_applies_to_packet) { SCLogDebug("accept:tx: should be applied to the packet"); DetectRunAppendDefaultAccept(det_ctx, p); } if (s->flags & SIG_FLAG_FIREWALL) { SCLogDebug("APP_LAYER_TX_ACCEPT, so skip rule"); return DETECT_TX_FW_FC_SKIP; } /* threat detect rules will be inspected */ return DETECT_TX_FW_FC_OK; } /* handle missing rules case */ if (s->flags & SIG_FLAG_FW_HOOK_LTE) { SCLogDebug("SIG_FLAG_FW_HOOK_LTE"); return DETECT_TX_FW_FC_OK; // TODO check for other cases } if (can_idx == 0) { SCLogDebug("check if prior hooks are satisfied: s->app_progress_hook %u, " "tx->detect_progress_orig %u", s->app_progress_hook, tx->detect_progress_orig); if (DetectFwOtherLteCoversHook(det_ctx, fw_state, s, lte_counted)) { SCLogDebug("later LTE in-progress rule brought us here: us:%u covered:%u", s->app_progress_hook, det_ctx->fw_lte_cover[s->app_progress_hook]); return DETECT_TX_FW_FC_OK; } /* if our first rule is beyond the starting state, we need to check if * there are rules missing for states in between. */ if (s->app_progress_hook > tx->detect_progress_orig) { SCLogDebug("missing fw rules at list start: sid %u, progress %u (%u:%u)", s->id, s->app_progress_hook, tx->detect_progress, tx->detect_progress_orig); /* if this rule was after the state we expected meaning that there are * no rules for that state. Invoke the default policies. */ enum DetectTxFirewallFlowControl r = DetectFirewallApplyDefaultPolicies(det_ctx, det_ctx->de_ctx->fw_policies, tx, p, s->alproto, direction, tx->detect_progress_orig, s->app_progress_hook - 1); if (r != DETECT_TX_FW_FC_OK) { /* both SKIP and BREAK mean: no more fw rules to inspect. * SKIP applies to just this TX. * DROP applies to everything. */ fw_state->fw_skip_app_filter = true; } return r; } } return DETECT_TX_FW_FC_OK; } /** * \internal * \brief Check and update firewall rules state. * * \param fw_state pointer to flow control state * * \retval DETECT_TX_FW_FC_OK no action needed * \retval DETECT_TX_FW_FC_BREAK rest of rules shouldn't inspected * \retval DETECT_TX_FW_FC_SKIP skip this rule */ static enum DetectTxFirewallFlowControl DetectRunTxCheckRuleState(DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, DetectTransaction *tx, const Signature *s, const uint32_t can_idx, const uint32_t can_size, const bool lte_counted, struct DetectFirewallAppTxState *fw_state) { if (s->flags & SIG_FLAG_FIREWALL) { /* check if the next sig is on the same progress hook. If not, we need to apply our * default policy in case the current sig doesn't apply one. If the next sig has a * progress beyond our progress + 1, it means the next progress has no rules and needs * the default policy applied. But only after we evaluate the current rule first, as * that may override it. */ if (can_idx + 1 < can_size) { const Signature *next_s = det_ctx->tx_candidates[can_idx + 1].s; SCLogDebug( "peek: peeking at sid %u / progress %u", next_s->id, next_s->app_progress_hook); if (next_s->flags & SIG_FLAG_FIREWALL) { if (s->app_progress_hook != next_s->app_progress_hook) { if (DetectFwOtherLteCoversHook(det_ctx, fw_state, s, lte_counted)) { SCLogDebug("we have another LTE in-progress. Us:%u covered:%u", s->app_progress_hook, det_ctx->fw_lte_cover[s->app_progress_hook]); } else { SCLogDebug("peek: next sid progress %u != current progress %u, so current " "is last for progress", next_s->app_progress_hook, s->app_progress_hook); fw_state->fw_last_for_progress = true; if (next_s->app_progress_hook - s->app_progress_hook > 1) { SCLogDebug("peek: missing progress, so we'll drop that unless we get a " "sweeping accept first"); fw_state->fw_next_progress_missing = true; } } } } else { SCLogDebug("peek: next sid not a fw rule, so current is last for progress"); fw_state->fw_last_for_progress = true; fw_state->last_fw_rule = true; } } else { SCLogDebug("peek: no peek beyond last rule"); if (!DetectFwOtherLteCoversHook(det_ctx, fw_state, s, lte_counted)) { if (s->app_progress_hook < tx->tx_progress) { SCLogDebug("peek: there are no rules to allow the state after this rule"); fw_state->fw_next_progress_missing = true; } fw_state->fw_last_for_progress = true; fw_state->last_fw_rule = true; } } if (fw_state->skip_fw_hook == true) { if (s->app_progress_hook <= fw_state->skip_before_progress) { return DETECT_TX_FW_FC_SKIP; } fw_state->skip_fw_hook = false; } } else { /* fw mode, we skip anything after the fw rules if: * - flow pass is set * - packet pass (e.g. exception policy) */ if (p->flags & PKT_NOPACKET_INSPECTION || (f->flags & (FLOW_ACTION_PASS))) { SCLogDebug("skipping firewall rule %u", s->id); return DETECT_TX_FW_FC_BREAK; } } return DETECT_TX_FW_FC_OK; } // TODO move into det_ctx? thread_local Signature default_accept; static void DetectRunAppendDefaultAccept(DetectEngineThreadCtx *det_ctx, Packet *p) { DEBUG_VALIDATE_BUG_ON(!EngineModeIsFirewall()); SCLogDebug("packet %" PRIu64 ": appending default firewall accept", PcapPacketCntGet(p)); memset(&default_accept, 0, sizeof(default_accept)); default_accept.action = ACTION_ACCEPT; default_accept.action_scope = ACTION_SCOPE_PACKET; default_accept.iid = UINT32_MAX; default_accept.type = SIG_TYPE_PKT; default_accept.flags = SIG_FLAG_FIREWALL; default_accept.detect_table = DETECT_TABLE_APP_FILTER; // TODO review, hope this makes it last in sorting AlertQueueAppendPacket(det_ctx, &default_accept, p, PACKET_ALERT_FLAG_APPLY_ACTION_TO_PACKET); } /** \internal * \brief see if the accept rule needs to apply to the packet */ static inline bool ApplyAcceptToPacket(const DetectTransaction *tx, const Signature *s) { if ((s->flags & SIG_FLAG_FIREWALL) == 0) { return false; } if ((s->action & ACTION_ACCEPT) == 0) { return false; } /* for accept:tx we need: * - packet will only be accepted if this is set on the last tx */ if (s->action_scope == ACTION_SCOPE_TX) { if (tx->is_last) { return true; } } /* for accept:hook we need a bit more checking: * - packet will only be accepted if this is set on the last tx * - the hook accepted should be the last progress available. */ if (s->action_scope == ACTION_SCOPE_HOOK) { if (tx->is_last && (s->app_progress_hook == tx->tx_progress)) { return true; } } return false; } /** \internal * \brief apply an accept, but do check policies when needed * * Updates flow control where needed. * * */ static void DetectRunTxFirewallApplyAccept(DetectEngineThreadCtx *det_ctx, Packet *p, const uint8_t direction, const Signature *s, DetectTransaction *tx, struct DetectFirewallAppTxState *fw_state) { const enum ActionScope as = s->action_scope; /* accept:hook: jump to first rule of next state. * Implemented as skip until the first rule of next state. */ if (as == ACTION_SCOPE_HOOK) { fw_state->skip_fw_hook = true; fw_state->skip_before_progress = s->app_progress_hook; SCLogDebug("fw match sid:%u hook:%u", s->id, s->app_progress_hook); SCLogDebug("fw fw_skip_app_filter:%s skip_fw_hook:%s " "skip_before_progress:%u fw_last_for_progress:%s fw_next_progress_missing:%s", BOOL2STR(fw_state->fw_skip_app_filter), BOOL2STR(fw_state->skip_fw_hook), fw_state->skip_before_progress, BOOL2STR(fw_state->fw_last_for_progress), BOOL2STR(fw_state->fw_next_progress_missing)); /* if there is no fw rule for the next progress value, * we invoke the defaul policies for the remaining available hooks. * Only when the tx walked over this rule's hook: at progress == hook * the sweep range would start above its end. */ if (fw_state->fw_next_progress_missing && tx->tx_progress > s->app_progress_hook) { /* only states the packet actually walked over: states beyond the * current progress have not been reached, their defaults must not * decide this packet */ const uint8_t last_hook = fw_state->last_fw_rule ? tx->tx_progress : MIN(tx->tx_progress, s->app_progress_hook + 1); enum DetectTxFirewallFlowControl r = DetectFirewallApplyDefaultPolicies(det_ctx, det_ctx->de_ctx->fw_policies, tx, p, s->alproto, direction, s->app_progress_hook + 1, last_hook); if (r == DETECT_TX_FW_FC_BREAK) { fw_state->fw_skip_app_filter = true; return; } } } else if (as == ACTION_SCOPE_TX) { tx->tx_data_ptr->flags |= APP_LAYER_TX_ACCEPT; fw_state->skip_fw_hook = true; fw_state->skip_before_progress = tx->tx_end_state + 1; // skip all hooks SCLogDebug("accept:tx applied, skip_fw_hook, skip_before_progress %u", fw_state->skip_before_progress); } else if (as == ACTION_SCOPE_PACKET) { fw_state->fw_skip_app_filter = true; } else if (as == ACTION_SCOPE_FLOW) { SCLogDebug("sid %u: ACTION_ACCEPT with ACTION_SCOPE_FLOW", s->id); fw_state->fw_skip_app_filter = true; } } /** * \internal * \brief check if there are no (fw) rules, and apply the default policies if so * * \retval 3 policies handled, continue with inspection * \retval 2 continue with next tx * \retval 1 done with inspection * \retval 0 ok, continue as normal. No policies applied. */ static int DetectTxFirewallNoRulesApplyPolicies(DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, DetectTransaction *tx, const AppProto alproto, const uint8_t flow_flags, const int rule_cnt) { /* if there are no rules / rule candidates, handling invoking the default * policy. */ if (rule_cnt == 0 || (det_ctx->tx_candidates[0].s->flags & SIG_FLAG_FIREWALL) == 0) { /* if there are no rules, make sure to handle accept:flow and accept:tx */ if (rule_cnt == 0) { if (f->flags & FLOW_ACTION_ACCEPT) { SCLogDebug("default accept:flow: no rules"); DetectRunAppendDefaultAccept(det_ctx, p); return 1; } if (tx->tx_data_ptr->flags & APP_LAYER_TX_ACCEPT) { /* current tx is the last we have, append a blank accept:packet */ if (tx->is_last) { SCLogDebug("default accept:tx: no rules"); DetectRunAppendDefaultAccept(det_ctx, p); return 1; } return 2; } } /* if there are no fw rules, handle default policies */ if ((f->flags & FLOW_ACTION_ACCEPT) == 0 && (tx->tx_data_ptr->flags & APP_LAYER_TX_ACCEPT) == 0) { /* No rules to eval, so we need to see if there are default policies to apply. * Start at last inspected progress and check each hook. If all hooks accepted, * apply the accept to the packet. */ SCLogDebug("tx.detect_progress_orig %u tx.tx_progress %u", tx->detect_progress_orig, tx->tx_progress); enum DetectTxFirewallFlowControl r = DetectFirewallApplyDefaultPolicies(det_ctx, det_ctx->de_ctx->fw_policies, tx, p, alproto, flow_flags & (STREAM_TOSERVER | STREAM_TOCLIENT), tx->detect_progress_orig, tx->tx_progress); SCLogDebug("r %u", r); if (r == DETECT_TX_FW_FC_BREAK) return 1; if (r == DETECT_TX_FW_FC_SKIP) return 2; /* continue with TD rules */ SCLogDebug("continue with TD rules"); return 3; } } return 0; } /** \internal * \brief handle a full rule match for a firewall rule * * For a drop/reject rule, excute action immediately. * For an accept rule, add an alert even to the queue. This will * allow TD rules to override the accept. */ static void DetectRunTxFirewallRuleFullMatch(DetectEngineThreadCtx *det_ctx, const Signature *s, DetectTransaction *tx, struct DetectFirewallAppTxState *fw_state, Flow *f, Packet *p, const uint8_t flow_flags) { if (s->action & ACTION_ACCEPT) { /* add alert now, as ApplyAccept may also trigger * policy matches that could add alerts. */ SCLogDebug("append alert"); /* see if we need to apply tx/hook accept to the packet. This can be needed * when we've completed the inspection so far for an incomplete tx, and an * accept:tx or accept:hook is the last match.*/ const bool fw_accept_to_packet = ApplyAcceptToPacket(tx, s); if (fw_accept_to_packet) { SCLogDebug("packet %" PRIu64 ": apply accept to packet", PcapPacketCntGet(p)); SCLogDebug("accept:(tx|hook): should be applied to the packet"); AlertQueueAppendAppTx(det_ctx, s, p, tx->tx_id, tx->tx_type, PACKET_ALERT_FLAG_APPLY_ACTION_TO_PACKET); } else { AlertQueueAppendAppTx(det_ctx, s, p, tx->tx_id, tx->tx_type, 0); } DetectRunTxFirewallApplyAccept(det_ctx, p, flow_flags, s, tx, fw_state); } else if (s->action & ACTION_DROP) { SCLogDebug("drop packet because of rule with drop action"); PacketDrop(p, s->action, PKT_DROP_REASON_FW_RULES); if (s->action_scope == ACTION_SCOPE_FLOW) { SCLogDebug("drop flow because of rule with drop action"); FlowSetDropAction(f, FLOW_ACTION_BY_FIREWALL); } SCLogDebug("append alert"); AlertQueueAppendAppTx(det_ctx, s, p, tx->tx_id, tx->tx_type, 0); } else { SCLogDebug("append alert"); AlertQueueAppendAppTx(det_ctx, s, p, tx->tx_id, tx->tx_type, 0); } } /** \internal * \brief handle a partial match for firewall rules * * Currently only used for LTE mode. Regardless of the final action, * the partial match acts as a `accept:hook`. * * A default accept is appended. TD has a chance to override this accept. * * \retval 1 accept partial, caller must break loop * \retval 0 ok, caller must continue as normal */ static int DetectRunTxFirewallRulePartialMatch( DetectEngineThreadCtx *det_ctx, const Signature *s, const DetectTransaction *tx, Packet *p) { if ((s->flags & SIG_FLAG_FIREWALL) && (s->action & ACTION_ACCEPT)) { /* partial match always uses ACTION_SCOPE_HOOK. Final action only on the full * match */ if (tx->is_last) { SCLogDebug("need to apply accept to packet"); DetectRunAppendDefaultAccept(det_ctx, p); } if (s->action_scope == ACTION_SCOPE_FLOW) { SCLogDebug("only applying accept:flow on full match, downgrading to " "accept:hook"); } return 1; } return 0; } /** \internal * \brief handle the no-match case for a firewall rule * * If the rule did not match we need to see if we need invoke the default * policy for this hook. If that is the case, we handle a drop by telling the * caller about it. Flow control for various accept options is handled by the * next rule. * * \retval 1 dropped by policy, caller must return * \retval 0 ok, caller must continue as normal */ static int DetectRunTxFirewallRuleNoMatch(DetectEngineThreadCtx *det_ctx, const Signature *s, DetectTransaction *tx, struct DetectFirewallAppTxState *fw_state, Packet *p, const uint8_t flow_flags) { if (fw_state->fw_last_for_progress && (s->flags & SIG_FLAG_FIREWALL)) { SCLogDebug("%" PRIu64 ": %s default policy for progress %u", PcapPacketCntGet(p), flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", s->app_progress_hook); /* if this rule was the last for our progress state, and it didn't match, * we have to invoke the default policy. We only check the current rule hook. * DROP is immediate, flow control for various accept options is handled by * the DetectRunTxPreCheckFirewallPolicy function for the next rule. */ const struct DetectFirewallPolicy policy = DetectFirewallApplyDefaultAppPolicy(det_ctx, det_ctx->de_ctx->fw_policies, tx, p, s->alproto, flow_flags, s->app_progress_hook); SCLogDebug("fw_last_for_progress policy %02x", policy.action); if (policy.action & ACTION_DROP) { return 1; } /* an accept with tx or flow scope already carries the verdict. For an * accept:hook (which does not apply to this packet) the states walked * over must still be decided: if the tx progressed beyond this rule's * hook and there is no rule for the states in between, their default * policies must be applied too - same as the match path does. * Without this a packet that crosses two states (e.g. a hello * completing into the certificate phase) would be decided by the * packet default policy even though the app policy for the current * state says accept. */ if ((policy.action & ACTION_ACCEPT) && policy.action_scope == ACTION_SCOPE_HOOK && fw_state->fw_next_progress_missing && tx->tx_progress > s->app_progress_hook) { /* only states the packet actually walked over (see the match * path) */ const uint8_t last_hook = fw_state->last_fw_rule ? tx->tx_progress : MIN(tx->tx_progress, s->app_progress_hook + 1); enum DetectTxFirewallFlowControl r = DetectFirewallApplyDefaultPolicies(det_ctx, det_ctx->de_ctx->fw_policies, tx, p, s->alproto, flow_flags, s->app_progress_hook + 1, last_hook); if (r == DETECT_TX_FW_FC_BREAK) { /* the sweep dropped the packet: same as a current-hook * policy drop, the caller must stop the walk */ fw_state->fw_skip_app_filter = true; return 1; } } } return 0; } /** \internal * \brief handle full match on rule when state didn't progress yet * * For a full rule match on a certain hook, we need to continue to enforce the * match as long as the tx progress doesn't move beyond that hook. */ static void DetectRunTxFirewallRuleStatefulReApplyMatch(DetectEngineThreadCtx *det_ctx, const Signature *s, DetectTransaction *tx, struct DetectFirewallAppTxState *fw_state, Packet *p, const uint8_t flow_flags) { /* if we're still in the same progress state as an earlier full * match, we need to apply the same accept */ if ((s->flags & SIG_FLAG_FIREWALL) && (s->action & ACTION_ACCEPT) && s->app_progress_hook == tx->tx_progress) { const bool fw_accept_to_packet = ApplyAcceptToPacket(tx, s); DetectRunTxFirewallApplyAccept(det_ctx, p, flow_flags, s, tx, fw_state); if (fw_accept_to_packet) { SCLogDebug("packet %" PRIu64 ": apply accept to packet", PcapPacketCntGet(p)); DetectRunAppendDefaultAccept(det_ctx, p); } } } static void DetectRunTx(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, DetectRunScratchpad *scratch) { const uint8_t flow_flags = scratch->flow_flags; const SigGroupHead * const sgh = scratch->sgh; void * const alstate = f->alstate; const uint8_t ipproto = f->proto; const AppProto alproto = f->alproto; const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate); uint64_t tx_id_min = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags); AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto); AppLayerGetTxIterState state = { 0 }; uint32_t tx_inspected = 0; const bool have_fw_rules = EngineModeIsFirewall(); /* if we skipped the last tx, we did not have a chance to apply a fw accept to the packet. Since * the tx is skipped, we should consider it accepted. */ bool last_tx_skipped = false; SCLogDebug("packet %" PRIu64, PcapPacketCntGet(p)); SCLogDebug("total_txs %" PRIu64, total_txs); while (1) { AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id_min, total_txs, &state); if (ires.tx_ptr == NULL) { SCLogDebug("%p/%" PRIu64 " no transaction to inspect", ires.tx_ptr, tx_id_min); break; } DetectTransaction tx = GetDetectTx(ipproto, alproto, ires.tx_id, ires.tx_ptr, flow_flags); if (tx.tx_ptr == NULL) { SCLogDebug("%p/%"PRIu64" no transaction to inspect", tx.tx_ptr, tx_id_min); last_tx_skipped = !ires.has_next; tx_id_min++; // next (if any) run look for +1 goto next; } tx.is_last = !ires.has_next; tx_id_min = tx.tx_id + 1; // next look for cur + 1 tx_inspected++; SCLogDebug("%p/%" PRIu64 " txd flags %02x", tx.tx_ptr, tx.tx_id, tx.tx_data_ptr->flags); SCLogDebug("%p/%" PRIu64 " is_last %s", tx.tx_ptr, tx.tx_id, BOOL2STR(tx.is_last)); det_ctx->tx_id = tx.tx_id; det_ctx->tx_id_set = true; det_ctx->p = p; bool do_sort = false; // do we need to sort the tx candidate list? bool fw_lte_candidates = false; // does the list have a pending LTE rule? uint32_t array_idx = 0; uint32_t total_rules = det_ctx->match_array_cnt; total_rules += (tx.de_state ? tx.de_state->cnt : 0); /* run prefilter engines and merge results into a candidates array */ if (sgh && sgh->tx_engines) { PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_TX); DetectRunPrefilterTx(det_ctx, sgh, p, ipproto, flow_flags, alproto, alstate, &tx); PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_TX); SCLogDebug("%p/%"PRIu64" rules added from prefilter: %u candidates", tx.tx_ptr, tx.tx_id, det_ctx->pmq.rule_id_array_cnt); total_rules += det_ctx->pmq.rule_id_array_cnt; if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) { RuleMatchCandidateTxArrayExpand(det_ctx, total_rules); } for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) { const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]]; const SigIntId id = s->iid; det_ctx->tx_candidates[array_idx].s = s; det_ctx->tx_candidates[array_idx].id = id; det_ctx->tx_candidates[array_idx].flags = NULL; det_ctx->tx_candidates[array_idx].fw_lte_counted = false; det_ctx->tx_candidates[array_idx].fw_lte_header_ok = false; det_ctx->tx_candidates[array_idx].stream_reset = 0; if (s->flags & SIG_FLAG_FW_HOOK_LTE) { fw_lte_candidates = true; } array_idx++; } PMQ_RESET(&det_ctx->pmq); } else { if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, total_rules))) { RuleMatchCandidateTxArrayExpand(det_ctx, total_rules); } } /* merge 'state' rules from the regular prefilter */ #ifdef PROFILING uint32_t x = array_idx; #endif RuleMatchCandidateMergeStateRules(det_ctx, &array_idx, &fw_lte_candidates); /* merge stored state into results */ if (tx.de_state != NULL) { const uint32_t old = array_idx; /* if tx.de_state->flags has 'new file' set and sig below has * 'file inspected' flag, reset the file part of the state */ const bool have_new_file = (tx.de_state->flags & DETECT_ENGINE_STATE_FLAG_FILE_NEW); if (have_new_file) { SCLogDebug("%p/%"PRIu64" destate: need to consider new file", tx.tx_ptr, tx.tx_id); tx.de_state->flags &= ~DETECT_ENGINE_STATE_FLAG_FILE_NEW; } SigIntId state_cnt = 0; DeStateStore *tx_store = tx.de_state->head; for (; tx_store != NULL; tx_store = tx_store->next) { SCLogDebug("tx_store %p", tx_store); SigIntId store_cnt = 0; for (store_cnt = 0; store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < tx.de_state->cnt; store_cnt++, state_cnt++) { DeStateStoreItem *item = &tx_store->store[store_cnt]; SCLogDebug("rule id %u, inspect_flags %u", item->sid, item->flags); if (have_new_file && (item->flags & DE_STATE_FLAG_FILE_INSPECT)) { /* remove part of the state. File inspect engine will now * be able to run again */ item->flags &= ~(DE_STATE_FLAG_SIG_CANT_MATCH|DE_STATE_FLAG_FULL_INSPECT|DE_STATE_FLAG_FILE_INSPECT); SCLogDebug("rule id %u, post file reset inspect_flags %u", item->sid, item->flags); } const Signature *ss = de_ctx->sig_array[item->sid]; det_ctx->tx_candidates[array_idx].s = ss; det_ctx->tx_candidates[array_idx].id = item->sid; det_ctx->tx_candidates[array_idx].flags = &item->flags; det_ctx->tx_candidates[array_idx].fw_lte_counted = false; det_ctx->tx_candidates[array_idx].fw_lte_header_ok = false; det_ctx->tx_candidates[array_idx].stream_reset = 0; if (ss->flags & SIG_FLAG_FW_HOOK_LTE) { fw_lte_candidates = true; } array_idx++; } } do_sort |= (array_idx > old); // sort if continue list adds sids SCLogDebug("%p/%" PRIu64 " rules added from 'continue' list: %u", tx.tx_ptr, tx.tx_id, array_idx - old); } if (do_sort) { qsort(det_ctx->tx_candidates, array_idx, sizeof(RuleMatchCandidateTx), DetectRunTxSortHelper); } #ifdef PROFILING if (array_idx >= de_ctx->profile_match_logging_threshold) RulesDumpTxMatchArray(det_ctx, scratch->sgh, p, tx.tx_id, array_idx, x); #endif #ifdef DEBUG for (uint32_t i = 0; i < array_idx; i++) { RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i]; const Signature *s = det_ctx->tx_candidates[i].s; SCLogDebug("%u: sid %u flags %p", i, s->id, can->flags); } #endif /* Only firewall mode uses the per-tx firewall state. The LTE flag is * tracked where the candidates are added: the list mixes fw and TD * rules across tables in iid order, so a scan cannot stop early. */ struct DetectFirewallAppTxState fw_state = { 0 }; if (have_fw_rules && fw_lte_candidates) { DetectFwBuildLteCoverage(det_ctx, p, f, &fw_state, array_idx); } SCLogDebug("%s: tx_progress %u tx %p have_fw_rules %s array_idx %u detect_progress_orig %u " "cur detect_progress %u", flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", tx.tx_progress, tx.tx_data_ptr, BOOL2STR(have_fw_rules), array_idx, tx.detect_progress_orig, tx.detect_progress); if (have_fw_rules) { /* if there are no firewall rules to consider, handle invoking the default * policies. */ const int r = DetectTxFirewallNoRulesApplyPolicies( det_ctx, p, f, &tx, alproto, flow_flags, array_idx); if (r == 1) { SCLogDebug("done"); return; } else if (r == 2) { goto next_tx_fw; /* next tx, need to clean up buffers */ } } /* run rules: inspect the match candidates */ for (uint32_t i = 0; i < array_idx; i++) { RuleMatchCandidateTx *can = &det_ctx->tx_candidates[i]; const Signature *s = det_ctx->tx_candidates[i].s; uint32_t *inspect_flags = det_ctx->tx_candidates[i].flags; SCLogDebug("%" PRIu64 ": sid:%u: %s tx %u/%u/%u sig %u", PcapPacketCntGet(p), s->id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", tx.tx_progress, tx.detect_progress, tx.detect_progress_orig, s->app_progress_hook); if (have_fw_rules) { const enum DetectTxFirewallFlowControl fw_r = DetectRunTxPreCheckFirewallPolicy( det_ctx, p, &tx, flow_flags & (STREAM_TOSERVER | STREAM_TOCLIENT), s, i, can->fw_lte_counted, &fw_state); SCLogDebug("fw fw_skip_app_filter:%s skip_fw_hook:%s " "skip_before_progress:%u fw_last_for_progress:%s " "fw_next_progress_missing:%s", BOOL2STR(fw_state.fw_skip_app_filter), BOOL2STR(fw_state.skip_fw_hook), fw_state.skip_before_progress, BOOL2STR(fw_state.fw_last_for_progress), BOOL2STR(fw_state.fw_next_progress_missing)); if (fw_r == DETECT_TX_FW_FC_SKIP) { continue; } else if (fw_r == DETECT_TX_FW_FC_BREAK) { break; } fw_state.fw_last_for_progress = false; fw_state.fw_next_progress_missing = false; // reset } /* deduplicate: rules_array is sorted, but not deduplicated: * both mpm and stored state could give us the same sid. * As they are back to back in that case we can check for it * here. We select the stored state one as that comes first * in the array. */ while ((i + 1) < array_idx && det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i + 1].s) { SCLogDebug("%p/%" PRIu64 " inspecting SKIP NEXT: sid %u (%u), flags %08x", tx.tx_ptr, tx.tx_id, s->id, s->iid, inspect_flags ? *inspect_flags : 0); i++; } SCLogDebug("%p/%" PRIu64 " inspecting: sid %u (%u), flags %08x", tx.tx_ptr, tx.tx_id, s->id, s->iid, inspect_flags ? *inspect_flags : 0); if (inspect_flags) { if (*inspect_flags & DE_STATE_FLAG_FULL_INSPECT) { SCLogDebug("%p/%" PRIu64 " inspecting: sid %u (%u), flags %08x DE_STATE_FLAG_FULL_INSPECT", tx.tx_ptr, tx.tx_id, s->id, s->iid, *inspect_flags); /* if we're still in the same progress state as an earlier full * match, we need to apply the same accept */ if (have_fw_rules) { DetectRunTxFirewallRuleStatefulReApplyMatch( det_ctx, s, &tx, &fw_state, p, flow_flags); } continue; } if (*inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) { SCLogDebug("%p/%" PRIu64 " inspecting: sid %u (%u), flags %08x DE_STATE_FLAG_SIG_CANT_MATCH", tx.tx_ptr, tx.tx_id, s->id, s->iid, *inspect_flags); continue; } /* continue previous inspection */ SCLogDebug("%p/%" PRIu64 " Continuing sid %u", tx.tx_ptr, tx.tx_id, s->id); } else { /* start new inspection */ SCLogDebug("%p/%"PRIu64" Start sid %u", tx.tx_ptr, tx.tx_id, s->id); } if (have_fw_rules) { /* check if we should run this rule and update the firewall flow state */ const enum DetectTxFirewallFlowControl fw_r = DetectRunTxCheckRuleState( det_ctx, p, f, &tx, s, i, array_idx, can->fw_lte_counted, &fw_state); SCLogDebug("fw fw_skip_app_filter:%s skip_fw_hook:%s " "skip_before_progress:%u fw_last_for_progress:%s " "fw_next_progress_missing:%s", BOOL2STR(fw_state.fw_skip_app_filter), BOOL2STR(fw_state.skip_fw_hook), fw_state.skip_before_progress, BOOL2STR(fw_state.fw_last_for_progress), BOOL2STR(fw_state.fw_next_progress_missing)); if (fw_r == DETECT_TX_FW_FC_SKIP) continue; else if (fw_r == DETECT_TX_FW_FC_BREAK) break; } /* call individual rule inspection */ RULE_PROFILING_START(p); const int r = DetectRunTxInspectRule(tv, de_ctx, det_ctx, p, f, flow_flags, alstate, &tx, s, inspect_flags, can, scratch); SCLogDebug("s %u r %d", s->id, r); if (have_fw_rules && r == -1 && (s->flags & SIG_FLAG_FW_HOOK_LTE) && can->fw_lte_counted && tx.tx_progress >= s->app_progress_hook) { DetectFwRetireLteRule(det_ctx, can); } if (r == 1) { /* match */ DetectRunPostMatch(tv, det_ctx, p, s); SCLogDebug( "%p/%" PRIu64 " sig %u (%u) matched", tx.tx_ptr, tx.tx_id, s->id, s->iid); if ((s->flags & SIG_FLAG_FIREWALL) == 0) { AlertQueueAppendAppTx(det_ctx, s, p, tx.tx_id, tx.tx_type, 0); } else { DetectRunTxFirewallRuleFullMatch(det_ctx, s, &tx, &fw_state, f, p, flow_flags); } } else if (r == 0) { SCLogDebug("sid %u partial match", s->id); if (DetectRunTxFirewallRulePartialMatch(det_ctx, s, &tx, p) == 1) { break; } } else if (r == -1) { if ((s->flags & SIG_FLAG_FIREWALL) != 0 && DetectRunTxFirewallRuleNoMatch(det_ctx, s, &tx, &fw_state, p, flow_flags) == 1) { return; } } /* r == -2: no match, but not final: the rule is revisited, so a * hook default now could decide the packet before the revisit can */ DetectVarProcessList(det_ctx, p->flow, p); RULE_PROFILING_END(det_ctx, s, r == 1, p); if (det_ctx->post_rule_work_queue.len > 0) { SCLogDebug("%p/%" PRIu64 " post_rule_work_queue len %u", tx.tx_ptr, tx.tx_id, det_ctx->post_rule_work_queue.len); /* run post match prefilter engines on work queue */ PrefilterPostRuleMatch(det_ctx, scratch->sgh, p, f); uint32_t prev_array_idx = array_idx; /* Check if there is room for the pmq results before appending them. */ const uint32_t needed = array_idx + det_ctx->pmq.rule_id_array_cnt; if (!(RuleMatchCandidateTxArrayHasSpace(det_ctx, needed))) { RuleMatchCandidateTxArrayExpand(det_ctx, needed); } for (uint32_t j = 0; j < det_ctx->pmq.rule_id_array_cnt; j++) { const Signature *ts = de_ctx->sig_array[det_ctx->pmq.rule_id_array[j]]; if (ts->app_inspect != NULL) { const SigIntId id = ts->iid; det_ctx->tx_candidates[array_idx].s = ts; det_ctx->tx_candidates[array_idx].id = id; det_ctx->tx_candidates[array_idx].fw_lte_counted = false; det_ctx->tx_candidates[array_idx].fw_lte_header_ok = false; det_ctx->tx_candidates[array_idx].flags = NULL; det_ctx->tx_candidates[array_idx].stream_reset = 0; /* a mid-walk enabled LTE rule must count as coverage * for the remaining checks of this walk */ if (have_fw_rules) { DetectFwCountAppendedLteRule(det_ctx, p, f, &fw_state, ts, array_idx); } array_idx++; SCLogDebug("%p/%" PRIu64 " rule %u (%u) added from 'post match' prefilter", tx.tx_ptr, tx.tx_id, ts->id, id); } } SCLogDebug("%p/%" PRIu64 " rules added from 'post match' prefilter: %u", tx.tx_ptr, tx.tx_id, array_idx - prev_array_idx); if (prev_array_idx != array_idx) { /* sort, but only part of array we're still going to process */ qsort(det_ctx->tx_candidates + i, array_idx - i, sizeof(RuleMatchCandidateTx), DetectRunTxSortHelper); } det_ctx->post_rule_work_queue.len = 0; PMQ_RESET(&det_ctx->pmq); } } det_ctx->tx_id = 0; det_ctx->tx_id_set = false; det_ctx->p = NULL; /* see if we have any updated state to store in the tx */ /* this side of the tx is done */ if (tx.tx_progress >= tx.tx_end_state) { SCLogDebug("%" PRIu64 ": %s tx done", PcapPacketCntGet(p), flow_flags & STREAM_TOSERVER ? "toserver" : "toclient"); const uint8_t inspected_flag = (flow_flags & STREAM_TOSERVER) ? APP_LAYER_TX_INSPECTED_TS : APP_LAYER_TX_INSPECTED_TC; tx.tx_data_ptr->flags |= inspected_flag; SCLogDebug("%p/%" PRIu64 " tx is done for direction %s. Progress %02x", tx.tx_ptr, tx.tx_id, flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", tx.detect_progress); } if (tx.detect_progress != tx.detect_progress_orig) { SCLogDebug("%" PRIu64 ": %s tx state change %u -> %u", PcapPacketCntGet(p), flow_flags & STREAM_TOSERVER ? "toserver" : "toclient", tx.detect_progress_orig, tx.detect_progress); SCLogDebug("%p/%" PRIu64 " Storing new progress %02x (was %02x)", tx.tx_ptr, tx.tx_id, tx.detect_progress, tx.detect_progress_orig); StoreDetectProgress(&tx, flow_flags, tx.detect_progress); } next_tx_fw: InspectionBufferClean(det_ctx); next: if (!ires.has_next) break; } SCLogDebug("packet %" PRIu64 ": tx_inspected %u", PcapPacketCntGet(p), tx_inspected); if (have_fw_rules) { if (tx_inspected == 0) { /* if all tables have been bypassed, we accept:packet */ SCLogDebug("default accept: no app inspect performed"); DetectRunAppendDefaultAccept(det_ctx, p); } else if (last_tx_skipped) { /* if the last tx was skipped, we need to apply accept:packet */ // TODO should we check drops first? DEBUG_VALIDATE_BUG_ON(p->action & ACTION_DROP); SCLogDebug("default accept: last tx skipped"); DetectRunAppendDefaultAccept(det_ctx, p); } } } static void DetectRunFrames(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p, Flow *f, DetectRunScratchpad *scratch) { const SigGroupHead *const sgh = scratch->sgh; const AppProto alproto = f->alproto; /* for TCP, limit inspection to pseudo packets or real packet that did * an app-layer update. */ if (p->proto == IPPROTO_TCP && !PKT_IS_PSEUDOPKT(p) && ((PKT_IS_TOSERVER(p) && (f->flags & FLOW_TS_APP_UPDATED) == 0) || (PKT_IS_TOCLIENT(p) && (f->flags & FLOW_TC_APP_UPDATED) == 0))) { SCLogDebug("pcap_cnt %" PRIu64 ": %s: skip frame inspection for TCP w/o APP UPDATE", PcapPacketCntGet(p), PKT_IS_TOSERVER(p) ? "toserver" : "toclient"); return; } FramesContainer *frames_container = AppLayerFramesGetContainer(f); if (frames_container == NULL) { return; } Frames *frames; if (PKT_IS_TOSERVER(p)) { frames = &frames_container->toserver; } else { frames = &frames_container->toclient; } for (uint32_t idx = 0; idx < frames->cnt; idx++) { SCLogDebug("frame %u", idx); Frame *frame = FrameGetByIndex(frames, idx); if (frame == NULL) { continue; } det_ctx->frame_inspect_progress = 0; uint32_t array_idx = 0; uint32_t total_rules = det_ctx->match_array_cnt; /* run prefilter engines and merge results into a candidates array */ if (sgh->frame_engines) { // PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_TX); DetectRunPrefilterFrame(det_ctx, sgh, p, frames, frame, alproto); // PACKET_PROFILING_DETECT_END(p, PROF_DETECT_PF_TX); SCLogDebug("%p/%" PRIi64 " rules added from prefilter: %u candidates", frame, frame->id, det_ctx->pmq.rule_id_array_cnt); total_rules += det_ctx->pmq.rule_id_array_cnt; if (!(RuleMatchCandidateTxArrayHasSpace( det_ctx, total_rules))) { // TODO is it safe to overload? RuleMatchCandidateTxArrayExpand(det_ctx, total_rules); } for (uint32_t i = 0; i < det_ctx->pmq.rule_id_array_cnt; i++) { const Signature *s = de_ctx->sig_array[det_ctx->pmq.rule_id_array[i]]; const SigIntId id = s->iid; det_ctx->tx_candidates[array_idx].s = s; det_ctx->tx_candidates[array_idx].id = id; det_ctx->tx_candidates[array_idx].flags = NULL; det_ctx->tx_candidates[array_idx].fw_lte_counted = false; det_ctx->tx_candidates[array_idx].fw_lte_header_ok = false; det_ctx->tx_candidates[array_idx].stream_reset = 0; array_idx++; } PMQ_RESET(&det_ctx->pmq); } /* merge 'state' rules from the regular prefilter */ uint32_t x = array_idx; for (uint32_t i = 0; i < det_ctx->match_array_cnt; i++) { const Signature *s = det_ctx->match_array[i]; if (s->frame_inspect != NULL) { const SigIntId id = s->iid; det_ctx->tx_candidates[array_idx].s = s; det_ctx->tx_candidates[array_idx].id = id; det_ctx->tx_candidates[array_idx].flags = NULL; det_ctx->tx_candidates[array_idx].fw_lte_counted = false; det_ctx->tx_candidates[array_idx].fw_lte_header_ok = false; det_ctx->tx_candidates[array_idx].stream_reset = 0; array_idx++; SCLogDebug("%p/%" PRIi64 " rule %u (%u) added from 'match' list", frame, frame->id, s->id, id); } } SCLogDebug("%p/%" PRIi64 " rules added from 'match' list: %u", frame, frame->id, array_idx - x); (void)x; /* run rules: inspect the match candidates */ for (uint32_t i = 0; i < array_idx; i++) { const Signature *s = det_ctx->tx_candidates[i].s; /* deduplicate: rules_array is sorted, but not deduplicated. * As they are back to back in that case we can check for it * here. We select the stored state one as that comes first * in the array. */ while ((i + 1) < array_idx && det_ctx->tx_candidates[i].s == det_ctx->tx_candidates[i + 1].s) { i++; } SCLogDebug("%p/%" PRIi64 " inspecting: sid %u (%u)", frame, frame->id, s->id, s->iid); /* start new inspection */ SCLogDebug("%p/%" PRIi64 " Start sid %u", frame, frame->id, s->id); /* call individual rule inspection */ RULE_PROFILING_START(p); bool r = DetectRunInspectRuleHeader(p, f, s, s->flags); if (r) { r = DetectRunFrameInspectRule(tv, det_ctx, s, f, p, frames, frame); if (r) { /* match */ DetectRunPostMatch(tv, det_ctx, p, s); det_ctx->frame_id = frame->id; SCLogDebug( "%p/%" PRIi64 " sig %u (%u) matched", frame, frame->id, s->id, s->iid); const uint8_t alert_flags = (PACKET_ALERT_FLAG_STATE_MATCH | PACKET_ALERT_FLAG_FRAME); if (frame->flags & FRAME_FLAG_TX_ID_SET) { const uint8_t ipproto = p->proto; uint8_t sub_state = 0; void *tx = AppLayerParserGetTx( ipproto, alproto, p->flow->alstate, frame->tx_id); if (tx) { AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx); sub_state = txd->tx_type; } AlertQueueAppendAppTx(det_ctx, s, p, frame->tx_id, sub_state, alert_flags); } else { AlertQueueAppendPacket(det_ctx, s, p, alert_flags); } } } DetectVarProcessList(det_ctx, p->flow, p); RULE_PROFILING_END(det_ctx, s, r == 1, p); } /* update Frame::inspect_progress here instead of in the code above. The reason is that a * frame might be used more than once in buffers with transforms. */ if (frame->inspect_progress < det_ctx->frame_inspect_progress) { frame->inspect_progress = det_ctx->frame_inspect_progress; SCLogDebug("frame->inspect_progress: %" PRIu64 " -> updated", frame->inspect_progress); } else { SCLogDebug( "frame->inspect_progress: %" PRIu64 " -> not updated", frame->inspect_progress); } SCLogDebug("%p/%" PRIi64 " rules inspected, running cleanup", frame, frame->id); InspectionBufferClean(det_ctx); } } static DetectEngineThreadCtx *GetTenantById(HashTable *h, uint32_t id) { /* technically we need to pass a DetectEngineThreadCtx struct with the * tenant_id member. But as that member is the first in the struct, we * can use the id directly. */ return HashTableLookup(h, &id, 0); } static void DetectFlow(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p) { Flow *const f = p->flow; /* we check the flow drop here, and not the packet drop. This is * to allow stream engine "invalid" drop packets to still be * evaluated by the stream event rules. */ if (f->flags & FLOW_ACTION_DROP) { DEBUG_VALIDATE_BUG_ON(!(PKT_IS_PSEUDOPKT(p)) && !PacketCheckAction(p, ACTION_DROP)); SCReturn; } /* in firewall mode, we still need to run the fw rulesets even for exception policy pass */ bool skip = (p->flags & PKT_NOPACKET_INSPECTION || f->flags & (FLOW_ACTION_PASS)); if (EngineModeIsFirewall() && (f->flags & FLOW_ACTION_ACCEPT) == 0) { skip = false; } if (skip) { /* enforce prior accept:flow */ if (f->flags & FLOW_ACTION_ACCEPT) { p->action |= ACTION_ACCEPT; } /* hack: if we are in pass the entire flow mode, we need to still * update the inspect_id forward. So test for the condition here, * and call the update code if necessary. */ const int pass = (f->flags & (FLOW_ACTION_PASS | FLOW_ACTION_ACCEPT)); if (pass) { uint8_t flags = STREAM_FLAGS_FOR_PACKET(p); flags = FlowGetDisruptionFlags(f, flags); if (f->alstate) { AppLayerParserSetTransactionInspectId(f, f->alparser, f->alstate, flags, true); } } SCLogDebug("p->pcap %" PRIu64 ": no detection on packet, " "PKT_NOPACKET_INSPECTION is set", PcapPacketCntGet(p)); return; } /* see if the packet matches one or more of the sigs */ DetectRun(tv, de_ctx, det_ctx, p); } static void DetectNoFlow(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p) { /* No need to perform any detection on this packet, if the given flag is set.*/ if ((p->flags & PKT_NOPACKET_INSPECTION) || (PacketCheckAction(p, ACTION_DROP))) { return; } /* see if the packet matches one or more of the sigs */ DetectRun(tv, de_ctx, det_ctx, p); } uint8_t DetectPreFlow(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Packet *p) { const DetectEngineCtx *de_ctx = det_ctx->de_ctx; const SigGroupHead *sgh = de_ctx->pre_flow_sgh; SCLogDebug("thread id: %u, packet %" PRIu64 ", sgh %p", tv->id, PcapPacketCntGet(p), sgh); DetectRunPacketHook(tv, de_ctx, det_ctx, sgh, p, DETECT_FIREWALL_POLICY_PRE_FLOW); return p->action; } uint8_t DetectPreStream(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, Packet *p) { const DetectEngineCtx *de_ctx = det_ctx->de_ctx; const int direction = (PKT_IS_TOCLIENT(p) != 0); const SigGroupHead *sgh = de_ctx->pre_stream_sgh[direction]; SCLogDebug("thread id: %u, packet %" PRIu64 ", sgh %p", tv->id, PcapPacketCntGet(p), sgh); DetectRunPacketHook(tv, de_ctx, det_ctx, sgh, p, DETECT_FIREWALL_POLICY_PRE_STREAM); return p->action; } /** \brief Detection engine thread wrapper. * \param tv thread vars * \param p packet to inspect * \param data thread specific data * \param pq packet queue * \retval TM_ECODE_FAILED error * \retval TM_ECODE_OK ok */ TmEcode Detect(ThreadVars *tv, Packet *p, void *data) { DEBUG_VALIDATE_PACKET(p); DetectEngineCtx *de_ctx = NULL; DetectEngineThreadCtx *det_ctx = (DetectEngineThreadCtx *)data; if (det_ctx == NULL) { printf("ERROR: Detect has no thread ctx\n"); goto error; } if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) { (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1); SCLogDebug("Detect Engine using new det_ctx - %p", det_ctx); } /* if in MT mode _and_ we have tenants registered, use * MT logic. */ if (det_ctx->mt_det_ctxs_cnt > 0 && det_ctx->TenantGetId != NULL) { uint32_t tenant_id = p->tenant_id; if (tenant_id == 0) tenant_id = det_ctx->TenantGetId(det_ctx, p); if (tenant_id > 0 && tenant_id < det_ctx->mt_det_ctxs_cnt) { p->tenant_id = tenant_id; det_ctx = GetTenantById(det_ctx->mt_det_ctxs_hash, tenant_id); if (det_ctx == NULL) return TM_ECODE_OK; de_ctx = det_ctx->de_ctx; if (de_ctx == NULL) return TM_ECODE_OK; if (unlikely(SC_ATOMIC_GET(det_ctx->so_far_used_by_detect) == 0)) { (void)SC_ATOMIC_SET(det_ctx->so_far_used_by_detect, 1); SCLogDebug("MT de_ctx %p det_ctx %p (tenant %u)", de_ctx, det_ctx, tenant_id); } } else { /* use default if no tenants are registered for this packet */ de_ctx = det_ctx->de_ctx; } } else { de_ctx = det_ctx->de_ctx; } if (p->flow) { DetectFlow(tv, de_ctx, det_ctx, p); } else { DetectNoFlow(tv, de_ctx, det_ctx, p); } #ifdef PROFILE_RULES /* aggregate statistics */ struct timeval ts; gettimeofday(&ts, NULL); if (ts.tv_sec != det_ctx->rule_perf_last_sync) { SCProfilingRuleThreatAggregate(det_ctx); det_ctx->rule_perf_last_sync = ts.tv_sec; } #endif return TM_ECODE_OK; error: return TM_ECODE_FAILED; } /** \brief disable file features we don't need * Called if we have no detection engine. */ void DisableDetectFlowFileFlags(Flow *f) { DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOSERVER); DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOCLIENT); } #if defined(UNITTESTS) || defined(FUZZ) /** * \brief wrapper for old tests */ void SigMatchSignatures( ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, Packet *p) { if (p->flow) { DetectFlow(tv, de_ctx, det_ctx, p); } else { DetectNoFlow(tv, de_ctx, det_ctx, p); } } #endif /* * TESTS */ #ifdef UNITTESTS #include "tests/detect.c" #endif