aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

eve/stream: per packet stream engine logging

Browse Source 
Debug facility to get a per packet view into the stream engine's state.

Logs after a packet has been processed in the stream engine, so the view
into the state includes the updates based on the current packet.

Marked as experimental so it can be changed w/o notice.

Bug: #5876.
pull/8562/head
Victor Julien 2 years ago
parent 3948b160c7
commit 83a16a7a89
7 changed files with 477 additions and 1 deletions
Show Stats Download Patch File Download Diff File

7
etc/schema.json
Unescape Escape View File

@ -297,6 +297,11 @@
},
"additionalProperties": false
},
"stream_tcp": {
"type": "object",
"optional": true,
"additionalProperties": true
},
"anomaly": {
"type": "object",
"optional": true,
@ -5309,7 +5314,7 @@
"type": "boolean"
}
},
"additionalProperties": false
"additionalProperties": true
},
"template": {
"type": "object",

2
src/Makefile.am
Unescape Escape View File

@ -385,6 +385,7 @@ noinst_HEADERS = \
log-tcp-data.h \
log-tlslog.h \
log-tlsstore.h \
output-eve-stream.h \
output-filedata.h \
output-file.h \
output-filestore.h \
@ -993,6 +994,7 @@ libsuricata_c_a_SOURCES = \
log-tlslog.c \
log-tlsstore.c \
output.c \
output-eve-stream.c \
output-file.c \
output-filedata.c \
output-filestore.c \

425
src/output-eve-stream.c
Unescape Escape View File

@ -0,0 +1,425 @@
/* Copyright (C) 2023 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*/
#include "suricata-common.h"
#include "packet.h"
#include "detect.h"
#include "flow.h"
#include "conf.h"
#include "threads.h"
#include "tm-threads.h"
#include "threadvars.h"
#include "util-debug.h"
#include "decode-ipv4.h"
#include "detect-parse.h"
#include "detect-engine.h"
#include "detect-engine-mpm.h"
#include "detect-reference.h"
#include "output.h"
#include "output-json.h"
#include "output-json-flow.h"
#include "output-eve-stream.h"
#include "stream-tcp.h"
#include "util-unittest.h"
#include "util-unittest-helper.h"
#include "util-classification-config.h"
#include "util-privs.h"
#include "util-print.h"
#include "util-proto-name.h"
#include "util-logopenfile.h"
#include "util-time.h"
#include "util-buffer.h"
#include "action-globals.h"
#define MODULE_NAME "EveStreamLog"
#define LOG_DROP_ALERTS 1
typedef struct EveStreamOutputCtx_ {
uint16_t trigger_flags; /**< presence of flags in packet trigger logging. 0xffff for all. */
OutputJsonCtx *eve_ctx;
} EveStreamOutputCtx;
typedef struct EveStreamLogThread_ {
EveStreamOutputCtx *stream_ctx;
OutputJsonThreadCtx *ctx;
} EveStreamLogThread;
static TmEcode EveStreamLogThreadInit(ThreadVars *t, const void *initdata, void **data)
{
EveStreamLogThread *aft = SCCalloc(1, sizeof(EveStreamLogThread));
if (unlikely(aft == NULL))
return TM_ECODE_FAILED;
if (initdata == NULL) {
SCLogDebug("Error getting context for EveLogDrop. \"initdata\" argument NULL");
goto error_exit;
}
/** Use the Ouptut Context (file pointer and mutex) */
aft->stream_ctx = ((OutputCtx *)initdata)->data;
aft->ctx = CreateEveThreadCtx(t, aft->stream_ctx->eve_ctx);
if (!aft->ctx) {
goto error_exit;
}
*data = (void *)aft;
return TM_ECODE_OK;
error_exit:
SCFree(aft);
return TM_ECODE_FAILED;
}
static TmEcode EveStreamLogThreadDeinit(ThreadVars *t, void *data)
{
EveStreamLogThread *aft = (EveStreamLogThread *)data;
if (aft == NULL) {
return TM_ECODE_OK;
}
FreeEveThreadCtx(aft->ctx);
/* clear memory */
memset(aft, 0, sizeof(*aft));
SCFree(aft);
return TM_ECODE_OK;
}
static void EveStreamOutputCtxFree(EveStreamOutputCtx *ctx)
{
if (ctx != NULL) {
SCFree(ctx);
}
}
static void EveStreamLogDeInitCtxSub(OutputCtx *output_ctx)
{
OutputDropLoggerDisable();
EveStreamOutputCtx *ctx = output_ctx->data;
SCFree(ctx);
SCLogDebug("cleaning up sub output_ctx %p", output_ctx);
SCFree(output_ctx);
}
static uint16_t SetFlag(ConfNode *conf, const char *opt, const uint16_t inflag)
{
const char *v = ConfNodeLookupChildValue(conf, opt);
if (v != NULL && ConfValIsTrue(v)) {
return inflag;
}
return 0;
}
static OutputInitResult EveStreamLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ctx)
{
OutputInitResult result = { NULL, false };
OutputJsonCtx *ajt = parent_ctx->data;
EveStreamOutputCtx *ctx = SCCalloc(1, sizeof(*ctx));
if (ctx == NULL)
return result;
OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
if (unlikely(output_ctx == NULL)) {
EveStreamOutputCtxFree(ctx);
return result;
}
if (conf) {
// TODO add all flags
ctx->trigger_flags |= SetFlag(conf, "event-set", STREAM_PKT_FLAG_EVENTSET);
ctx->trigger_flags |= SetFlag(conf, "state-update", STREAM_PKT_FLAG_STATE_UPDATE);
ctx->trigger_flags |=
SetFlag(conf, "spurious-retransmission", STREAM_PKT_FLAG_SPURIOUS_RETRANSMISSION);
ctx->trigger_flags |= SetFlag(conf, "all", 0xFFFF);
SCLogDebug("trigger_flags %04x", ctx->trigger_flags);
}
ctx->eve_ctx = ajt;
output_ctx->data = ctx;
output_ctx->DeInit = EveStreamLogDeInitCtxSub;
result.ctx = output_ctx;
result.ok = true;
return result;
}
void EveAddFlowTcpStreamFlags(const TcpStream *stream, const char *name, JsonBuilder *jb)
{
jb_open_array(jb, name);
if (stream->flags & STREAMTCP_STREAM_FLAG_HAS_GAP)
jb_append_string(jb, "has_gap");
if (stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)
jb_append_string(jb, "noreassembly");
if (stream->flags & STREAMTCP_STREAM_FLAG_KEEPALIVE)
jb_append_string(jb, "keepalive");
if (stream->flags & STREAMTCP_STREAM_FLAG_DEPTH_REACHED)
jb_append_string(jb, "depth_reached");
if (stream->flags & STREAMTCP_STREAM_FLAG_TRIGGER_RAW)
jb_append_string(jb, "trigger_raw");
if (stream->flags & STREAMTCP_STREAM_FLAG_TIMESTAMP)
jb_append_string(jb, "timestamp");
if (stream->flags & STREAMTCP_STREAM_FLAG_ZERO_TIMESTAMP)
jb_append_string(jb, "zero_timestamp");
if (stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_COMPLETED)
jb_append_string(jb, "appproto_detection_completed");
if (stream->flags & STREAMTCP_STREAM_FLAG_APPPROTO_DETECTION_SKIPPED)
jb_append_string(jb, "appproto_detection_skipped");
if (stream->flags & STREAMTCP_STREAM_FLAG_NEW_RAW_DISABLED)
jb_append_string(jb, "new_raw_disabled");
if (stream->flags & STREAMTCP_STREAM_FLAG_DISABLE_RAW)
jb_append_string(jb, "disable_raw");
if (stream->flags & STREAMTCP_STREAM_FLAG_RST_RECV)
jb_append_string(jb, "rst_recv");
jb_close(jb);
}
void EveAddFlowTcpFlags(const TcpSession *ssn, const char *name, JsonBuilder *jb)
{
jb_open_object(jb, "flags");
jb_open_array(jb, name);
if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM) {
jb_append_string(jb, "midstream");
}
if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM_ESTABLISHED) {
jb_append_string(jb, "midstream_established");
}
if (ssn->flags & STREAMTCP_FLAG_MIDSTREAM_SYNACK) {
jb_append_string(jb, "midstream_synack");
}
if (ssn->flags & STREAMTCP_FLAG_TIMESTAMP) {
jb_append_string(jb, "timestamp");
}
if (ssn->flags & STREAMTCP_FLAG_SERVER_WSCALE) {
jb_append_string(jb, "server_wscale");
}
if (ssn->flags & STREAMTCP_FLAG_CLOSED_BY_RST) {
jb_append_string(jb, "closed_by_rst");
}
if (ssn->flags & STREAMTCP_FLAG_4WHS) {
jb_append_string(jb, "4whs");
}
if (ssn->flags & STREAMTCP_FLAG_DETECTION_EVASION_ATTEMPT) {
jb_append_string(jb, "detect_evasion_attempt");
}
if (ssn->flags & STREAMTCP_FLAG_CLIENT_SACKOK) {
jb_append_string(jb, "client_sackok");
}
if (ssn->flags & STREAMTCP_FLAG_CLIENT_SACKOK) {
jb_append_string(jb, "sackok");
}
if (ssn->flags & STREAMTCP_FLAG_3WHS_CONFIRMED) {
jb_append_string(jb, "3whs_confirmed");
}
if (ssn->flags & STREAMTCP_FLAG_APP_LAYER_DISABLED) {
jb_append_string(jb, "app_layer_disabled");
}
if (ssn->flags & STREAMTCP_FLAG_BYPASS) {
jb_append_string(jb, "bypass");
}
if (ssn->flags & STREAMTCP_FLAG_TCP_FAST_OPEN) {
jb_append_string(jb, "tcp_fast_open");
}
jb_close(jb);
jb_close(jb);
}
static void LogStream(const TcpStream *stream, JsonBuilder *js)
{
jb_set_uint(js, "isn", stream->isn);
jb_set_uint(js, "next_seq", stream->next_seq);
jb_set_uint(js, "last_ack", stream->last_ack);
jb_set_uint(js, "next_win", stream->next_win);
if (!(stream->flags & STREAMTCP_STREAM_FLAG_NOREASSEMBLY)) {
jb_set_uint(js, "base_seq", stream->base_seq);
jb_set_uint(js, "segs_right_edge", stream->segs_right_edge);
}
jb_set_uint(js, "window", stream->window);
jb_set_uint(js, "wscale", stream->wscale);
EveAddFlowTcpStreamFlags(stream, "flags", js);
}
/**
* \brief Log the stream packets
*
* \param tv Pointer the current thread variables
* \param data Pointer to the EveStreamLogThread struct
* \param p Pointer the packet which is being logged
*
* \retval 0 on succes
*/
static int EveStreamLogger(ThreadVars *tv, void *thread_data, const Packet *p)
{
EveStreamLogThread *td = thread_data;
EveStreamOutputCtx *ctx = td->stream_ctx;
JsonAddrInfo addr = json_addr_info_zero;
JsonAddrInfoInit(p, LOG_DIR_PACKET, &addr);
JsonBuilder *js = CreateEveHeader(p, LOG_DIR_PACKET, "stream_tcp", &addr, ctx->eve_ctx);
if (unlikely(js == NULL))
return TM_ECODE_OK;
if (p->flow != NULL) {
if (p->flowflags & FLOW_PKT_TOSERVER) {
jb_set_string(js, "direction", "to_server");
} else {
jb_set_string(js, "direction", "to_client");
}
}
jb_open_object(js, "stream_tcp");
jb_open_object(js, "packet");
if (PKT_IS_IPV4(p)) {
jb_set_uint(js, "len", IPV4_GET_IPLEN(p));
jb_set_uint(js, "tos", IPV4_GET_IPTOS(p));
jb_set_uint(js, "ttl", IPV4_GET_IPTTL(p));
jb_set_uint(js, "ipid", IPV4_GET_IPID(p));
} else if (PKT_IS_IPV6(p)) {
jb_set_uint(js, "len", IPV6_GET_PLEN(p));
jb_set_uint(js, "tc", IPV6_GET_CLASS(p));
jb_set_uint(js, "hoplimit", IPV6_GET_HLIM(p));
jb_set_uint(js, "flowlbl", IPV6_GET_FLOW(p));
}
if (PKT_IS_TCP(p)) {
jb_set_uint(js, "tcpseq", TCP_GET_SEQ(p));
jb_set_uint(js, "tcpack", TCP_GET_ACK(p));
jb_set_uint(js, "tcpwin", TCP_GET_WINDOW(p));
jb_set_bool(js, "syn", TCP_ISSET_FLAG_SYN(p) ? true : false);
jb_set_bool(js, "ack", TCP_ISSET_FLAG_ACK(p) ? true : false);
jb_set_bool(js, "psh", TCP_ISSET_FLAG_PUSH(p) ? true : false);
jb_set_bool(js, "rst", TCP_ISSET_FLAG_RST(p) ? true : false);
jb_set_bool(js, "urg", TCP_ISSET_FLAG_URG(p) ? true : false);
jb_set_bool(js, "fin", TCP_ISSET_FLAG_FIN(p) ? true : false);
jb_set_uint(js, "tcpres", TCP_GET_RAW_X2(p->tcph));
jb_set_uint(js, "tcpurgp", TCP_GET_URG_POINTER(p));
jb_open_array(js, "flags");
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_RETRANSMISSION)
jb_append_string(js, "retransmission");
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_SPURIOUS_RETRANSMISSION)
jb_append_string(js, "spurious_retransmission");
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_KEEPALIVE)
jb_append_string(js, "keepalive");
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_KEEPALIVEACK)
jb_append_string(js, "keepalive_ack");
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_WINDOWUPDATE)
jb_append_string(js, "window_update");
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_EVENTSET)
jb_append_string(js, "event_set");
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_STATE_UPDATE)
jb_append_string(js, "state_update");
jb_close(js);
}
jb_close(js);
jb_open_object(js, "session");
if (p->flow != NULL && p->flow->protoctx != NULL) {
const TcpSession *ssn = p->flow->protoctx;
const char *tcp_state = StreamTcpStateAsString(ssn->state);
if (tcp_state != NULL)
jb_set_string(js, "state", tcp_state);
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_STATE_UPDATE) {
const char *tcp_pstate = StreamTcpStateAsString(ssn->pstate);
if (tcp_pstate != NULL)
jb_set_string(js, "pstate", tcp_pstate);
}
EveAddFlowTcpFlags(ssn, "flags", js);
jb_open_object(js, "client");
LogStream(&ssn->client, js);
jb_close(js);
jb_open_object(js, "server");
LogStream(&ssn->server, js);
jb_close(js);
}
jb_close(js);
if (p->tcpvars.stream_pkt_flags & STREAM_PKT_FLAG_EVENTSET) {
jb_open_array(js, "events");
for (int i = 0; i < p->events.cnt; i++) {
uint8_t event_code = p->events.events[i];
bool is_decode = EVENT_IS_DECODER_PACKET_ERROR(event_code);
if (is_decode)
continue;
if (event_code >= DECODE_EVENT_MAX)
continue;
const char *event = DEvents[event_code].event_name;
if (event == NULL)
continue;
jb_append_string(js, event);
}
jb_close(js);
}
if (p->drop_reason != 0) {
const char *str = PacketDropReasonToString(p->drop_reason);
jb_set_string(js, "reason", str);
}
/* Close stream. */
jb_close(js);
OutputJsonBuilderBuffer(js, td->ctx);
jb_free(js);
return TM_ECODE_OK;
}
/**
* \brief Check if we need to log this packet
*
* \param tv Pointer the current thread variables
* \param p Pointer the packet which is tested
*
* \retval bool TRUE or FALSE
*/
static int EveStreamLogCondition(ThreadVars *tv, void *data, const Packet *p)
{
EveStreamLogThread *td = data;
EveStreamOutputCtx *ctx = td->stream_ctx;
return (p->proto == IPPROTO_TCP &&
(ctx->trigger_flags == 0xffff ||
(p->tcpvars.stream_pkt_flags & ctx->trigger_flags) != 0));
}
void EveStreamLogRegister(void)
{
OutputRegisterPacketSubModule(LOGGER_JSON_STREAM, "eve-log", MODULE_NAME, "eve-log.stream",
EveStreamLogInitCtxSub, EveStreamLogger, EveStreamLogCondition, EveStreamLogThreadInit,
EveStreamLogThreadDeinit, NULL);
}

31
src/output-eve-stream.h
Unescape Escape View File

@ -0,0 +1,31 @@
/* Copyright (C) 2023 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*/
#ifndef __OUTPUT_EVE_STREAM_H__
#define __OUTPUT_EVE_STREAM_H__
#include "stream-tcp.h"
void EveStreamLogRegister(void);
void EveAddFlowTcpStreamFlags(const TcpStream *stream, const char *name, JsonBuilder *jb);
void EveAddFlowTcpFlags(const TcpSession *ssn, const char *name, JsonBuilder *jb);
#endif /* __OUTPUT_EVE_STREAM_H__ */

2
src/output.c
Unescape Escape View File

@ -50,6 +50,7 @@
#include "output-json-netflow.h"
#include "log-cf-common.h"
#include "output-json-drop.h"
#include "output-eve-stream.h"
#include "log-httplog.h"
#include "output-json-http.h"
#include "output-json-dns.h"
@ -1047,6 +1048,7 @@ void OutputRegisterLoggers(void)
/* syslog log */
AlertSyslogRegister();
JsonDropLogRegister();
EveStreamLogRegister();
/* json log */
OutputJsonRegister();
/* email logs */

1
src/suricata-common.h
Unescape Escape View File

@ -478,6 +478,7 @@ typedef enum {
LOGGER_PCAP,
LOGGER_JSON_METADATA,
LOGGER_JSON_FRAME,
LOGGER_JSON_STREAM,
LOGGER_SIZE,
} LoggerId;

10
suricata.yaml.in
Unescape Escape View File

@ -313,6 +313,16 @@ outputs:
# flowints.
#- metadata
# EXPERIMENTAL per packet output giving TCP state tracking details
# including internal state, flags, etc.
# This output is experimental, meant for debugging and subject to
# change in both config and output without any notice.
#- stream:
# all: false # log all TCP packets
# event-set: false # log packets that have a decoder/stream event
# state-update: false # log packets triggering a TCP state update
# spurious-retransmission: false # log spurious retransmission packets
# a line based log of HTTP requests (no alerts)
- http-log:
enabled: no

Write Preview
Loading…
Cancel
Save