diff --git a/src/alert-unified2-alert.c b/src/alert-unified2-alert.c index 2abcd59528..e1523a3ef2 100644 --- a/src/alert-unified2-alert.c +++ b/src/alert-unified2-alert.c @@ -329,7 +329,7 @@ int Unified2Logger(ThreadVars *t, void *data, const Packet *p) int have_xff_ip = 0; if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) { - have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN); + have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN); } if (have_xff_ip) { @@ -916,9 +916,9 @@ static int Unified2IPv6TypeAlert(ThreadVars *t, const Packet *p, void *data) if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) { if (pa->flags & PACKET_ALERT_FLAG_TX) { - have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN); + have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN); } else { - have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN); + have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN); } } @@ -1104,9 +1104,9 @@ static int Unified2IPv4TypeAlert (ThreadVars *tv, const Packet *p, void *data) if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) { if (pa->flags & PACKET_ALERT_FLAG_TX) { - have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN); + have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN); } else { - have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN); + have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN); } } diff --git a/src/app-layer-htp-xff.c b/src/app-layer-htp-xff.c index aaf7196a2c..fd48b7e93b 100644 --- a/src/app-layer-htp-xff.c +++ b/src/app-layer-htp-xff.c @@ -110,7 +110,7 @@ static int ParseXFFString(char *input, char *output, int output_size) * \retval 1 if the IP has been found and returned in dstbuf * \retval 0 if the IP has not being found or error */ -int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg, +int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen) { uint8_t xff_chain[XFF_CHAIN_MAXLEN]; @@ -119,18 +119,18 @@ int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg, uint64_t total_txs = 0; uint8_t *p_xff = NULL; - htp_state = (HtpState *)FlowGetAppState(p->flow); + htp_state = (HtpState *)FlowGetAppState(f); if (htp_state == NULL) { SCLogDebug("no http state, XFF IP cannot be retrieved"); return 0; } - total_txs = AppLayerParserGetTxCnt(p->flow, htp_state); + total_txs = AppLayerParserGetTxCnt(f, htp_state); if (tx_id >= total_txs) return 0; - tx = AppLayerParserGetTx(p->flow->proto, ALPROTO_HTTP, htp_state, tx_id); + tx = AppLayerParserGetTx(f->proto, ALPROTO_HTTP, htp_state, tx_id); if (tx == NULL) { SCLogDebug("tx is NULL, XFF cannot be retrieved"); return 0; @@ -174,21 +174,21 @@ int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg, * \retval 1 if the IP has been found and returned in dstbuf * \retval 0 if the IP has not being found or error */ -int HttpXFFGetIP(const Packet *p, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen) +int HttpXFFGetIP(const Flow *f, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen) { HtpState *htp_state = NULL; uint64_t tx_id = 0; uint64_t total_txs = 0; - htp_state = (HtpState *)FlowGetAppState(p->flow); + htp_state = (HtpState *)FlowGetAppState(f); if (htp_state == NULL) { SCLogDebug("no http state, XFF IP cannot be retrieved"); goto end; } - total_txs = AppLayerParserGetTxCnt(p->flow, htp_state); + total_txs = AppLayerParserGetTxCnt(f, htp_state); for (; tx_id < total_txs; tx_id++) { - if (HttpXFFGetIPFromTx(p, tx_id, xff_cfg, dstbuf, dstbuflen) == 1) + if (HttpXFFGetIPFromTx(f, tx_id, xff_cfg, dstbuf, dstbuflen) == 1) return 1; } diff --git a/src/app-layer-htp-xff.h b/src/app-layer-htp-xff.h index b02d95de2a..03671d3fbb 100644 --- a/src/app-layer-htp-xff.h +++ b/src/app-layer-htp-xff.h @@ -45,9 +45,9 @@ typedef struct HttpXFFCfg_ { void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result); -int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen); +int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen); -int HttpXFFGetIP(const Packet *p, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen); +int HttpXFFGetIP(const Flow *f, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen); void HTPXFFParserRegisterTests(void); diff --git a/src/output-filestore.c b/src/output-filestore.c index 8adb01aefe..6f4d8f3f68 100644 --- a/src/output-filestore.c +++ b/src/output-filestore.c @@ -19,6 +19,7 @@ #include "app-layer-parser.h" #include "app-layer-htp.h" +#include "app-layer-htp-xff.h" #include "app-layer-smtp.h" #include "output.h" @@ -49,6 +50,7 @@ typedef struct OutputFilestoreCtx_ { char prefix[FILESTORE_PREFIX_MAX]; char tmpdir[FILESTORE_PREFIX_MAX]; bool fileinfo; + HttpXFFCfg *xff_cfg; } OutputFilestoreCtx; typedef struct OutputFilestoreLogThread_ { @@ -162,7 +164,8 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv, snprintf(js_metadata_filename, sizeof(js_metadata_filename), "%s.%"PRIuMAX".%u.json", final_filename, (uintmax_t)p->ts.tv_sec, ff->file_store_id); - json_t *js_fileinfo = JsonBuildFileInfoRecord(p, ff, true, dir); + json_t *js_fileinfo = JsonBuildFileInfoRecord(p, ff, true, dir, + ctx->xff_cfg); if (likely(js_fileinfo != NULL)) { json_dump_file(js_fileinfo, js_metadata_filename, 0); json_decref(js_fileinfo); @@ -307,6 +310,9 @@ static TmEcode OutputFilestoreLogThreadDeinit(ThreadVars *t, void *data) static void OutputFilestoreLogDeInitCtx(OutputCtx *output_ctx) { OutputFilestoreCtx *ctx = (OutputFilestoreCtx *)output_ctx->data; + if (ctx->xff_cfg != NULL) { + SCFree(ctx->xff_cfg); + } SCFree(ctx); SCFree(output_ctx); } @@ -404,6 +410,11 @@ static OutputInitResult OutputFilestoreLogInitCtx(ConfNode *conf) strlcpy(ctx->prefix, log_directory, sizeof(ctx->prefix)); snprintf(ctx->tmpdir, sizeof(ctx->tmpdir) - 1, "%s/tmp", log_directory); + ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg)); + if (ctx->xff_cfg != NULL) { + HttpXFFGetCfg(conf, ctx->xff_cfg); + } + OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); if (unlikely(output_ctx == NULL)) { SCFree(ctx); diff --git a/src/output-json-alert.c b/src/output-json-alert.c index f8b36097c5..45a19a3ba1 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -598,9 +598,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) { if (pa->flags & PACKET_ALERT_FLAG_TX) { - have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN); + have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN); } else { - have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN); + have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN); } } diff --git a/src/output-json-file.c b/src/output-json-file.c index f652a432af..c7a62b3417 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -65,6 +65,7 @@ #include "output-json-smb.h" #include "app-layer-htp.h" +#include "app-layer-htp-xff.h" #include "util-memcmp.h" #include "stream-tcp-reassemble.h" @@ -73,6 +74,7 @@ typedef struct OutputFileCtx_ { LogFileCtx *file_ctx; uint32_t file_cnt; + HttpXFFCfg *xff_cfg; } OutputFileCtx; typedef struct JsonFileLogThread_ { @@ -81,7 +83,7 @@ typedef struct JsonFileLogThread_ { } JsonFileLogThread; json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff, - const bool stored, uint8_t dir) + const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg) { json_t *hjs = NULL; enum OutputJsonLogDirection fdir = LOG_DIR_FLOW; @@ -205,6 +207,29 @@ json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff, json_object_set_new(fjs, "size", json_integer(FileTrackedSize(ff))); json_object_set_new(fjs, "tx_id", json_integer(ff->txid)); + /* xff header */ + if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) { + int have_xff_ip = 0; + char buffer[XFF_MAXLEN]; + + if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) { + have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, buffer, XFF_MAXLEN); + } + + if (have_xff_ip) { + if (xff_cfg->flags & XFF_EXTRADATA) { + json_object_set_new(js, "xff", json_string(buffer)); + } + else if (xff_cfg->flags & XFF_OVERWRITE) { + if (p->flowflags & FLOW_PKT_TOCLIENT) { + json_object_set(js, "dest_ip", json_string(buffer)); + } else { + json_object_set(js, "src_ip", json_string(buffer)); + } + } + } + } + /* originally just 'file', but due to bug 1127 naming it fileinfo */ json_object_set_new(js, "fileinfo", fjs); @@ -218,8 +243,9 @@ json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff, static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, uint32_t dir) { + HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg; json_t *js = JsonBuildFileInfoRecord(p, ff, - ff->flags & FILE_STORED ? true : false, dir); + ff->flags & FILE_STORED ? true : false, dir, xff_cfg); if (unlikely(js == NULL)) { return; } @@ -290,6 +316,9 @@ static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data) static void OutputFileLogDeinitSub(OutputCtx *output_ctx) { OutputFileCtx *ff_ctx = output_ctx->data; + if (ff_ctx->xff_cfg != NULL) { + SCFree(ff_ctx->xff_cfg); + } SCFree(ff_ctx); SCFree(output_ctx); } @@ -330,6 +359,10 @@ static OutputInitResult OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_c FileForceHashParseCfg(conf); } + output_file_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg)); + if (output_file_ctx->xff_cfg != NULL) { + HttpXFFGetCfg(conf, output_file_ctx->xff_cfg); + } output_ctx->data = output_file_ctx; output_ctx->DeInit = OutputFileLogDeinitSub; diff --git a/src/output-json-file.h b/src/output-json-file.h index 9b7657d216..3d1b723902 100644 --- a/src/output-json-file.h +++ b/src/output-json-file.h @@ -27,8 +27,10 @@ void JsonFileLogRegister(void); #ifdef HAVE_LIBJANSSON +#include "app-layer-htp-xff.h" + json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff, - const bool stored, uint8_t dir); + const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg); #endif #endif /* __OUTPUT_JSON_FILE_H__ */ diff --git a/src/output-json-http.c b/src/output-json-http.c index 7110e6b44e..47add0a0fe 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -40,6 +40,7 @@ #include "output.h" #include "app-layer-htp.h" +#include "app-layer-htp-xff.h" #include "app-layer.h" #include "app-layer-parser.h" #include "util-privs.h" @@ -59,6 +60,7 @@ typedef struct LogHttpFileCtx_ { uint32_t flags; /** Store mode */ uint64_t fields;/** Store fields */ bool include_metadata; + HttpXFFCfg *xff_cfg; } LogHttpFileCtx; typedef struct JsonHttpLogThread_ { @@ -466,6 +468,28 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl MemBufferReset(jhl->buffer); JsonHttpLogJSON(jhl, js, tx, tx_id); + HttpXFFCfg *xff_cfg = jhl->httplog_ctx->xff_cfg; + + /* xff header */ + if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) { + int have_xff_ip = 0; + char buffer[XFF_MAXLEN]; + + have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, buffer, XFF_MAXLEN); + + if (have_xff_ip) { + if (xff_cfg->flags & XFF_EXTRADATA) { + json_object_set_new(js, "xff", json_string(buffer)); + } + else if (xff_cfg->flags & XFF_OVERWRITE) { + if (p->flowflags & FLOW_PKT_TOCLIENT) { + json_object_set(js, "dest_ip", json_string(buffer)); + } else { + json_object_set(js, "src_ip", json_string(buffer)); + } + } + } + } OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, &jhl->buffer); json_object_del(js, "http"); @@ -501,6 +525,9 @@ static void OutputHttpLogDeinit(OutputCtx *output_ctx) LogHttpFileCtx *http_ctx = output_ctx->data; LogFileCtx *logfile_ctx = http_ctx->file_ctx; LogFileFreeCtx(logfile_ctx); + if (http_ctx->xff_cfg) { + SCFree(http_ctx->xff_cfg); + } SCFree(http_ctx); SCFree(output_ctx); } @@ -545,6 +572,11 @@ static OutputInitResult OutputHttpLogInit(ConfNode *conf) } } } + http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg)); + if (http_ctx->xff_cfg != NULL) { + HttpXFFGetCfg(conf, http_ctx->xff_cfg); + } + output_ctx->data = http_ctx; output_ctx->DeInit = OutputHttpLogDeinit; @@ -559,6 +591,9 @@ static OutputInitResult OutputHttpLogInit(ConfNode *conf) static void OutputHttpLogDeinitSub(OutputCtx *output_ctx) { LogHttpFileCtx *http_ctx = output_ctx->data; + if (http_ctx->xff_cfg) { + SCFree(http_ctx->xff_cfg); + } SCFree(http_ctx); SCFree(output_ctx); } @@ -615,6 +650,11 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c } } } + http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg)); + if (http_ctx->xff_cfg != NULL) { + HttpXFFGetCfg(conf, http_ctx->xff_cfg); + } + output_ctx->data = http_ctx; output_ctx->DeInit = OutputHttpLogDeinitSub; diff --git a/suricata.yaml.in b/suricata.yaml.in index 634d153390..d09c8aeac3 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -208,6 +208,23 @@ outputs: # custom allows additional http fields to be included in eve-log # the example below adds three additional fields when uncommented #custom: [Accept-Encoding, Accept-Language, Authorization] + # HTTP X-Forwarded-For support by adding an extra field or overwriting + # the source or destination IP address (depending on flow direction) + # with the one reported in the X-Forwarded-For HTTP header. This is + # helpful when reviewing alerts for traffic that is being reverse + # or forward proxied. + xff: + enabled: no + # Two operation modes are available, "extra-data" and "overwrite". + mode: extra-data + # Two proxy deployments are supported, "reverse" and "forward". In + # a "reverse" deployment the IP address used is the last one, in a + # "forward" deployment the first IP address is used. + deployment: reverse + # Header name where the actual IP address will be reported, if more + # than one IP address is present, the last IP address will be the + # one taken into consideration. + header: X-Forwarded-For - dns: # This configuration uses the new DNS logging format, # the old configuration is still available: @@ -251,6 +268,23 @@ outputs: # force logging of checksums, available hash functions are md5, # sha1 and sha256 #force-hash: [md5] + # HTTP X-Forwarded-For support by adding an extra field or overwriting + # the source or destination IP address (depending on flow direction) + # with the one reported in the X-Forwarded-For HTTP header. This is + # helpful when reviewing alerts for traffic that is being reverse + # or forward proxied. + xff: + enabled: no + # Two operation modes are available, "extra-data" and "overwrite". + mode: extra-data + # Two proxy deployments are supported, "reverse" and "forward". In + # a "reverse" deployment the IP address used is the last one, in a + # "forward" deployment the first IP address is used. + deployment: reverse + # Header name where the actual IP address will be reported, if more + # than one IP address is present, the last IP address will be the + # one taken into consideration. + header: X-Forwarded-For #- drop: # alerts: yes # log alerts that caused drops # flows: all # start or all: 'start' logs only a single drop @@ -515,6 +549,24 @@ outputs: # the use of this output module as it uses the SHA256 as the # file naming scheme. #force-hash: [sha1, md5] + # NOTE: X-Forwarded configuration is ignored if write-fileinfo is disabled + # HTTP X-Forwarded-For support by adding an extra field or overwriting + # the source or destination IP address (depending on flow direction) + # with the one reported in the X-Forwarded-For HTTP header. This is + # helpful when reviewing alerts for traffic that is being reverse + # or forward proxied. + xff: + enabled: no + # Two operation modes are available, "extra-data" and "overwrite". + mode: extra-data + # Two proxy deployments are supported, "reverse" and "forward". In + # a "reverse" deployment the IP address used is the last one, in a + # "forward" deployment the first IP address is used. + deployment: reverse + # Header name where the actual IP address will be reported, if more + # than one IP address is present, the last IP address will be the + # one taken into consideration. + header: X-Forwarded-For # output module to store extracted files to disk (old style, deprecated) #