xff: Use XFF configuration in eve and filestore

XFF configuration is already set in app-layer-htp-xff, and in
output-json-alert. Extending XFF configuration to files and HTTP allow
to get the same behavior as for alerts.

Extend the configuration of filestore json to let filestore metafile
dump be aware of xff. This is available only if write-fileinfo is set
to yes and file-store version is 2.

src/alert-unified2-alert.c

@@ -329,7 +329,7 @@ int Unified2Logger(ThreadVars *t, void *data, const Packet *p)
         int have_xff_ip = 0;
 
         if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
-            have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN);
+            have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
         }
 
         if (have_xff_ip) {
@@ -916,9 +916,9 @@ static int Unified2IPv6TypeAlert(ThreadVars *t, const Packet *p, void *data)
 
             if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
                 if (pa->flags & PACKET_ALERT_FLAG_TX) {
-                    have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
                 } else {
-                    have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
                 }
             }
 
@@ -1104,9 +1104,9 @@ static int Unified2IPv4TypeAlert (ThreadVars *tv, const Packet *p, void *data)
 
             if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
                 if (pa->flags & PACKET_ALERT_FLAG_TX) {
-                    have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
                 } else {
-                    have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
                 }
             }
 

src/app-layer-htp-xff.c

@@ -110,7 +110,7 @@ static int ParseXFFString(char *input, char *output, int output_size)
  * \retval 1 if the IP has been found and returned in dstbuf
  * \retval 0 if the IP has not being found or error
  */
-int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg,
+int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg,
         char *dstbuf, int dstbuflen)
 {
     uint8_t xff_chain[XFF_CHAIN_MAXLEN];
@@ -119,18 +119,18 @@ int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg,
     uint64_t total_txs = 0;
     uint8_t *p_xff = NULL;
 
-    htp_state = (HtpState *)FlowGetAppState(p->flow);
+    htp_state = (HtpState *)FlowGetAppState(f);
 
     if (htp_state == NULL) {
         SCLogDebug("no http state, XFF IP cannot be retrieved");
         return 0;
     }
 
-    total_txs = AppLayerParserGetTxCnt(p->flow, htp_state);
+    total_txs = AppLayerParserGetTxCnt(f, htp_state);
     if (tx_id >= total_txs)
         return 0;
 
-    tx = AppLayerParserGetTx(p->flow->proto, ALPROTO_HTTP, htp_state, tx_id);
+    tx = AppLayerParserGetTx(f->proto, ALPROTO_HTTP, htp_state, tx_id);
     if (tx == NULL) {
         SCLogDebug("tx is NULL, XFF cannot be retrieved");
         return 0;
@@ -174,21 +174,21 @@ int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg,
  *  \retval 1 if the IP has been found and returned in dstbuf
  *  \retval 0 if the IP has not being found or error
  */
-int HttpXFFGetIP(const Packet *p, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
+int HttpXFFGetIP(const Flow *f, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen)
 {
     HtpState *htp_state = NULL;
     uint64_t tx_id = 0;
     uint64_t total_txs = 0;
 
-    htp_state = (HtpState *)FlowGetAppState(p->flow);
+    htp_state = (HtpState *)FlowGetAppState(f);
     if (htp_state == NULL) {
         SCLogDebug("no http state, XFF IP cannot be retrieved");
         goto end;
     }
 
-    total_txs = AppLayerParserGetTxCnt(p->flow, htp_state);
+    total_txs = AppLayerParserGetTxCnt(f, htp_state);
     for (; tx_id < total_txs; tx_id++) {
-        if (HttpXFFGetIPFromTx(p, tx_id, xff_cfg, dstbuf, dstbuflen) == 1)
+        if (HttpXFFGetIPFromTx(f, tx_id, xff_cfg, dstbuf, dstbuflen) == 1)
             return 1;
     }
 

src/app-layer-htp-xff.h

@@ -45,9 +45,9 @@ typedef struct HttpXFFCfg_ {
 
 void HttpXFFGetCfg(ConfNode *conf, HttpXFFCfg *result);
 
-int HttpXFFGetIPFromTx(const Packet *p, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen);
+int HttpXFFGetIPFromTx(const Flow *f, uint64_t tx_id, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen);
 
-int HttpXFFGetIP(const Packet *p, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen);
+int HttpXFFGetIP(const Flow *f, HttpXFFCfg *xff_cfg, char *dstbuf, int dstbuflen);
 
 void HTPXFFParserRegisterTests(void);
 

src/output-filestore.c

@@ -19,6 +19,7 @@
 
 #include "app-layer-parser.h"
 #include "app-layer-htp.h"
+#include "app-layer-htp-xff.h"
 #include "app-layer-smtp.h"
 
 #include "output.h"
@@ -49,6 +50,7 @@ typedef struct OutputFilestoreCtx_ {
     char prefix[FILESTORE_PREFIX_MAX];
     char tmpdir[FILESTORE_PREFIX_MAX];
     bool fileinfo;
+    HttpXFFCfg *xff_cfg;
 } OutputFilestoreCtx;
 
 typedef struct OutputFilestoreLogThread_ {
@@ -162,7 +164,8 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv,
         snprintf(js_metadata_filename, sizeof(js_metadata_filename),
                 "%s.%"PRIuMAX".%u.json", final_filename,
                 (uintmax_t)p->ts.tv_sec, ff->file_store_id);
-        json_t *js_fileinfo = JsonBuildFileInfoRecord(p, ff, true, dir);
+        json_t *js_fileinfo = JsonBuildFileInfoRecord(p, ff, true, dir,
+                ctx->xff_cfg);
         if (likely(js_fileinfo != NULL)) {
             json_dump_file(js_fileinfo, js_metadata_filename, 0);
             json_decref(js_fileinfo);
@@ -307,6 +310,9 @@ static TmEcode OutputFilestoreLogThreadDeinit(ThreadVars *t, void *data)
 static void OutputFilestoreLogDeInitCtx(OutputCtx *output_ctx)
 {
     OutputFilestoreCtx *ctx = (OutputFilestoreCtx *)output_ctx->data;
+    if (ctx->xff_cfg != NULL) {
+        SCFree(ctx->xff_cfg);
+    }
     SCFree(ctx);
     SCFree(output_ctx);
 }
@@ -404,6 +410,11 @@ static OutputInitResult OutputFilestoreLogInitCtx(ConfNode *conf)
     strlcpy(ctx->prefix, log_directory, sizeof(ctx->prefix));
     snprintf(ctx->tmpdir, sizeof(ctx->tmpdir) - 1, "%s/tmp", log_directory);
 
+    ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, ctx->xff_cfg);
+    }
+
     OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx));
     if (unlikely(output_ctx == NULL)) {
         SCFree(ctx);

src/output-json-alert.c

@@ -598,9 +598,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
 
             if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
                 if (pa->flags & PACKET_ALERT_FLAG_TX) {
-                    have_xff_ip = HttpXFFGetIPFromTx(p, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIPFromTx(p->flow, pa->tx_id, xff_cfg, buffer, XFF_MAXLEN);
                 } else {
-                    have_xff_ip = HttpXFFGetIP(p, xff_cfg, buffer, XFF_MAXLEN);
+                    have_xff_ip = HttpXFFGetIP(p->flow, xff_cfg, buffer, XFF_MAXLEN);
                 }
             }
 

src/output-json-file.c

@@ -65,6 +65,7 @@
 #include "output-json-smb.h"
 
 #include "app-layer-htp.h"
+#include "app-layer-htp-xff.h"
 #include "util-memcmp.h"
 #include "stream-tcp-reassemble.h"
 
@@ -73,6 +74,7 @@
 typedef struct OutputFileCtx_ {
     LogFileCtx *file_ctx;
     uint32_t file_cnt;
+    HttpXFFCfg *xff_cfg;
 } OutputFileCtx;
 
 typedef struct JsonFileLogThread_ {
@@ -81,7 +83,7 @@ typedef struct JsonFileLogThread_ {
 } JsonFileLogThread;
 
 json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
-        const bool stored, uint8_t dir)
+        const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg)
 {
     json_t *hjs = NULL;
     enum OutputJsonLogDirection fdir = LOG_DIR_FLOW;
@@ -205,6 +207,29 @@ json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
     json_object_set_new(fjs, "size", json_integer(FileTrackedSize(ff)));
     json_object_set_new(fjs, "tx_id", json_integer(ff->txid));
 
+    /* xff header */
+    if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) {
+        int have_xff_ip = 0;
+        char buffer[XFF_MAXLEN];
+
+        if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP) {
+            have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, buffer, XFF_MAXLEN);
+        }
+
+        if (have_xff_ip) {
+            if (xff_cfg->flags & XFF_EXTRADATA) {
+                json_object_set_new(js, "xff", json_string(buffer));
+            }
+            else if (xff_cfg->flags & XFF_OVERWRITE) {
+                if (p->flowflags & FLOW_PKT_TOCLIENT) {
+                    json_object_set(js, "dest_ip", json_string(buffer));
+                } else {
+                    json_object_set(js, "src_ip", json_string(buffer));
+                }
+            }
+        }
+    }
+
     /* originally just 'file', but due to bug 1127 naming it fileinfo */
     json_object_set_new(js, "fileinfo", fjs);
 
@@ -218,8 +243,9 @@ json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
 static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p,
                                 const File *ff, uint32_t dir)
 {
+    HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg;
     json_t *js = JsonBuildFileInfoRecord(p, ff,
-            ff->flags & FILE_STORED ? true : false, dir);
+            ff->flags & FILE_STORED ? true : false, dir, xff_cfg);
     if (unlikely(js == NULL)) {
         return;
     }
@@ -290,6 +316,9 @@ static TmEcode JsonFileLogThreadDeinit(ThreadVars *t, void *data)
 static void OutputFileLogDeinitSub(OutputCtx *output_ctx)
 {
     OutputFileCtx *ff_ctx = output_ctx->data;
+    if (ff_ctx->xff_cfg != NULL) {
+        SCFree(ff_ctx->xff_cfg);
+    }
     SCFree(ff_ctx);
     SCFree(output_ctx);
 }
@@ -330,6 +359,10 @@ static OutputInitResult OutputFileLogInitSub(ConfNode *conf, OutputCtx *parent_c
 
         FileForceHashParseCfg(conf);
     }
+    output_file_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (output_file_ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, output_file_ctx->xff_cfg);
+    }
 
     output_ctx->data = output_file_ctx;
     output_ctx->DeInit = OutputFileLogDeinitSub;

src/output-json-file.h

@@ -27,8 +27,10 @@
 void JsonFileLogRegister(void);
 
 #ifdef HAVE_LIBJANSSON
+#include "app-layer-htp-xff.h"
+
 json_t *JsonBuildFileInfoRecord(const Packet *p, const File *ff,
-        const bool stored, uint8_t dir);
+        const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg);
 #endif
 
 #endif /* __OUTPUT_JSON_FILE_H__ */

src/output-json-http.c

@@ -40,6 +40,7 @@
 
 #include "output.h"
 #include "app-layer-htp.h"
+#include "app-layer-htp-xff.h"
 #include "app-layer.h"
 #include "app-layer-parser.h"
 #include "util-privs.h"
@@ -59,6 +60,7 @@ typedef struct LogHttpFileCtx_ {
     uint32_t flags; /** Store mode */
     uint64_t fields;/** Store fields */
     bool include_metadata;
+    HttpXFFCfg *xff_cfg;
 } LogHttpFileCtx;
 
 typedef struct JsonHttpLogThread_ {
@@ -466,6 +468,28 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl
     MemBufferReset(jhl->buffer);
 
     JsonHttpLogJSON(jhl, js, tx, tx_id);
+    HttpXFFCfg *xff_cfg = jhl->httplog_ctx->xff_cfg;
+
+    /* xff header */
+    if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) {
+        int have_xff_ip = 0;
+        char buffer[XFF_MAXLEN];
+
+        have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, buffer, XFF_MAXLEN);
+
+        if (have_xff_ip) {
+            if (xff_cfg->flags & XFF_EXTRADATA) {
+                json_object_set_new(js, "xff", json_string(buffer));
+            }
+            else if (xff_cfg->flags & XFF_OVERWRITE) {
+                if (p->flowflags & FLOW_PKT_TOCLIENT) {
+                    json_object_set(js, "dest_ip", json_string(buffer));
+                } else {
+                    json_object_set(js, "src_ip", json_string(buffer));
+                }
+            }
+        }
+    }
 
     OutputJSONBuffer(js, jhl->httplog_ctx->file_ctx, &jhl->buffer);
     json_object_del(js, "http");
@@ -501,6 +525,9 @@ static void OutputHttpLogDeinit(OutputCtx *output_ctx)
     LogHttpFileCtx *http_ctx = output_ctx->data;
     LogFileCtx *logfile_ctx = http_ctx->file_ctx;
     LogFileFreeCtx(logfile_ctx);
+    if (http_ctx->xff_cfg) {
+        SCFree(http_ctx->xff_cfg);
+    }
     SCFree(http_ctx);
     SCFree(output_ctx);
 }
@@ -545,6 +572,11 @@ static OutputInitResult OutputHttpLogInit(ConfNode *conf)
             }
         }
     }
+    http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (http_ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, http_ctx->xff_cfg);
+    }
+
     output_ctx->data = http_ctx;
     output_ctx->DeInit = OutputHttpLogDeinit;
 
@@ -559,6 +591,9 @@ static OutputInitResult OutputHttpLogInit(ConfNode *conf)
 static void OutputHttpLogDeinitSub(OutputCtx *output_ctx)
 {
     LogHttpFileCtx *http_ctx = output_ctx->data;
+    if (http_ctx->xff_cfg) {
+        SCFree(http_ctx->xff_cfg);
+    }
     SCFree(http_ctx);
     SCFree(output_ctx);
 }
@@ -615,6 +650,11 @@ static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_c
             }
         }
     }
+    http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg));
+    if (http_ctx->xff_cfg != NULL) {
+        HttpXFFGetCfg(conf, http_ctx->xff_cfg);
+    }
+
     output_ctx->data = http_ctx;
     output_ctx->DeInit = OutputHttpLogDeinitSub;
 

suricata.yaml.in

@@ -208,6 +208,23 @@ outputs:
             # custom allows additional http fields to be included in eve-log
             # the example below adds three additional fields when uncommented
             #custom: [Accept-Encoding, Accept-Language, Authorization]
+            # HTTP X-Forwarded-For support by adding an extra field or overwriting
+            # the source or destination IP address (depending on flow direction)
+            # with the one reported in the X-Forwarded-For HTTP header. This is
+            # helpful when reviewing alerts for traffic that is being reverse
+            # or forward proxied.
+            xff:
+              enabled: no
+              # Two operation modes are available, "extra-data" and "overwrite".
+              mode: extra-data
+              # Two proxy deployments are supported, "reverse" and "forward". In
+              # a "reverse" deployment the IP address used is the last one, in a
+              # "forward" deployment the first IP address is used.
+              deployment: reverse
+              # Header name where the actual IP address will be reported, if more
+              # than one IP address is present, the last IP address will be the
+              # one taken into consideration.
+              header: X-Forwarded-For
         - dns:
             # This configuration uses the new DNS logging format,
             # the old configuration is still available:
@@ -251,6 +268,23 @@ outputs:
             # force logging of checksums, available hash functions are md5,
             # sha1 and sha256
             #force-hash: [md5]
+            # HTTP X-Forwarded-For support by adding an extra field or overwriting
+            # the source or destination IP address (depending on flow direction)
+            # with the one reported in the X-Forwarded-For HTTP header. This is
+            # helpful when reviewing alerts for traffic that is being reverse
+            # or forward proxied.
+            xff:
+              enabled: no
+              # Two operation modes are available, "extra-data" and "overwrite".
+              mode: extra-data
+              # Two proxy deployments are supported, "reverse" and "forward". In
+              # a "reverse" deployment the IP address used is the last one, in a
+              # "forward" deployment the first IP address is used.
+              deployment: reverse
+              # Header name where the actual IP address will be reported, if more
+              # than one IP address is present, the last IP address will be the
+              # one taken into consideration.
+              header: X-Forwarded-For
         #- drop:
         #    alerts: yes      # log alerts that caused drops
         #    flows: all       # start or all: 'start' logs only a single drop
@@ -515,6 +549,24 @@ outputs:
       # the use of this output module as it uses the SHA256 as the
       # file naming scheme.
       #force-hash: [sha1, md5]
+      # NOTE: X-Forwarded configuration is ignored if write-fileinfo is disabled
+      # HTTP X-Forwarded-For support by adding an extra field or overwriting
+      # the source or destination IP address (depending on flow direction)
+      # with the one reported in the X-Forwarded-For HTTP header. This is
+      # helpful when reviewing alerts for traffic that is being reverse
+      # or forward proxied.
+      xff:
+        enabled: no
+        # Two operation modes are available, "extra-data" and "overwrite".
+        mode: extra-data
+        # Two proxy deployments are supported, "reverse" and "forward". In
+        # a "reverse" deployment the IP address used is the last one, in a
+        # "forward" deployment the first IP address is used.
+        deployment: reverse
+        # Header name where the actual IP address will be reported, if more
+        # than one IP address is present, the last IP address will be the
+        # one taken into consideration.
+        header: X-Forwarded-For
 
   # output module to store extracted files to disk (old style, deprecated)
   #