app-layer-ftp: add ftp-data support

Use expectation to be able to identify connections that are
ftp data. It parses the PASV response, STOR message and the
RETR message to provide extraction of files.

Implementation in Rust of FTP messages parsing is available.

Also this patch changes some var name prefixed by ssh to ftp.

rust/src/ftp/mod.rs

@@ -0,0 +1,110 @@
+/* Copyright (C) 2017 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+extern crate libc;
+extern crate nom;
+
+use nom::{digit};
+use std::str;
+use std;
+use std::str::FromStr;
+
+use log::*;
+
+// We transform an integer string into a i64, ignoring surrounding whitespaces
+// We look for a digit suite, and try to convert it.
+// If either str::from_utf8 or FromStr::from_str fail,
+// we fallback to the parens parser defined above
+named!(getu16<u16>,
+    map_res!(
+      map_res!(
+        ws!(digit),
+        str::from_utf8
+      ),
+      FromStr::from_str
+    )
+);
+
+// 227 Entering Passive Mode (212,27,32,66,221,243).
+named!(pub ftp_pasv_response<u16>,
+       do_parse!(
+            tag!("227") >>
+            take_until_and_consume!("(") >>
+            digit >> tag!(",") >> digit >> tag!(",") >>
+            digit >> tag!(",") >> digit >> tag!(",") >>
+            part1: getu16 >>
+            tag!(",") >>
+            part2: getu16 >>
+            alt! (tag!(").") | tag!(")")) >>
+            (
+                part1 * 256 + part2
+            )
+        )
+);
+
+
+#[no_mangle]
+pub extern "C" fn rs_ftp_pasv_response(input: *const libc::uint8_t, len: libc::uint32_t) -> u16 {
+    let buf = unsafe{std::slice::from_raw_parts(input, len as usize)};
+    match ftp_pasv_response(buf) {
+        nom::IResult::Done(_, dport) => {
+            return dport;
+        }
+        nom::IResult::Incomplete(_) => {
+            let buf = unsafe{std::slice::from_raw_parts(input, len as usize)};
+            SCLogDebug!("pasv incomplete: '{:?}'", String::from_utf8_lossy(buf));
+        },
+        nom::IResult::Error(_) => {
+            let buf = unsafe{std::slice::from_raw_parts(input, len as usize)};
+            SCLogDebug!("pasv error on '{:?}'", String::from_utf8_lossy(buf));
+        },
+    }
+    return 0;
+}
+
+// 229 Entering Extended Passive Mode (|||48758|).
+named!(pub ftp_epsv_response<u16>,
+       do_parse!(
+            tag!("229") >>
+            take_until_and_consume!("|||") >>
+            port: getu16 >>
+            alt! (tag!("|).") | tag!("|)")) >>
+            (
+                port
+            )
+        )
+);
+
+#[no_mangle]
+pub extern "C" fn rs_ftp_epsv_response(input: *const libc::uint8_t, len: libc::uint32_t) -> u16 {
+    let buf = unsafe{std::slice::from_raw_parts(input, len as usize)};
+    match ftp_epsv_response(buf) {
+        nom::IResult::Done(_, dport) => {
+            return dport;
+        },
+        nom::IResult::Incomplete(_) => {
+            let buf = unsafe{std::slice::from_raw_parts(input, len as usize)};
+            SCLogDebug!("epsv incomplete: '{:?}'", String::from_utf8_lossy(buf));
+        },
+        nom::IResult::Error(_) => {
+            let buf = unsafe{std::slice::from_raw_parts(input, len as usize)};
+            SCLogDebug!("epsv incomplete: '{:?}'", String::from_utf8_lossy(buf));
+        },
+
+    }
+    return 0;
+}

rust/src/lib.rs

@@ -43,6 +43,7 @@ pub mod lua;
 
 pub mod dns;
 pub mod nfs;
+pub mod ftp;
 
 #[cfg(feature = "experimental")]
 pub mod ntp;

src/app-layer-detect-proto.c

@@ -692,6 +692,8 @@ static void AppLayerProtoDetectPrintProbingParsers(AppLayerProtoDetectProbingPar
                         printf("            alproto: ALPROTO_HTTP\n");
                     else if (pp_pe->alproto == ALPROTO_FTP)
                         printf("            alproto: ALPROTO_FTP\n");
+                    else if (pp_pe->alproto == ALPROTO_FTPDATA)
+                        printf("            alproto: ALPROTO_FTPDATA\n");
                     else if (pp_pe->alproto == ALPROTO_SMTP)
                         printf("            alproto: ALPROTO_SMTP\n");
                     else if (pp_pe->alproto == ALPROTO_TLS)
@@ -753,6 +755,8 @@ static void AppLayerProtoDetectPrintProbingParsers(AppLayerProtoDetectProbingPar
                     printf("            alproto: ALPROTO_HTTP\n");
                 else if (pp_pe->alproto == ALPROTO_FTP)
                     printf("            alproto: ALPROTO_FTP\n");
+                else if (pp_pe->alproto == ALPROTO_FTPDATA)
+                    printf("            alproto: ALPROTO_FTPDATA\n");
                 else if (pp_pe->alproto == ALPROTO_SMTP)
                     printf("            alproto: ALPROTO_SMTP\n");
                 else if (pp_pe->alproto == ALPROTO_TLS)

src/app-layer-ftp.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2017 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -19,6 +19,7 @@
  * \file
  *
  * \author Pablo Rincon Crespo <pablo.rincon.crespo@gmail.com>
+ * \author Eric Leblond <eric@regit.org>
  *
  * App Layer Parser for FTP
  */
@@ -32,6 +33,7 @@
 #include "util-pool.h"
 
 #include "flow-util.h"
+#include "flow-storage.h"
 
 #include "detect-engine-state.h"
 
@@ -44,11 +46,18 @@
 #include "app-layer-protos.h"
 #include "app-layer-parser.h"
 #include "app-layer-ftp.h"
+#include "app-layer-expectation.h"
 
 #include "util-spm.h"
 #include "util-unittest.h"
 #include "util-debug.h"
 #include "util-memcmp.h"
+#include "util-memrchr.h"
+#include "util-byte.h"
+
+#ifdef HAVE_RUST
+#include "rust-ftp-mod-gen.h"
+#endif
 
 static int FTPGetLineForDirection(FtpState *state, FtpLineState *line_state)
 {
@@ -194,9 +203,46 @@ static int FTPParseRequestCommand(void *ftp_state, uint8_t *input,
         fstate->command = FTP_COMMAND_AUTH_TLS;
     }
 
+    if (input_len >= 4 && SCMemcmpLowercase("pasv", input, 4) == 0) {
+        fstate->command = FTP_COMMAND_PASV;
+    }
+
+    if (input_len > 5 && SCMemcmpLowercase("retr", input, 4) == 0) {
+        fstate->command = FTP_COMMAND_RETR;
+    }
+
+    if (input_len >= 4 && SCMemcmpLowercase("epsv", input, 4) == 0) {
+        fstate->command = FTP_COMMAND_EPSV;
+    }
+
+    if (input_len > 5 && SCMemcmpLowercase("stor", input, 4) == 0) {
+        fstate->command = FTP_COMMAND_STOR;
+    }
+
     return 1;
 }
 
+struct FtpTransferCmd {
+    /** Need to look like a ExpectationData so DFree must
+     *  be first field . */
+    void (*DFree)(void *);
+    uint64_t flow_id;
+    uint8_t *file_name;
+    uint16_t file_len;
+    FtpRequestCommand cmd;
+};
+
+static void FtpTransferCmdFree(void *data)
+{
+    struct FtpTransferCmd *cmd = (struct FtpTransferCmd *) data;
+    if (cmd == NULL)
+        return;
+    if (cmd->file_name) {
+        SCFree(cmd->file_name);
+    }
+    SCFree(cmd);
+}
+
 /**
  * \brief This function is called to retrieve a ftp request
  * \param ftp_state the ftp state structure for the parser
@@ -228,31 +274,131 @@ static int FTPParseRequest(Flow *f, void *ftp_state,
     /* toserver stream */
     state->direction = 0;
 
+    int direction = STREAM_TOSERVER;
     while (FTPGetLine(state) >= 0) {
         FTPParseRequestCommand(state,
                                state->current_line, state->current_line_len);
-        if (state->command == FTP_COMMAND_PORT) {
-            if (state->current_line_len > state->port_line_size) {
-                ptmp = SCRealloc(state->port_line, state->current_line_len);
-                if (ptmp == NULL) {
-                    SCFree(state->port_line);
-                    state->port_line = NULL;
-                    state->port_line_size = 0;
-                    return 0;
+        switch (state->command) {
+            case FTP_COMMAND_PORT:
+                if (state->current_line_len > state->port_line_size) {
+                    ptmp = SCRealloc(state->port_line, state->current_line_len);
+                    if (ptmp == NULL) {
+                        SCFree(state->port_line);
+                        state->port_line = NULL;
+                        state->port_line_size = 0;
+                        return 0;
+                    }
+                    state->port_line = ptmp;
+
+                    state->port_line_size = state->current_line_len;
                 }
-                state->port_line = ptmp;
-
-                state->port_line_size = state->current_line_len;
-            }
-            memcpy(state->port_line, state->current_line,
-                   state->current_line_len);
-            state->port_line_len = state->current_line_len;
+                memcpy(state->port_line, state->current_line,
+                        state->current_line_len);
+                state->port_line_len = state->current_line_len;
+                break;
+            case FTP_COMMAND_RETR:
+                /* change direction (default to server) so expectation will handle
+                 * the correct message when expectation will match.
+                 */
+                direction = STREAM_TOCLIENT;
+                // fallthrough
+            case FTP_COMMAND_STOR:
+                {
+                    /* No dyn port negotiated so get out */
+                    if (state->dyn_port == 0) {
+                        SCReturnInt(-1);
+                    }
+                    struct FtpTransferCmd *data = SCCalloc(1, sizeof(struct FtpTransferCmd));
+                    if (data == NULL)
+                        SCReturnInt(-1);
+                    data->DFree = FtpTransferCmdFree;
+                    /* Min size has been checked in FTPParseRequestCommand */
+                    data->file_name = SCCalloc(state->current_line_len - 4, sizeof(char));
+                    if (data->file_name == NULL) {
+                        SCFree(data);
+                        SCReturnInt(-1);
+                    }
+                    data->file_name[state->current_line_len - 5] = 0;
+                    data->file_len = state->current_line_len - 5;
+                    memcpy(data->file_name, state->current_line + 5, state->current_line_len - 5);
+                    data->cmd = state->command;
+                    data->flow_id = FlowGetId(f);
+                    int ret = AppLayerExpectationCreate(f, direction, 0,
+                                                         state->dyn_port,
+                                                         ALPROTO_FTPDATA, data);
+                    if (ret == -1) {
+                        SCFree(data);
+                        SCLogDebug("No expectation created.");
+                        SCReturnInt(-1);
+                    } else {
+                        SCLogDebug("Expectation created.");
+                    }
+                    /* reset the dyn port to avoid duplicate */
+                    state->dyn_port = 0;
+                }
+                break;
+            default:
+                break;
         }
     }
 
     return 1;
 }
 
+static int FTPParsePassiveResponse(Flow *f, FtpState *state, uint8_t *input, uint32_t input_len)
+{
+    uint16_t dyn_port;
+
+#ifdef HAVE_RUST
+    dyn_port = rs_ftp_pasv_response(input, input_len);
+    if (dyn_port == 0) {
+        return -1;
+    }
+#else
+    uint16_t part1, part2;
+    uint8_t *ptr = memrchr(input, ',', input_len);
+    if (ptr == NULL)
+        return -1;
+
+    part2 = atoi((char *)ptr + 1);
+    ptr = memrchr(input, ',', (ptr - input) - 1);
+    if (ptr == NULL)
+        return -1;
+    part1 = atoi((char *)ptr + 1);
+
+    dyn_port = 256 * part1 + part2;
+#endif
+    state->dyn_port = dyn_port;
+
+    return 0;
+}
+
+static int FTPParsePassiveResponseV6(Flow *f, FtpState *state, uint8_t *input, uint32_t input_len)
+{
+#ifdef HAVE_RUST
+    uint16_t dyn_port = rs_ftp_epsv_response(input, input_len);
+    if (dyn_port == 0) {
+        return -1;
+    }
+
+    state->dyn_port = dyn_port;
+#else
+    uint8_t *ptr = memrchr(input, '|', input_len);
+    if (ptr == NULL) {
+        return -1;
+    } else {
+        int n_length =  ptr - input - 1;
+        if (n_length < 4)
+            return -1;
+        ptr = memrchr(input, '|', n_length);
+        if (ptr == NULL)
+            return -1;
+    }
+    state->dyn_port = atoi((char *)ptr + 1);
+#endif
+    return 0;
+}
+
 /**
  * \brief This function is called to retrieve a ftp response
  * \param ftp_state the ftp state structure for the parser
@@ -274,6 +420,18 @@ static int FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserState *pstat
         }
     }
 
+    if (state->command == FTP_COMMAND_PASV) {
+        if (input_len >= 4 && SCMemcmp("227 ", input, 4) == 0) {
+            FTPParsePassiveResponse(f, ftp_state, input, input_len);
+        }
+    }
+
+    if (state->command == FTP_COMMAND_EPSV) {
+        if (input_len >= 4 && SCMemcmp("229 ", input, 4) == 0) {
+            FTPParsePassiveResponseV6(f, ftp_state, input, input_len);
+        }
+    }
+
     return 1;
 }
 
@@ -327,23 +485,23 @@ static void FTPStateFree(void *s)
 
 static int FTPStateHasTxDetectState(void *state)
 {
-    FtpState *ssh_state = (FtpState *)state;
-    if (ssh_state->de_state)
+    FtpState *ftp_state = (FtpState *)state;
+    if (ftp_state->de_state)
         return 1;
     return 0;
 }
 
 static int FTPSetTxDetectState(void *state, void *vtx, DetectEngineState *de_state)
 {
-    FtpState *ssh_state = (FtpState *)state;
-    ssh_state->de_state = de_state;
+    FtpState *ftp_state = (FtpState *)state;
+    ftp_state->de_state = de_state;
     return 0;
 }
 
 static DetectEngineState *FTPGetTxDetectState(void *vtx)
 {
-    FtpState *ssh_state = (FtpState *)vtx;
-    return ssh_state->de_state;
+    FtpState *ftp_state = (FtpState *)vtx;
+    return ftp_state->de_state;
 }
 
 static void FTPStateTransactionFree(void *state, uint64_t tx_id)
@@ -353,8 +511,8 @@ static void FTPStateTransactionFree(void *state, uint64_t tx_id)
 
 static void *FTPGetTx(void *state, uint64_t tx_id)
 {
-    FtpState *ssh_state = (FtpState *)state;
-    return ssh_state;
+    FtpState *ftp_state = (FtpState *)state;
+    return ftp_state;
 }
 
 static uint64_t FTPGetTxCnt(void *state)
@@ -414,15 +572,217 @@ static int FTPRegisterPatternsForProtocolDetection(void)
     return 0;
 }
 
+
+static StreamingBufferConfig sbcfg = STREAMING_BUFFER_CONFIG_INITIALIZER;
+
+/**
+ * \brief This function is called to retrieve a ftp request
+ * \param ftp_state the ftp state structure for the parser
+ * \param input input line of the command
+ * \param input_len length of the request
+ * \param output the resulting output
+ *
+ * \retval 1 when the command is parsed, 0 otherwise
+ */
+static int FTPDataParse(Flow *f, FtpDataState *ftpdata_state,
+        AppLayerParserState *pstate,
+        uint8_t *input, uint32_t input_len,
+        void *local_data, int direction)
+{
+    uint16_t flags = FileFlowToFlags(f, direction);
+    int ret = 0;
+    /* we depend on detection engine for file pruning */
+    flags |= FILE_USE_DETECT;
+    if (ftpdata_state->files == NULL) {
+        struct FtpTransferCmd *data = (struct FtpTransferCmd *)FlowGetStorageById(f, AppLayerExpectationGetDataId());
+        if (data == NULL) {
+            SCReturnInt(-1);
+        }
+
+        ftpdata_state->files = FileContainerAlloc();
+        if (ftpdata_state->files == NULL) {
+            FlowFreeStorageById(f, AppLayerExpectationGetDataId());
+            SCReturnInt(-1);
+        }
+
+        ftpdata_state->file_name = data->file_name;
+        ftpdata_state->file_len = data->file_len;
+        data->file_name = NULL;
+        data->file_len = 0;
+        f->parent_id = data->flow_id;
+        ftpdata_state->command = data->cmd;
+
+        if (FileOpenFile(ftpdata_state->files, &sbcfg,
+                         (uint8_t *) ftpdata_state->file_name,
+                         ftpdata_state->file_len,
+                         input, input_len, flags) == NULL) {
+            SCLogDebug("Can't open file");
+            ret = -1;
+        }
+        FlowFreeStorageById(f, AppLayerExpectationGetDataId());
+    } else {
+        if (input_len != 0) {
+            ret = FileAppendData(ftpdata_state->files, input, input_len);
+            if (ret == -2) {
+                ret = 0;
+                SCLogDebug("FileAppendData() - file no longer being extracted");
+                goto out;
+            } else if (ret < 0) {
+                SCLogDebug("FileAppendData() failed: %d", ret);
+                ret = -2;
+                goto out;
+            }
+        } else {
+            ret = FileCloseFile(ftpdata_state->files, NULL, 0, flags);
+            ftpdata_state->state = FTPDATA_STATE_FINISHED;
+            if (ret < 0)
+                goto out;
+        }
+    }
+
+    if (input_len && AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF)) {
+        ret = FileCloseFile(ftpdata_state->files, (uint8_t *) NULL, 0, flags);
+        ftpdata_state->state = FTPDATA_STATE_FINISHED;
+    }
+
+out:
+    if (ftpdata_state->files) {
+        FilePrune(ftpdata_state->files);
+    }
+    return ret;
+}
+
+static int FTPDataParseRequest(Flow *f, void *ftp_state,
+        AppLayerParserState *pstate,
+        uint8_t *input, uint32_t input_len,
+        void *local_data)
+{
+    return FTPDataParse(f, ftp_state, pstate, input, input_len,
+                               local_data, STREAM_TOSERVER);
+}
+
+static int FTPDataParseResponse(Flow *f, void *ftp_state,
+        AppLayerParserState *pstate,
+        uint8_t *input, uint32_t input_len,
+        void *local_data)
+{
+    return FTPDataParse(f, ftp_state, pstate, input, input_len,
+                               local_data, STREAM_TOCLIENT);
+}
+
+#ifdef DEBUG
+static SCMutex ftpdata_state_mem_lock = SCMUTEX_INITIALIZER;
+static uint64_t ftpdata_state_memuse = 0;
+static uint64_t ftpdata_state_memcnt = 0;
+#endif
+
+static void *FTPDataStateAlloc(void)
+{
+    void *s = SCMalloc(sizeof(FtpDataState));
+    if (unlikely(s == NULL))
+        return NULL;
+
+    memset(s, 0, sizeof(FtpDataState));
+    ((FtpDataState *)s)->state = FTPDATA_STATE_IN_PROGRESS;
+
+#ifdef DEBUG
+    SCMutexLock(&ftpdata_state_mem_lock);
+    ftpdata_state_memcnt++;
+    ftpdata_state_memuse+=sizeof(FtpDataState);
+    SCMutexUnlock(&ftpdata_state_mem_lock);
+#endif
+    return s;
+}
+
+static void FTPDataStateFree(void *s)
+{
+    FtpDataState *fstate = (FtpDataState *) s;
+
+    if (fstate->de_state != NULL) {
+        DetectEngineStateFree(fstate->de_state);
+    }
+    if (fstate->file_name != NULL) {
+        SCFree(fstate->file_name);
+    }
+
+    FileContainerFree(fstate->files);
+
+    SCFree(s);
+#ifdef DEBUG
+    SCMutexLock(&ftpdata_state_mem_lock);
+    ftpdata_state_memcnt--;
+    ftpdata_state_memuse-=sizeof(FtpDataState);
+    SCMutexUnlock(&ftpdata_state_mem_lock);
+#endif
+}
+
+static int FTPDataStateHasTxDetectState(void *state)
+{
+    FtpDataState *ftp_state = (FtpDataState *)state;
+    if (ftp_state->de_state)
+        return 1;
+    return 0;
+}
+
+static int FTPDataSetTxDetectState(void *state, void *vtx, DetectEngineState *de_state)
+{
+    FtpDataState *ftp_state = (FtpDataState *)state;
+    ftp_state->de_state = de_state;
+    return 0;
+}
+
+static DetectEngineState *FTPDataGetTxDetectState(void *vtx)
+{
+    FtpDataState *ftp_state = (FtpDataState *)vtx;
+    return ftp_state->de_state;
+}
+
+static void FTPDataStateTransactionFree(void *state, uint64_t tx_id)
+{
+    /* do nothing */
+}
+
+static void *FTPDataGetTx(void *state, uint64_t tx_id)
+{
+    FtpDataState *ftp_state = (FtpDataState *)state;
+    return ftp_state;
+}
+
+static uint64_t FTPDataGetTxCnt(void *state)
+{
+    /* ftp-data is single tx */
+    return 1;
+}
+
+static int FTPDataGetAlstateProgressCompletionStatus(uint8_t direction)
+{
+    return FTPDATA_STATE_FINISHED;
+}
+
+static int FTPDataGetAlstateProgress(void *tx, uint8_t direction)
+{
+    FtpDataState *ftpdata_state = (FtpDataState *)tx;
+    return ftpdata_state->state;
+}
+
+static FileContainer *FTPDataStateGetFiles(void *state, uint8_t direction)
+{
+    FtpDataState *ftpdata_state = (FtpDataState *)state;
+
+    SCReturnPtr(ftpdata_state->files, "FileContainer");
+}
+
 void RegisterFTPParsers(void)
 {
     const char *proto_name = "ftp";
+    const char *proto_data_name = "ftp-data";
 
     /** FTP */
     if (AppLayerProtoDetectConfProtoDetectionEnabled("tcp", proto_name)) {
         AppLayerProtoDetectRegisterProtocol(ALPROTO_FTP, proto_name);
         if (FTPRegisterPatternsForProtocolDetection() < 0 )
             return;
+        AppLayerProtoDetectRegisterProtocol(ALPROTO_FTPDATA, proto_data_name);
     }
 
     if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
@@ -446,6 +806,32 @@ void RegisterFTPParsers(void)
 
         AppLayerParserRegisterGetStateProgressCompletionStatus(ALPROTO_FTP,
                                                                FTPGetAlstateProgressCompletionStatus);
+
+
+        AppLayerRegisterExpectationProto(IPPROTO_TCP, ALPROTO_FTPDATA);
+        AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_FTPDATA, STREAM_TOSERVER,
+                                     FTPDataParseRequest);
+        AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_FTPDATA, STREAM_TOCLIENT,
+                                     FTPDataParseResponse);
+        AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateAlloc, FTPDataStateFree);
+        AppLayerParserRegisterParserAcceptableDataDirection(IPPROTO_TCP, ALPROTO_FTPDATA, STREAM_TOSERVER | STREAM_TOCLIENT);
+        AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateTransactionFree);
+        AppLayerParserRegisterDetectStateFuncs(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateHasTxDetectState,
+                FTPDataGetTxDetectState, FTPDataSetTxDetectState);
+
+        AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateGetFiles);
+
+        AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetTx);
+
+        AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetTxCnt);
+
+        AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetAlstateProgress);
+
+        AppLayerParserRegisterGetStateProgressCompletionStatus(ALPROTO_FTPDATA,
+                FTPDataGetAlstateProgressCompletionStatus);
+
+        sbcfg.buf_size = 4096;
+
     } else {
         SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
                   "still on.", proto_name);
@@ -465,6 +851,37 @@ void FTPAtExitPrintStats(void)
 #endif
 }
 
+
+#ifdef HAVE_LIBJANSSON
+json_t *JsonFTPDataAddMetadata(const Flow *f)
+{
+    const FtpDataState *ftp_state = NULL;
+    if (f->alstate == NULL)
+        return NULL;
+    ftp_state = (FtpDataState *)f->alstate;
+    json_t *ftpd = json_object();
+    if (ftpd == NULL)
+        return NULL;
+    if (ftp_state->file_name) {
+        char *s = BytesToString(ftp_state->file_name, ftp_state->file_len);
+        json_object_set_new(ftpd, "filename", json_string(s));
+        if (s != NULL)
+            SCFree(s);
+    }
+    switch (ftp_state->command) {
+        case FTP_COMMAND_STOR:
+            json_object_set_new(ftpd, "command", json_string("STOR"));
+            break;
+        case FTP_COMMAND_RETR:
+            json_object_set_new(ftpd, "command", json_string("RETR"));
+            break;
+        default:
+            break;
+    }
+    return ftpd;
+}
+#endif /* HAVE_LIBJANSSON */
+
 /* UNITTESTS */
 #ifdef UNITTESTS
 

src/app-layer-ftp.h

@@ -41,6 +41,7 @@ typedef enum {
     FTP_COMMAND_CHMOD,
     FTP_COMMAND_CWD,
     FTP_COMMAND_DELE,
+    FTP_COMMAND_EPSV,
     FTP_COMMAND_HELP,
     FTP_COMMAND_IDLE,
     FTP_COMMAND_LIST,
@@ -131,15 +132,38 @@ typedef struct FtpState_ {
     uint32_t port_line_size;
     uint8_t *port_line;
 
+    uint16_t dyn_port;
     /* specifies which loggers are done logging */
     uint32_t logged;
 
     DetectEngineState *de_state;
 } FtpState;
 
+enum {
+    FTPDATA_STATE_IN_PROGRESS,
+    FTPDATA_STATE_FINISHED,
+};
+
+/** FTP Data State for app layer parser */
+typedef struct FtpDataState_ {
+    uint8_t *input;
+    uint8_t *file_name;
+    FileContainer *files;
+    DetectEngineState *de_state;
+    int32_t input_len;
+    int16_t file_len;
+    FtpRequestCommand command;
+    uint8_t state;
+    uint8_t direction;
+} FtpDataState;
+
 void RegisterFTPParsers(void);
 void FTPParserRegisterTests(void);
 void FTPAtExitPrintStats(void);
 
+#ifdef HAVE_LIBJANSSON
+json_t *JsonFTPDataAddMetadata(const Flow *f);
+#endif
+
 #endif /* __APP_LAYER_FTP_H__ */
 

src/app-layer-protos.c

@@ -87,6 +87,9 @@ const char *AppProtoToString(AppProto alproto)
         case ALPROTO_NTP:
             proto_name = "ntp";
             break;
+        case ALPROTO_FTPDATA:
+            proto_name = "ftp-data";
+            break;
         case ALPROTO_TEMPLATE:
             proto_name = "template";
             break;

src/app-layer-protos.h

@@ -46,6 +46,7 @@ enum AppProtoEnum {
     ALPROTO_DNP3,
     ALPROTO_NFS,
     ALPROTO_NTP,
+    ALPROTO_FTPDATA,
     ALPROTO_TEMPLATE,
 
     /* used by the probing parser when alproto detection fails

src/detect-engine-build.c

@@ -462,6 +462,10 @@ PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto,
                     SCLogDebug("packet/flow has ftp state");
                     (*mask) |= SIG_MASK_REQUIRE_FTP_STATE;
                     break;
+                case ALPROTO_FTPDATA:
+                    SCLogDebug("packet/flow has ftpdata state");
+                    (*mask) |= SIG_MASK_REQUIRE_FTPDATA_STATE;
+                    break;
                 case ALPROTO_SMTP:
                     SCLogDebug("packet/flow has smtp state");
                     (*mask) |= SIG_MASK_REQUIRE_SMTP_STATE;
@@ -609,6 +613,10 @@ static int SignatureCreateMask(Signature *s)
         s->mask |= SIG_MASK_REQUIRE_FTP_STATE;
         SCLogDebug("sig requires ftp state");
     }
+    if (s->alproto == ALPROTO_FTPDATA) {
+        s->mask |= SIG_MASK_REQUIRE_FTPDATA_STATE;
+        SCLogDebug("sig requires ftp data state");
+    }
     if (s->alproto == ALPROTO_SMTP) {
         s->mask |= SIG_MASK_REQUIRE_SMTP_STATE;
         SCLogDebug("sig requires smtp state");
@@ -628,6 +636,7 @@ static int SignatureCreateMask(Signature *s)
         (s->mask & SIG_MASK_REQUIRE_DNS_STATE) ||
         (s->mask & SIG_MASK_REQUIRE_DNP3_STATE) ||
         (s->mask & SIG_MASK_REQUIRE_FTP_STATE) ||
+        (s->mask & SIG_MASK_REQUIRE_FTPDATA_STATE) ||
         (s->mask & SIG_MASK_REQUIRE_SMTP_STATE) ||
         (s->mask & SIG_MASK_REQUIRE_ENIP_STATE) ||
         (s->mask & SIG_MASK_REQUIRE_TEMPLATE_STATE) ||

src/detect-filename.c

@@ -91,6 +91,14 @@ void DetectFilenameRegister(void)
             ALPROTO_NFS, SIG_FLAG_TOCLIENT, 0,
             DetectFileInspectGeneric);
 
+    DetectAppLayerInspectEngineRegister("files",
+            ALPROTO_FTPDATA, SIG_FLAG_TOSERVER, 0,
+            DetectFileInspectGeneric);
+
+    DetectAppLayerInspectEngineRegister("files",
+            ALPROTO_FTPDATA, SIG_FLAG_TOCLIENT, 0,
+            DetectFileInspectGeneric);
+
     g_file_match_list_id = DetectBufferTypeGetByName("files");
 
 	SCLogDebug("registering filename rule option");

src/detect.h

@@ -267,10 +267,11 @@ typedef struct DetectPort_ {
 #define SIG_MASK_REQUIRE_TLS_STATE          (1<<9)
 #define SIG_MASK_REQUIRE_DNS_STATE          (1<<10)
 #define SIG_MASK_REQUIRE_FTP_STATE          (1<<11)
-#define SIG_MASK_REQUIRE_SMTP_STATE         (1<<12)
-#define SIG_MASK_REQUIRE_TEMPLATE_STATE     (1<<13)
-#define SIG_MASK_REQUIRE_ENIP_STATE         (1<<14)
-#define SIG_MASK_REQUIRE_DNP3_STATE         (1<<15)
+#define SIG_MASK_REQUIRE_FTPDATA_STATE      (1<<12)
+#define SIG_MASK_REQUIRE_SMTP_STATE         (1<<13)
+#define SIG_MASK_REQUIRE_TEMPLATE_STATE     (1<<14)
+#define SIG_MASK_REQUIRE_ENIP_STATE         (1<<15)
+#define SIG_MASK_REQUIRE_DNP3_STATE         (1<<16)
 
 /* for now a uint8_t is enough */
 #define SignatureMask uint32_t

src/output-json-alert.c

@@ -47,6 +47,7 @@
 #include "app-layer-dnp3.h"
 #include "app-layer-htp.h"
 #include "app-layer-htp-xff.h"
+#include "app-layer-ftp.h"
 #include "util-classification-config.h"
 #include "util-syslog.h"
 #include "util-logopenfile.h"
@@ -426,9 +427,9 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
                 }
             }
         }
-#ifdef HAVE_RUST
         if ((json_output_ctx->flags & LOG_JSON_APP_LAYER) && p->flow != NULL) {
             uint16_t alproto = FlowGetAppProtocol(p->flow);
+#ifdef HAVE_RUST
             if (alproto == ALPROTO_NFS) {
                 hjs = JsonNFSAddMetadataRPC(p->flow, pa->tx_id);
                 if (hjs)
@@ -437,8 +438,13 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
                 if (hjs)
                     json_object_set_new(js, "nfs", hjs);
             }
-        }
 #endif
+            if (alproto == ALPROTO_FTPDATA) {
+                hjs = JsonFTPDataAddMetadata(p->flow);
+                if (hjs)
+                    json_object_set_new(js, "ftp-data", hjs);
+            }
+        }
         if (json_output_ctx->flags & LOG_JSON_DNP3) {
             if (p->flow != NULL) {
                 uint16_t proto = FlowGetAppProtocol(p->flow);