proto-detect: improve midstream support

When Suricata picks up a flow it assumes the first packet is
toserver. In a perfect world without packet loss and where all
sessions neatly start after Suricata itself started, this would be
true. However, in reality we have to account for packet loss and
Suricata starting to get packets for flows already active be for
Suricata is (re)started.

The protocol records on the wire would often be able to tell us more
though. For example in SMB1 and SMB2 records there is a flag that
indicates whether the record is a request or a response. This patch
is enabling the procotol detection engine to utilize this information
to 'reverse' the flow.

There are three ways in which this is supported in this patch:

1. patterns for detection are registered per direction. If the proto
   was not recognized in the traffic direction, and midstream is
   enabled, the pattern set for the opposing direction is also
   evaluated. If that matches, the flow is considered to be in the
   wrong direction and is reversed.

2. probing parsers now have a way to feed back their understanding
   of the flow direction. They are now passed the direction as
   Suricata sees the traffic when calling the probing parsers. The
   parser can then see if its own observation matches that, and
   pass back it's own view to the caller.

3. a new pattern + probing parser set up: probing parsers can now
   be registered with a pattern, so that when the pattern matches
   the probing parser is called as well. The probing parser can
   then provide the protocol detection engine with the direction
   of the traffic.

The process of reversing takes a multi step approach as well:

a. reverse the current packets direction
b. reverse most of the flows direction sensitive flags
c. tag the flow as 'reversed'. This is because the 5 tuple is
   *not* reversed, since it is immutable after the flows creation.

Most of the currently registered parsers benefit already:

- HTTP/SMTP/FTP/TLS patterns are registered per direction already
  so they will benefit from the pattern midstream logic in (1)
  above.

- the Rust based SMB parser uses a mix of pattern + probing parser
  as described in (3) above.

- the NFS detection is purely done by probing parser and is updated
  to consider the direction in that parser.

Other protocols, such as DNS, are still to do.

Ticket: #2572

rust/src/applayertemplate/template.rs

@@ -257,8 +257,10 @@ export_tx_set_detect_state!(
 #[no_mangle]
 pub extern "C" fn rs_template_probing_parser(
     _flow: *const Flow,
+    _direction: u8,
     input: *const libc::uint8_t,
     input_len: u32,
+    _rdir: *mut u8
 ) -> AppProto {
     // Need at least 2 bytes.
     if input_len > 1 && input != std::ptr::null_mut() {

rust/src/dhcp/dhcp.rs

@@ -219,8 +219,11 @@ impl DHCPState {
 
 #[no_mangle]
 pub extern "C" fn rs_dhcp_probing_parser(_flow: *const Flow,
+                                         _direction: u8,
                                          input: *const libc::uint8_t,
-                                         input_len: u32) -> AppProto {
+                                         input_len: u32,
+                                         _rdir: *mut u8) -> AppProto
+{
     if input_len < DHCP_MIN_FRAME_LEN {
         return ALPROTO_UNKNOWN;
     }

rust/src/ikev2/ikev2.rs

@@ -624,7 +624,11 @@ pub extern "C" fn rs_ikev2_state_get_event_info(event_name: *const libc::c_char,
 static mut ALPROTO_IKEV2 : AppProto = ALPROTO_UNKNOWN;
 
 #[no_mangle]
-pub extern "C" fn rs_ikev2_probing_parser(_flow: *const Flow, input:*const libc::uint8_t, input_len: u32) -> AppProto {
+pub extern "C" fn rs_ikev2_probing_parser(_flow: *const Flow,
+        _direction: u8,
+        input:*const libc::uint8_t, input_len: u32,
+        _rdir: *mut u8) -> AppProto
+{
     let slice = build_slice!(input,input_len as usize);
     let alproto = unsafe{ ALPROTO_IKEV2 };
     match parse_ikev2_header(slice) {

rust/src/krb/krb5.rs

@@ -405,7 +405,11 @@ pub extern "C" fn rs_krb5_state_get_event_info(event_name: *const libc::c_char,
 static mut ALPROTO_KRB5 : AppProto = ALPROTO_UNKNOWN;
 
 #[no_mangle]
-pub extern "C" fn rs_krb5_probing_parser(_flow: *const Flow, input:*const libc::uint8_t, input_len: u32) -> AppProto {
+pub extern "C" fn rs_krb5_probing_parser(_flow: *const Flow,
+        _direction: u8,
+        input:*const libc::uint8_t, input_len: u32,
+        _rdir: *mut u8) -> AppProto
+{
     let slice = build_slice!(input,input_len as usize);
     let alproto = unsafe{ ALPROTO_KRB5 };
     if slice.len() <= 10 { return unsafe{ALPROTO_FAILED}; }
@@ -439,14 +443,19 @@ pub extern "C" fn rs_krb5_probing_parser(_flow: *const Flow, input:*const libc::
 }
 
 #[no_mangle]
-pub extern "C" fn rs_krb5_probing_parser_tcp(_flow: *const Flow, input:*const libc::uint8_t, input_len: u32) -> AppProto {
+pub extern "C" fn rs_krb5_probing_parser_tcp(_flow: *const Flow,
+        direction: u8,
+        input:*const libc::uint8_t, input_len: u32,
+        rdir: *mut u8) -> AppProto
+{
     let slice = build_slice!(input,input_len as usize);
     if slice.len() <= 14 { return unsafe{ALPROTO_FAILED}; }
     match be_u32(slice) {
         Ok((rem, record_mark)) => {
             // protocol implementations forbid very large requests
             if record_mark > 16384 { return unsafe{ALPROTO_FAILED}; }
-            return rs_krb5_probing_parser(_flow, rem.as_ptr(), rem.len() as u32);
+            return rs_krb5_probing_parser(_flow, direction,
+                    rem.as_ptr(), rem.len() as u32, rdir);
         },
         Err(nom::Err::Incomplete(_)) => {
             return ALPROTO_UNKNOWN;

rust/src/nfs/nfs.rs

@@ -1681,6 +1681,26 @@ pub extern "C" fn rs_nfs_init(context: &'static mut SuricataFileContext)
     }
 }
 
+fn nfs_probe_dir(i: &[u8], rdir: *mut u8) -> i8 {
+    match parse_rpc_packet_header(i) {
+        Ok((_, ref hdr)) => {
+            let dir = if hdr.msgtype == 0 {
+                STREAM_TOSERVER
+            } else {
+                STREAM_TOCLIENT
+            };
+            unsafe { *rdir = dir };
+            return 1;
+        },
+        Err(nom::Err::Incomplete(_)) => {
+            return 0;
+        },
+        Err(_) => {
+            return -1;
+        },
+    }
+}
+
 pub fn nfs_probe(i: &[u8], direction: u8) -> i8 {
     if direction == STREAM_TOCLIENT {
         match parse_rpc_reply(i) {
@@ -1772,6 +1792,33 @@ pub fn nfs_probe_udp(i: &[u8], direction: u8) -> i8 {
     }
 }
 
+/// MIDSTREAM
+#[no_mangle]
+pub extern "C" fn rs_nfs_probe_ms(input: *const libc::uint8_t,
+        len: libc::uint32_t, rdir: *mut u8) -> libc::int8_t
+{
+    let slice: &[u8] = unsafe {
+        std::slice::from_raw_parts(input as *mut u8, len as usize)
+    };
+    let mut direction : u8 = 0;
+    match nfs_probe_dir(slice, &mut direction) {
+        1 => {
+            let r = nfs_probe(slice, direction);
+            if r == 1 {
+                unsafe { *rdir = direction; }
+                return 1;
+            }
+            return r;
+        },
+        0 => {
+            return 0;
+        },
+        _ => {
+            return -1;
+        }
+    }
+}
+
 /// TOSERVER probe function
 #[no_mangle]
 pub extern "C" fn rs_nfs_probe_ts(input: *const libc::uint8_t, len: libc::uint32_t)

rust/src/ntp/ntp.rs

@@ -343,7 +343,11 @@ pub extern "C" fn rs_ntp_state_get_event_info(event_name: *const libc::c_char,
 static mut ALPROTO_NTP : AppProto = ALPROTO_UNKNOWN;
 
 #[no_mangle]
-pub extern "C" fn ntp_probing_parser(_flow: *const Flow, input:*const u8, input_len: u32) -> AppProto {
+pub extern "C" fn ntp_probing_parser(_flow: *const Flow,
+        _direction: u8,
+        input:*const u8, input_len: u32,
+        _rdir: *mut u8) -> AppProto
+{
     let slice: &[u8] = unsafe { std::slice::from_raw_parts(input as *mut u8, input_len as usize) };
     let alproto = unsafe{ ALPROTO_NTP };
     match parse_ntp(slice) {

rust/src/parser.rs

@@ -125,7 +125,7 @@ pub type ParseFn      = extern "C" fn (flow: *const Flow,
                                        input_len: u32,
                                        data: *const c_void,
                                        flags: u8) -> i32;
-pub type ProbeFn      = extern "C" fn (flow: *const Flow,input:*const u8, input_len: u32) -> AppProto;
+pub type ProbeFn      = extern "C" fn (flow: *const Flow,direction: u8,input:*const u8, input_len: u32, rdir: *mut u8) -> AppProto;
 pub type StateAllocFn = extern "C" fn () -> *mut c_void;
 pub type StateFreeFn  = extern "C" fn (*mut c_void);
 pub type StateTxFreeFn  = extern "C" fn (*mut c_void, u64);

rust/src/smb/smb.rs

@@ -1870,17 +1870,68 @@ pub extern "C" fn rs_smb_parse_response_tcp_gap(
 // probing parser
 // return 1 if found, 0 is not found
 #[no_mangle]
-pub extern "C" fn rs_smb_probe_tcp(input: *const libc::uint8_t, len: libc::uint32_t)
-                               -> libc::int8_t
+pub extern "C" fn rs_smb_probe_tcp(direction: libc::uint8_t,
+        input: *const libc::uint8_t, len: libc::uint32_t,
+        rdir: *mut libc::uint8_t)
+    -> libc::int8_t
 {
     let slice = build_slice!(input, len as usize);
     match search_smb_record(slice) {
-        Ok((_, _)) => {
+        Ok((_, ref data)) => {
             SCLogDebug!("smb found");
-            return 1;
+            match parse_smb_version(data) {
+                Ok((_, ref smb)) => {
+                    SCLogDebug!("SMB {:?}", smb);
+                    if smb.version == 0xff_u8 { // SMB1
+                        SCLogDebug!("SMBv1 record");
+                        match parse_smb_record(data) {
+                            Ok((_, ref smb_record)) => {
+                                if smb_record.flags & 0x80 != 0 {
+                                    SCLogDebug!("RESPONSE {:02x}", smb_record.flags);
+                                    if direction & STREAM_TOSERVER != 0 {
+                                        unsafe { *rdir = STREAM_TOCLIENT; }
+                                    }
+                                } else {
+                                    SCLogDebug!("REQUEST {:02x}", smb_record.flags);
+                                    if direction & STREAM_TOCLIENT != 0 {
+                                        unsafe { *rdir = STREAM_TOSERVER; }
+                                    }
+                                }
+                                return 1;
+                            },
+                            _ => { },
+                        }
+                    } else if smb.version == 0xfe_u8 { // SMB2
+                        SCLogDebug!("SMB2 record");
+                        match parse_smb2_record_direction(data) {
+                            Ok((_, ref smb_record)) => {
+                                if direction & STREAM_TOSERVER != 0 {
+                                    SCLogDebug!("direction STREAM_TOSERVER smb_record {:?}", smb_record);
+                                    if !smb_record.request {
+                                        unsafe { *rdir = STREAM_TOCLIENT; }
+                                    }
+                                } else {
+                                    SCLogDebug!("direction STREAM_TOCLIENT smb_record {:?}", smb_record);
+                                    if smb_record.request {
+                                        unsafe { *rdir = STREAM_TOSERVER; }
+                                    }
+                                }
+                            },
+                            _ => {},
+                        }
+                    }
+                    else if smb.version == 0xfd_u8 { // SMB3 transform
+                        SCLogDebug!("SMB3 record");
+                    }
+                    return 1;
+                },
+                    _ => {
+                        SCLogDebug!("smb not found in {:?}", slice);
+                    },
+            }
         },
         _ => {
-            SCLogDebug!("smb not found in {:?}", slice);
+            SCLogDebug!("no dice");
         },
     }
     match parse_nbss_record_partial(slice) {

rust/src/smb/smb2_records.rs

@@ -32,6 +32,21 @@ named!(pub parse_smb2_sec_blob<Smb2SecBlobRecord>,
             })
 ));
 
+#[derive(Debug,PartialEq)]
+pub struct Smb2RecordDir<> {
+    pub request: bool,
+}
+
+named!(pub parse_smb2_record_direction<Smb2RecordDir>,
+    do_parse!(
+            _server_component: tag!(b"\xfeSMB")
+        >>  _skip: take!(12)
+        >>  flags: le_u8
+        >> (Smb2RecordDir {
+                request: flags & 0x01 == 0,
+           })
+));
+
 #[derive(Debug,PartialEq)]
 pub struct Smb2Record<'a> {
     pub direction: u8,    // 0 req, 1 res

src/app-layer-detect-proto.h

@@ -27,8 +27,9 @@
 
 typedef struct AppLayerProtoDetectThreadCtx_ AppLayerProtoDetectThreadCtx;
 
-typedef AppProto (*ProbingParserFPtr)(Flow *f,
-                                      uint8_t *input, uint32_t input_len);
+typedef AppProto (*ProbingParserFPtr)(Flow *f, uint8_t dir,
+                                      uint8_t *input, uint32_t input_len,
+                                      uint8_t *rdir);
 
 /***** Protocol Retrieval *****/
 
@@ -41,13 +42,15 @@ typedef AppProto (*ProbingParserFPtr)(Flow *f,
  * \param buflen The length of the above buffer.
  * \param ipproto The ip protocol.
  * \param direction The direction bitfield - STREAM_TOSERVER/STREAM_TOCLIENT.
+ * \param[out] reverse_flow true if flow is detected to be reversed
  *
  * \retval The app layer protocol.
  */
 AppProto AppLayerProtoDetectGetProto(AppLayerProtoDetectThreadCtx *tctx,
                                      Flow *f,
                                      uint8_t *buf, uint32_t buflen,
-                                     uint8_t ipproto, uint8_t direction);
+                                     uint8_t ipproto, uint8_t direction,
+                                     bool *reverse_flow);
 
 /***** State Preparation *****/
 
@@ -84,9 +87,14 @@ int AppLayerProtoDetectPPParseConfPorts(const char *ipproto_name,
  * \brief Registers a case-sensitive pattern for protocol detection.
  */
 int AppLayerProtoDetectPMRegisterPatternCS(uint8_t ipproto, AppProto alproto,
-                                           const char *pattern,
-                                           uint16_t depth, uint16_t offset,
-                                           uint8_t direction);
+        const char *pattern, uint16_t depth, uint16_t offset,
+        uint8_t direction);
+int AppLayerProtoDetectPMRegisterPatternCSwPP(uint8_t ipproto, AppProto alproto,
+        const char *pattern, uint16_t depth, uint16_t offset,
+        uint8_t direction,
+        ProbingParserFPtr PPFunc,
+        uint16_t pp_min_depth, uint16_t pp_max_depth);
+
 /**
  * \brief Registers a case-insensitive pattern for protocol detection.
  */

src/app-layer-dnp3.c

@@ -264,7 +264,9 @@ static int DNP3ContainsBanner(const uint8_t *input, uint32_t len)
 /**
  * \brief DNP3 probing parser.
  */
-static uint16_t DNP3ProbingParser(Flow *f, uint8_t *input, uint32_t len)
+static uint16_t DNP3ProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t len,
+        uint8_t *rdir)
 {
     DNP3LinkHeader *hdr = (DNP3LinkHeader *)input;
 
@@ -2041,27 +2043,28 @@ static int DNP3ProbingParserTest(void)
         0x05, 0x64, 0x05, 0xc9, 0x03, 0x00, 0x04, 0x00,
         0xbd, 0x71
     };
+    uint8_t rdir = 0;
 
     /* Valid frame. */
-    FAIL_IF(DNP3ProbingParser(NULL, pkt, sizeof(pkt)) != ALPROTO_DNP3);
+    FAIL_IF(DNP3ProbingParser(NULL, STREAM_TOSERVER, pkt, sizeof(pkt), &rdir) != ALPROTO_DNP3);
 
     /* Send too little bytes. */
-    FAIL_IF(DNP3ProbingParser(NULL, pkt, sizeof(DNP3LinkHeader) - 1) != ALPROTO_UNKNOWN);
+    FAIL_IF(DNP3ProbingParser(NULL, STREAM_TOSERVER, pkt, sizeof(DNP3LinkHeader) - 1, &rdir) != ALPROTO_UNKNOWN);
 
     /* Bad start bytes. */
     pkt[0] = 0x06;
-    FAIL_IF(DNP3ProbingParser(NULL, pkt, sizeof(pkt)) != ALPROTO_FAILED);
+    FAIL_IF(DNP3ProbingParser(NULL, STREAM_TOSERVER, pkt, sizeof(pkt), &rdir) != ALPROTO_FAILED);
 
     /* Restore start byte. */
     pkt[0] = 0x05;
 
     /* Set the length to a value less than the minimum length of 5. */
     pkt[2] = 0x03;
-    FAIL_IF(DNP3ProbingParser(NULL, pkt, sizeof(pkt)) != ALPROTO_FAILED);
+    FAIL_IF(DNP3ProbingParser(NULL, STREAM_TOSERVER, pkt, sizeof(pkt), &rdir) != ALPROTO_FAILED);
 
     /* Send a banner. */
     char mybanner[] = "Welcome to DNP3 SCADA.";
-    FAIL_IF(DNP3ProbingParser(NULL, (uint8_t *)mybanner, sizeof(mybanner)) != ALPROTO_DNP3);
+    FAIL_IF(DNP3ProbingParser(NULL, STREAM_TOSERVER, (uint8_t *)mybanner, sizeof(mybanner), &rdir) != ALPROTO_DNP3);
 
     PASS;
 }

src/app-layer-dns-tcp-rust.c

@@ -52,7 +52,8 @@ static int RustDNSTCPParseResponse(Flow *f, void *state,
             local_data);
 }
 
-static uint16_t RustDNSTCPProbe(Flow *f, uint8_t *input, uint32_t len)
+static uint16_t RustDNSTCPProbe(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t len, uint8_t *rdir)
 {
     SCLogDebug("RustDNSTCPProbe");
     if (len == 0 || len < sizeof(DNSHeader)) {

src/app-layer-dns-tcp.c

@@ -63,7 +63,9 @@ struct DNSTcpHeader_ {
 } __attribute__((__packed__));
 typedef struct DNSTcpHeader_ DNSTcpHeader;
 
-static uint16_t DNSTcpProbingParser(Flow *f, uint8_t *input, uint32_t ilen);
+static uint16_t DNSTcpProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t ilen,
+        uint8_t *rdir);
 
 /** \internal
  *  \param input_len at least enough for the DNSTcpHeader
@@ -317,7 +319,8 @@ static int DNSTCPRequestParse(Flow *f, void *dstate,
 
     /* Clear gap state. */
     if (dns_state->gap_ts) {
-        if (DNSTcpProbingParser(f, input, input_len) == ALPROTO_DNS) {
+        if (DNSTcpProbingParser(f, STREAM_TOSERVER,
+                    input, input_len, NULL) == ALPROTO_DNS) {
             SCLogDebug("New data probed as DNS, clearing gap state.");
             BufferReset(dns_state);
             dns_state->gap_ts = 0;
@@ -557,7 +560,8 @@ static int DNSTCPResponseParse(Flow *f, void *dstate,
 
     /* Clear gap state. */
     if (dns_state->gap_tc) {
-        if (DNSTcpProbingParser(f, input, input_len) == ALPROTO_DNS) {
+        if (DNSTcpProbingParser(f, STREAM_TOCLIENT,
+                    input, input_len, NULL) == ALPROTO_DNS) {
             SCLogDebug("New data probed as DNS, clearing gap state.");
             BufferReset(dns_state);
             dns_state->gap_tc = 0;
@@ -639,7 +643,9 @@ bad_data:
     SCReturnInt(-1);
 }
 
-static uint16_t DNSTcpProbingParser(Flow *f, uint8_t *input, uint32_t ilen)
+static uint16_t DNSTcpProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t ilen,
+        uint8_t *rdir)
 {
     if (ilen == 0 || ilen < sizeof(DNSTcpHeader)) {
         SCLogDebug("ilen too small, hoped for at least %"PRIuMAX, (uintmax_t)sizeof(DNSTcpHeader));
@@ -679,7 +685,9 @@ static uint16_t DNSTcpProbingParser(Flow *f, uint8_t *input, uint32_t ilen)
  * This is a minimal parser that just checks that the input contains enough
  * data for a TCP DNS response.
  */
-static uint16_t DNSTcpProbeResponse(Flow *f, uint8_t *input, uint32_t len)
+static uint16_t DNSTcpProbeResponse(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t len,
+        uint8_t *rdir)
 {
     if (len == 0 || len < sizeof(DNSTcpHeader)) {
         return ALPROTO_UNKNOWN;

src/app-layer-dns-udp-rust.c

@@ -50,7 +50,8 @@ static int RustDNSUDPParseResponse(Flow *f, void *state,
             local_data);
 }
 
-static uint16_t DNSUDPProbe(Flow *f, uint8_t *input, uint32_t len)
+static uint16_t DNSUDPProbe(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t len, uint8_t *rdir)
 {
     if (len == 0 || len < sizeof(DNSHeader)) {
         return ALPROTO_UNKNOWN;

src/app-layer-dns-udp.c

@@ -331,7 +331,8 @@ insufficient_data:
     SCReturnInt(-1);
 }
 
-static uint16_t DNSUdpProbingParser(Flow *f, uint8_t *input, uint32_t ilen)
+static uint16_t DNSUdpProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t ilen, uint8_t *rdir)
 {
     if (ilen == 0 || ilen < sizeof(DNSHeader)) {
         SCLogDebug("ilen too small, hoped for at least %"PRIuMAX, (uintmax_t)sizeof(DNSHeader));

src/app-layer-enip.c

@@ -359,7 +359,8 @@ static int ENIPParse(Flow *f, void *state, AppLayerParserState *pstate,
 
 
 
-static uint16_t ENIPProbingParser(Flow *f, uint8_t *input, uint32_t input_len)
+static uint16_t ENIPProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t input_len, uint8_t *rdir)
 {
     // SCLogDebug("ENIPProbingParser %d", input_len);
     if (input_len < sizeof(ENIPEncapHdr))

src/app-layer-modbus.c

@@ -1427,8 +1427,10 @@ static void ModbusStateFree(void *state)
 }
 
 static uint16_t ModbusProbingParser(Flow *f,
+                                    uint8_t direction,
                                     uint8_t     *input,
-                                    uint32_t    input_len)
+                                    uint32_t    input_len,
+                                    uint8_t *rdir)
 {
     ModbusHeader *header = (ModbusHeader *) input;
 

src/app-layer-nfs-tcp.c

@@ -112,30 +112,21 @@ static AppLayerDecoderEvents *NFSTCPGetEvents(void *state, uint64_t id)
  * \retval ALPROTO_NFS if it looks like echo, otherwise
  *     ALPROTO_UNKNOWN.
  */
-static AppProto NFSTCPProbingParserTS(Flow *f, uint8_t *input, uint32_t input_len)
+static AppProto NFSTCPProbingParser(Flow *f,
+        uint8_t direction,
+        uint8_t *input, uint32_t input_len,
+        uint8_t *rdir)
 {
     if (input_len < NFSTCP_MIN_FRAME_LEN) {
         return ALPROTO_UNKNOWN;
     }
 
-    int8_t r = rs_nfs_probe_ts(input, input_len);
-    if (r == 1) {
-        return ALPROTO_NFS;
-    } else if (r == -1) {
-        return ALPROTO_FAILED;
-    }
-
-    SCLogDebug("Protocol not detected as ALPROTO_NFS.");
-    return ALPROTO_UNKNOWN;
-}
-
-static AppProto NFSTCPProbingParserTC(Flow *f, uint8_t *input, uint32_t input_len)
-{
-    if (input_len < NFSTCP_MIN_FRAME_LEN) {
-        return ALPROTO_UNKNOWN;
+    int8_t r = 0;
+    if (direction & STREAM_TOSERVER) {
+        r = rs_nfs_probe_ts(input, input_len);
+    } else {
+        r = rs_nfs_probe_tc(input, input_len);
     }
-
-    int8_t r = rs_nfs_probe_tc(input, input_len);
     if (r == 1) {
         return ALPROTO_NFS;
     } else if (r == -1) {
@@ -287,21 +278,22 @@ void RegisterNFSTCPParsers(void)
             SCLogDebug("Unittest mode, registering default configuration.");
             AppLayerProtoDetectPPRegister(IPPROTO_TCP, NFSTCP_DEFAULT_PORT,
                 ALPROTO_NFS, 0, NFSTCP_MIN_FRAME_LEN, STREAM_TOSERVER,
-                NFSTCPProbingParserTS, NFSTCPProbingParserTC);
+                NFSTCPProbingParser, NFSTCPProbingParser);
 
         }
         else {
 
             if (!AppLayerProtoDetectPPParseConfPorts("tcp", IPPROTO_TCP,
                     proto_name, ALPROTO_NFS, 0, NFSTCP_MIN_FRAME_LEN,
-                    NFSTCPProbingParserTS, NFSTCPProbingParserTC)) {
+                    NFSTCPProbingParser, NFSTCPProbingParser)) {
                 SCLogDebug("No NFSTCP app-layer configuration, enabling NFSTCP"
                     " detection TCP detection on port %s.",
                     NFSTCP_DEFAULT_PORT);
+                /* register 'midstream' probing parsers if midstream is enabled. */
                 AppLayerProtoDetectPPRegister(IPPROTO_TCP,
                     NFSTCP_DEFAULT_PORT, ALPROTO_NFS, 0,
                     NFSTCP_MIN_FRAME_LEN, STREAM_TOSERVER,
-                    NFSTCPProbingParserTS, NFSTCPProbingParserTC);
+                    NFSTCPProbingParser, NFSTCPProbingParser);
             }
 
         }

src/app-layer-nfs-udp.c

@@ -109,7 +109,8 @@ static AppLayerDecoderEvents *NFSGetEvents(void *state, uint64_t id)
  * \retval ALPROTO_NFS if it looks like echo, otherwise
  *     ALPROTO_UNKNOWN.
  */
-static AppProto NFSProbingParserTS(Flow *f, uint8_t *input, uint32_t input_len)
+static AppProto NFSProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t input_len, uint8_t *rdir)
 {
     SCLogDebug("probing");
     if (input_len < NFS_MIN_FRAME_LEN) {
@@ -117,28 +118,12 @@ static AppProto NFSProbingParserTS(Flow *f, uint8_t *input, uint32_t input_len)
         return ALPROTO_UNKNOWN;
     }
 
-    int8_t r = rs_nfs_probe_udp_ts(input, input_len);
-    if (r == 1) {
-        SCLogDebug("nfs");
-        return ALPROTO_NFS;
-    } else if (r == -1) {
-        SCLogDebug("failed");
-        return ALPROTO_FAILED;
-    }
-
-    SCLogDebug("Protocol not detected as ALPROTO_NFS.");
-    return ALPROTO_UNKNOWN;
-}
-
-static AppProto NFSProbingParserTC(Flow *f, uint8_t *input, uint32_t input_len)
-{
-    SCLogDebug("probing");
-    if (input_len < NFS_MIN_FRAME_LEN) {
-        SCLogDebug("unknown");
-        return ALPROTO_UNKNOWN;
-    }
+    int8_t r = 0;
+    if (direction & STREAM_TOSERVER)
+        r = rs_nfs_probe_udp_ts(input, input_len);
+    else
+        r = rs_nfs_probe_udp_tc(input, input_len);
 
-    int8_t r = rs_nfs_probe_tc(input, input_len);
     if (r == 1) {
         SCLogDebug("nfs");
         return ALPROTO_NFS;
@@ -280,21 +265,21 @@ void RegisterNFSUDPParsers(void)
             SCLogDebug("Unittest mode, registering default configuration.");
             AppLayerProtoDetectPPRegister(IPPROTO_UDP, NFS_DEFAULT_PORT,
                 ALPROTO_NFS, 0, NFS_MIN_FRAME_LEN, STREAM_TOSERVER,
-                NFSProbingParserTS, NFSProbingParserTC);
+                NFSProbingParser, NFSProbingParser);
 
         }
         else {
 
             if (!AppLayerProtoDetectPPParseConfPorts("udp", IPPROTO_UDP,
                     proto_name, ALPROTO_NFS, 0, NFS_MIN_FRAME_LEN,
-                    NFSProbingParserTS, NFSProbingParserTC)) {
+                    NFSProbingParser, NFSProbingParser)) {
                 SCLogDebug("No NFS app-layer configuration, enabling NFS"
                     " detection TCP detection on port %s.",
                     NFS_DEFAULT_PORT);
                 AppLayerProtoDetectPPRegister(IPPROTO_UDP,
                     NFS_DEFAULT_PORT, ALPROTO_NFS, 0,
                     NFS_MIN_FRAME_LEN, STREAM_TOSERVER,
-                    NFSProbingParserTS, NFSProbingParserTC);
+                    NFSProbingParser, NFSProbingParser);
             }
 
         }

src/app-layer-protos.h

@@ -68,6 +68,11 @@ enum AppProtoEnum {
 /* not using the enum as that is a unsigned int, so 4 bytes */
 typedef uint16_t AppProto;
 
+static inline bool AppProtoIsValid(AppProto a)
+{
+    return ((a > ALPROTO_UNKNOWN && a < ALPROTO_FAILED));
+}
+
 /**
  * \brief Maps the ALPROTO_*, to its string equivalent.
  *

src/app-layer-smb-tcp-rust.c

@@ -78,8 +78,8 @@ static int RustSMBTCPParseResponse(Flow *f, void *state,
     return res;
 }
 
-static uint16_t RustSMBTCPProbe(Flow *f,
-        uint8_t *input, uint32_t len)
+static uint16_t RustSMBTCPProbe(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t len, uint8_t *rdir)
 {
     SCLogDebug("RustSMBTCPProbe");
 
@@ -87,7 +87,7 @@ static uint16_t RustSMBTCPProbe(Flow *f,
         return ALPROTO_UNKNOWN;
     }
 
-    const int r = rs_smb_probe_tcp(input, len);
+    const int r = rs_smb_probe_tcp(direction, input, len, rdir);
     switch (r) {
         case 1:
             return ALPROTO_SMB;
@@ -184,22 +184,28 @@ static int RustSMBRegisterPatternsForProtocolDetection(void)
 {
     int r = 0;
     /* SMB1 */
-    r |= AppLayerProtoDetectPMRegisterPatternCS(IPPROTO_TCP, ALPROTO_SMB,
-            "|ff|SMB", 8, 4, STREAM_TOSERVER);
-    r |= AppLayerProtoDetectPMRegisterPatternCS(IPPROTO_TCP, ALPROTO_SMB,
-            "|ff|SMB", 8, 4, STREAM_TOCLIENT);
+    r |= AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_SMB,
+            "|ff|SMB", 8, 4, STREAM_TOSERVER, RustSMBTCPProbe,
+            MIN_REC_SIZE, MIN_REC_SIZE);
+    r |= AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_SMB,
+            "|ff|SMB", 8, 4, STREAM_TOCLIENT, RustSMBTCPProbe,
+            MIN_REC_SIZE, MIN_REC_SIZE);
 
     /* SMB2/3 */
-    r |= AppLayerProtoDetectPMRegisterPatternCS(IPPROTO_TCP, ALPROTO_SMB,
-            "|fe|SMB", 8, 4, STREAM_TOSERVER);
-    r |= AppLayerProtoDetectPMRegisterPatternCS(IPPROTO_TCP, ALPROTO_SMB,
-            "|fe|SMB", 8, 4, STREAM_TOCLIENT);
+    r |= AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_SMB,
+            "|fe|SMB", 8, 4, STREAM_TOSERVER, RustSMBTCPProbe,
+            MIN_REC_SIZE, MIN_REC_SIZE);
+    r |= AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_SMB,
+            "|fe|SMB", 8, 4, STREAM_TOCLIENT, RustSMBTCPProbe,
+            MIN_REC_SIZE, MIN_REC_SIZE);
 
     /* SMB3 encrypted records */
-    r |= AppLayerProtoDetectPMRegisterPatternCS(IPPROTO_TCP, ALPROTO_SMB,
-            "|fd|SMB", 8, 4, STREAM_TOSERVER);
-    r |= AppLayerProtoDetectPMRegisterPatternCS(IPPROTO_TCP, ALPROTO_SMB,
-            "|fd|SMB", 8, 4, STREAM_TOCLIENT);
+    r |= AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_SMB,
+            "|fd|SMB", 8, 4, STREAM_TOSERVER, RustSMBTCPProbe,
+            MIN_REC_SIZE, MIN_REC_SIZE);
+    r |= AppLayerProtoDetectPMRegisterPatternCSwPP(IPPROTO_TCP, ALPROTO_SMB,
+            "|fd|SMB", 8, 4, STREAM_TOCLIENT, RustSMBTCPProbe,
+            MIN_REC_SIZE, MIN_REC_SIZE);
     return r == 0 ? 0 : -1;
 }
 

src/app-layer-smb.c

@@ -1512,7 +1512,8 @@ static int SMBGetAlstateProgress(void *tx, uint8_t direction)
 
 #define SMB_PROBING_PARSER_MIN_DEPTH 8
 
-static uint16_t SMBProbingParser(Flow *f, uint8_t *input, uint32_t ilen)
+static uint16_t SMBProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t ilen, uint8_t *rdir)
 {
     int32_t len;
     int32_t input_len = ilen;
@@ -2298,10 +2299,11 @@ static int SMBParserTest05(void)
     AppLayerProtoDetectPrepareState();
     alpd_tctx = AppLayerProtoDetectGetCtxThread();
 
+    bool reverse_flow = false;
     alproto = AppLayerProtoDetectGetProto(alpd_tctx,
                                           &f,
                                           smbbuf1, smblen1,
-                                          IPPROTO_TCP, STREAM_TOSERVER);
+                                          IPPROTO_TCP, STREAM_TOSERVER, &reverse_flow);
     if (alproto != ALPROTO_UNKNOWN) {
         printf("alproto is %"PRIu16 ".  Should be ALPROTO_UNKNOWN\n",
                alproto);
@@ -2311,7 +2313,7 @@ static int SMBParserTest05(void)
     alproto = AppLayerProtoDetectGetProto(alpd_tctx,
                                           &f,
                                           smbbuf2, smblen2,
-                                          IPPROTO_TCP, STREAM_TOSERVER);
+                                          IPPROTO_TCP, STREAM_TOSERVER, &reverse_flow);
     if (alproto != ALPROTO_SMB) {
         printf("alproto is %"PRIu16 ".  Should be ALPROTO_SMB\n",
                alproto);
@@ -2382,10 +2384,11 @@ static int SMBParserTest06(void)
     AppLayerProtoDetectPrepareState();
     alpd_tctx = AppLayerProtoDetectGetCtxThread();
 
+    bool reverse_flow = false;
     alproto = AppLayerProtoDetectGetProto(alpd_tctx,
                                           &f,
                                           smbbuf1, smblen1,
-                                          IPPROTO_TCP, STREAM_TOSERVER);
+                                          IPPROTO_TCP, STREAM_TOSERVER, &reverse_flow);
     if (alproto != ALPROTO_UNKNOWN) {
         printf("alproto is %"PRIu16 ".  Should be ALPROTO_UNKNOWN\n",
                alproto);
@@ -2395,7 +2398,7 @@ static int SMBParserTest06(void)
     alproto = AppLayerProtoDetectGetProto(alpd_tctx,
                                           &f,
                                           smbbuf2, smblen2,
-                                          IPPROTO_TCP, STREAM_TOSERVER);
+                                          IPPROTO_TCP, STREAM_TOSERVER, &reverse_flow);
     if (alproto != ALPROTO_SMB) {
         printf("alproto is %"PRIu16 ".  Should be ALPROTO_SMB\n",
                alproto);

src/app-layer-ssl.c

@@ -2624,7 +2624,8 @@ static void SSLStateTransactionFree(void *state, uint64_t tx_id)
     /* do nothing */
 }
 
-static uint16_t SSLProbingParser(Flow *f, uint8_t *input, uint32_t ilen)
+static AppProto SSLProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t ilen, uint8_t *rdir)
 {
     /* probably a rst/fin sending an eof */
     if (ilen == 0)

src/app-layer-template.c

@@ -194,7 +194,8 @@ static AppLayerDecoderEvents *TemplateGetEvents(void *statev, uint64_t tx_id)
  * \retval ALPROTO_TEMPLATE if it looks like template, otherwise
  *     ALPROTO_UNKNOWN.
  */
-static AppProto TemplateProbingParser(Flow *f, uint8_t *input, uint32_t input_len)
+static AppProto TemplateProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t input_len, uint8_t *rdir)
 {
     /* Very simple test - if there is input, this is template. */
     if (input_len >= TEMPLATE_MIN_FRAME_LEN) {

src/app-layer-tftp.c

@@ -112,7 +112,8 @@ static AppLayerDecoderEvents *TFTPGetEvents(void *state, uint64_t tx_id)
  * \retval ALPROTO_TFTP if it looks like echo, otherwise
  *     ALPROTO_UNKNOWN.
  */
-static AppProto TFTPProbingParser(Flow *f, uint8_t *input, uint32_t input_len)
+static AppProto TFTPProbingParser(Flow *f, uint8_t direction,
+        uint8_t *input, uint32_t input_len, uint8_t *rdir)
 {
     /* Very simple test - if there is input, this is tftp.
      * Also check if it's starting by a zero */

src/app-layer.c

@@ -323,11 +323,13 @@ static int TCPProtoDetect(ThreadVars *tv,
     }
 #endif
 
+    bool reverse_flow = false;
     PACKET_PROFILING_APP_PD_START(app_tctx);
     *alproto = AppLayerProtoDetectGetProto(app_tctx->alpd_tctx,
             f, data, data_len,
-            IPPROTO_TCP, flags);
+            IPPROTO_TCP, flags, &reverse_flow);
     PACKET_PROFILING_APP_PD_END(app_tctx);
+    SCLogDebug("alproto %u rev %s", *alproto, reverse_flow ? "true" : "false");
 
     if (*alproto != ALPROTO_UNKNOWN) {
         if (*alproto_otherdir != ALPROTO_UNKNOWN && *alproto_otherdir != *alproto) {
@@ -353,6 +355,15 @@ static int TCPProtoDetect(ThreadVars *tv,
         TcpSessionSetReassemblyDepth(ssn,
                 AppLayerParserGetStreamDepth(f));
         FlagPacketFlow(p, f, flags);
+        /* if protocol detection indicated that we need to reverse
+         * the direction of the flow, do it now. We flip the flow,
+         * packet and the direction flags */
+        if (reverse_flow) {
+            SCLogDebug("reversing flow after proto detect told us so");
+            PacketSwap(p);
+            FlowSwap(f);
+            SWAP_FLAGS(flags, STREAM_TOSERVER, STREAM_TOCLIENT);
+        }
 
         /* account flow if we have both sides */
         if (*alproto_otherdir != ALPROTO_UNKNOWN) {
@@ -680,15 +691,23 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *
         SCLogDebug("Detecting AL proto on udp mesg (len %" PRIu32 ")",
                    p->payload_len);
 
+        bool reverse_flow = false;
         PACKET_PROFILING_APP_PD_START(tctx);
         f->alproto = AppLayerProtoDetectGetProto(tctx->alpd_tctx,
                                   f, p->payload, p->payload_len,
-                                  IPPROTO_UDP, flags);
+                                  IPPROTO_UDP, flags, &reverse_flow);
         PACKET_PROFILING_APP_PD_END(tctx);
 
         if (f->alproto != ALPROTO_UNKNOWN) {
             AppLayerIncFlowCounter(tv, f);
 
+            if (reverse_flow) {
+                SCLogDebug("reversing flow after proto detect told us so");
+                PacketSwap(p);
+                FlowSwap(f);
+                SWAP_FLAGS(flags, STREAM_TOSERVER, STREAM_TOCLIENT);
+            }
+
             PACKET_PROFILING_APP_START(tctx, f->alproto);
             r = AppLayerParserParse(tv, tctx->alp_tctx, f, f->alproto,
                                     flags, p->payload, p->payload_len);