diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs index fa275f9682..85cfedb45b 100644 --- a/rust/src/applayer.rs +++ b/rust/src/applayer.rs @@ -109,6 +109,8 @@ pub struct AppLayerTxData { pub files_logged: u32, pub files_stored: u32, + pub file_flags: u16, + /// detection engine flags for use by detection engine detect_flags_ts: u64, detect_flags_tc: u64, @@ -142,6 +144,7 @@ impl AppLayerTxData { files_opened: 0, files_logged: 0, files_stored: 0, + file_flags: 0, detect_flags_ts: 0, detect_flags_tc: 0, de_state: std::ptr::null_mut(), @@ -158,6 +161,13 @@ impl AppLayerTxData { pub fn set_event(&mut self, event: u8) { core::sc_app_layer_decoder_events_set_event_raw(&mut self.events, event as u8); } + + pub fn update_file_flags(&mut self, state_flags: u16) { + if (self.file_flags & state_flags) != state_flags { + SCLogDebug!("updating tx file_flags {:04x} with state flags {:04x}", self.file_flags, state_flags); + self.file_flags |= state_flags; + } + } } #[macro_export] @@ -322,7 +332,7 @@ pub struct RustParser { pub localstorage_free: Option, /// Function to get files - pub get_files: Option, + pub get_tx_files: Option, /// Function to get the TX iterator pub get_tx_iterator: Option, @@ -378,8 +388,7 @@ pub type GetEventInfoFn = unsafe extern "C" fn (*const c_char, *mut c_int, * pub type GetEventInfoByIdFn = unsafe extern "C" fn (c_int, *mut *const c_char, *mut AppLayerEventType) -> i8; pub type LocalStorageNewFn = extern "C" fn () -> *mut c_void; pub type LocalStorageFreeFn = extern "C" fn (*mut c_void); -pub type GetFilesFn = unsafe -extern "C" fn (*mut c_void, u8) -> *mut FileContainer; +pub type GetTxFilesFn = unsafe extern "C" fn (*mut c_void, u8) -> *mut FileContainer; pub type GetTxIteratorFn = unsafe extern "C" fn (ipproto: u8, alproto: AppProto, state: *mut c_void, min_tx_id: u64, diff --git a/rust/src/applayertemplate/template.rs b/rust/src/applayertemplate/template.rs index 12719b5b65..3b6a9c2046 100644 --- a/rust/src/applayertemplate/template.rs +++ b/rust/src/applayertemplate/template.rs @@ -455,7 +455,7 @@ pub unsafe extern "C" fn rs_template_register_parser() { get_eventinfo_byid : Some(TemplateEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_template_get_tx_data, get_state_data: rs_template_get_state_data, diff --git a/rust/src/core.rs b/rust/src/core.rs index 197aa54fb0..f1d6404219 100644 --- a/rust/src/core.rs +++ b/rust/src/core.rs @@ -94,10 +94,11 @@ pub static mut ALPROTO_FAILED : AppProto = 0; // updated during init pub const IPPROTO_TCP : u8 = 6; pub const IPPROTO_UDP : u8 = 17; +/* macro_rules!BIT_U8 { ($x:expr) => (1 << $x); } - +*/ macro_rules!BIT_U16 { ($x:expr) => (1 << $x); } @@ -181,10 +182,6 @@ pub type SCFilePrune = extern "C" fn ( pub type SCFileContainerRecycle = extern "C" fn ( file_container: &FileContainer); -pub type SCFileSetTx = extern "C" fn ( - file: &FileContainer, - tx_id: u64); - // A Suricata context that is passed in from C. This is alternative to // using functions from Suricata directly, so they can be wrapped so // Rust unit tests will still compile when they are not linked @@ -210,7 +207,6 @@ pub struct SuricataContext { pub FileAppendGAP: SCFileAppendGAPById, pub FileContainerRecycle: SCFileContainerRecycle, pub FilePrune: SCFilePrune, - pub FileSetTx: SCFileSetTx, pub AppLayerRegisterParser: extern fn(parser: *const crate::applayer::RustParser, alproto: AppProto) -> std::os::raw::c_int, } diff --git a/rust/src/dcerpc/dcerpc.rs b/rust/src/dcerpc/dcerpc.rs index 91e698d90e..d7e0d904cc 100644 --- a/rust/src/dcerpc/dcerpc.rs +++ b/rust/src/dcerpc/dcerpc.rs @@ -1367,7 +1367,7 @@ pub unsafe extern "C" fn rs_dcerpc_register_parser() { get_eventinfo_byid : None, localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_dcerpc_get_tx_data, get_state_data: rs_dcerpc_get_state_data, diff --git a/rust/src/dcerpc/dcerpc_udp.rs b/rust/src/dcerpc/dcerpc_udp.rs index c66bfdea04..1b5513eeb5 100644 --- a/rust/src/dcerpc/dcerpc_udp.rs +++ b/rust/src/dcerpc/dcerpc_udp.rs @@ -360,7 +360,7 @@ pub unsafe extern "C" fn rs_dcerpc_udp_register_parser() { get_eventinfo_byid: None, localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_dcerpc_udp_get_tx_data, get_state_data: rs_dcerpc_udp_get_state_data, diff --git a/rust/src/dhcp/dhcp.rs b/rust/src/dhcp/dhcp.rs index 2365194299..626c368809 100644 --- a/rust/src/dhcp/dhcp.rs +++ b/rust/src/dhcp/dhcp.rs @@ -296,7 +296,7 @@ pub unsafe extern "C" fn rs_dhcp_register_parser() { get_eventinfo_byid : Some(DHCPEvent::get_event_info_by_id), localstorage_new : None, localstorage_free : None, - get_files : None, + get_tx_files : None, get_tx_iterator : Some(applayer::state_get_tx_iterator::), get_tx_data : rs_dhcp_get_tx_data, get_state_data : rs_dhcp_get_state_data, diff --git a/rust/src/dns/dns.rs b/rust/src/dns/dns.rs index 5aafd73924..cbb38824d2 100644 --- a/rust/src/dns/dns.rs +++ b/rust/src/dns/dns.rs @@ -980,7 +980,7 @@ pub unsafe extern "C" fn rs_dns_udp_register_parser() { get_eventinfo_byid: Some(DNSEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::), get_tx_data: rs_dns_state_get_tx_data, get_state_data: rs_dns_get_state_data, @@ -1026,7 +1026,7 @@ pub unsafe extern "C" fn rs_dns_tcp_register_parser() { get_eventinfo_byid: Some(DNSEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::), get_tx_data: rs_dns_state_get_tx_data, get_state_data: rs_dns_get_state_data, diff --git a/rust/src/filecontainer.rs b/rust/src/filecontainer.rs index cc5ddafee2..d00ae0fa6c 100644 --- a/rust/src/filecontainer.rs +++ b/rust/src/filecontainer.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2017 Open Information Security Foundation +/* Copyright (C) 2017-2021 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -22,7 +22,7 @@ use crate::core::*; // Defined in util-file.h extern { - pub fn FileFlowToFlags(flow: *const Flow, flags: u8) -> u16; + pub fn FileFlowFlagsToFlags(flow_file_flags: u16, flags: u8) -> u16; } pub const FILE_USE_DETECT: u16 = BIT_U16!(13); @@ -74,11 +74,8 @@ impl FileContainer { } pub fn free(&mut self) { SCLogDebug!("freeing self"); - match unsafe {SC} { - None => panic!("BUG no suricata_config"), - Some(c) => { - (c.FileContainerRecycle)(self); - }, + if let Some(c) = unsafe {SC} { + (c.FileContainerRecycle)(self); } } @@ -145,13 +142,4 @@ impl FileContainer { } } } - - pub fn file_set_txid_on_last_file(&mut self, tx_id: u64) { - match unsafe {SC} { - None => panic!("BUG no suricata_config"), - Some(c) => { - (c.FileSetTx)(self, tx_id); - } - } - } } diff --git a/rust/src/filetracker.rs b/rust/src/filetracker.rs index d10acdbe6f..dda6ea8084 100644 --- a/rust/src/filetracker.rs +++ b/rust/src/filetracker.rs @@ -87,7 +87,6 @@ impl FileTransferTracker { { let r = files.file_open(config, &self.track_id, name, flags); if r == 0 { - files.file_set_txid_on_last_file(self.tx_id); self.file_open = true; } r diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index b83c5de74b..e9789dd482 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -142,6 +142,8 @@ pub struct HTTP2Transaction { //temporary escaped header for detection //must be attached to transaction for memory management (be freed at the right time) pub escaped: Vec>, + + pub files: Files, } impl Transaction for HTTP2Transaction { @@ -165,6 +167,7 @@ impl HTTP2Transaction { ft_tc: FileTransferTracker::new(), ft_ts: FileTransferTracker::new(), escaped: Vec::with_capacity(16), + files: Files::default(), } } @@ -200,9 +203,13 @@ impl HTTP2Transaction { } } + pub fn update_file_flags(&mut self, flow_file_flags: u16) { + self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT }; + self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT }; + } + fn decompress<'a>( - &'a mut self, input: &'a [u8], dir: Direction, sfcm: &'static SuricataFileContext, - over: bool, files: &mut FileContainer, flags: u16, flow: *const Flow, + &'a mut self, input: &'a [u8], dir: Direction, sfcm: &'static SuricataFileContext, over: bool, flow: *const Flow, ) -> io::Result<()> { let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE); let decompressed = self.decoder.decompress(input, &mut output, dir)?; @@ -220,9 +227,9 @@ impl HTTP2Transaction { ) { match range::http2_parse_check_content_range(&value) { Ok((_, v)) => { - range::http2_range_open(self, &v, flow, sfcm, flags, decompressed); + range::http2_range_open(self, &v, flow, sfcm, Direction::ToClient, decompressed); if over && !self.file_range.is_null() { - range::http2_range_close(self, files, flags, &[]) + range::http2_range_close(self, Direction::ToClient, &[]) } } _ => { @@ -233,12 +240,13 @@ impl HTTP2Transaction { } else { if !self.file_range.is_null() { if over { - range::http2_range_close(self, files, flags, decompressed) + range::http2_range_close(self, Direction::ToClient, decompressed) } else { range::http2_range_append(self.file_range, decompressed) } } } + let (files, flags) = self.files.get(Direction::ToClient); self.ft_tc.new_chunk( sfcm, files, @@ -256,6 +264,7 @@ impl HTTP2Transaction { if !self.ft_ts.file_open { self.tx_data.incr_files_opened(); } + let (files, flags) = self.files.get(Direction::ToServer); self.ft_ts.new_chunk( sfcm, files, @@ -401,7 +410,6 @@ pub struct HTTP2State { dynamic_headers_tc: HTTP2DynTable, transactions: VecDeque, progress: HTTP2ConnectionState, - pub files: Files, } impl State for HTTP2State { @@ -428,7 +436,6 @@ impl HTTP2State { dynamic_headers_tc: HTTP2DynTable::new(), transactions: VecDeque::new(), progress: HTTP2ConnectionState::Http2StateInit, - files: Files::default(), } } @@ -441,7 +448,7 @@ impl HTTP2State { None => panic!("BUG no suricata_config"), Some(c) => { (c.HTPFileCloseHandleRange)( - &mut self.files.files_tc, + &mut tx.files.files_tc, 0, tx.file_range, std::ptr::null_mut(), @@ -482,7 +489,7 @@ impl HTTP2State { None => panic!("BUG no suricata_config"), Some(c) => { (c.HTPFileCloseHandleRange)( - &mut self.files.files_tc, + &mut tx.files.files_tc, 0, tx.file_range, std::ptr::null_mut(), @@ -504,6 +511,8 @@ impl HTTP2State { pub fn get_tx(&mut self, tx_id: u64) -> Option<&HTTP2Transaction> { for tx in &mut self.transactions { if tx.tx_id == tx_id + 1 { + tx.tx_data.update_file_flags(self.state_data.file_flags); + tx.update_file_flags(tx.tx_data.file_flags); return Some(tx); } } @@ -541,6 +550,8 @@ impl HTTP2State { self.tx_id += 1; tx.tx_id = self.tx_id; tx.state = HTTP2TransactionState::HTTP2StateGlobal; + tx.tx_data.update_file_flags(self.state_data.file_flags); + tx.update_file_flags(tx.tx_data.file_flags); self.transactions.push_back(tx); return self.transactions.back_mut().unwrap(); } @@ -575,7 +586,11 @@ impl HTTP2State { self.set_event(HTTP2Event::StreamIdReuse); } } - return &mut self.transactions[index - 1]; + + let tx = &mut self.transactions[index - 1]; + tx.tx_data.update_file_flags(self.state_data.file_flags); + tx.update_file_flags(tx.tx_data.file_flags); + return tx; } else { let mut tx = HTTP2Transaction::new(); self.tx_id += 1; @@ -594,6 +609,8 @@ impl HTTP2State { } } } + tx.tx_data.update_file_flags(self.state_data.file_flags); + tx.update_file_flags(tx.tx_data.file_flags); self.transactions.push_back(tx); return self.transactions.back_mut().unwrap(); } @@ -934,15 +951,18 @@ impl HTTP2State { //borrow checker forbids to reuse directly tx let index = self.find_tx_index(sid); if index > 0 { - let tx_same = &mut self.transactions[index - 1]; - let (files, flags) = self.files.get(dir); + let mut tx_same = &mut self.transactions[index - 1]; + if dir == Direction::ToServer { + tx_same.ft_tc.tx_id = tx_same.tx_id - 1; + } else { + tx_same.ft_ts.tx_id = tx_same.tx_id - 1; + }; + match tx_same.decompress( &rem[..hlsafe], dir, sfcm, over, - files, - flags, flow, ) { Err(_e) => { @@ -1120,9 +1140,6 @@ pub unsafe extern "C" fn rs_http2_parse_ts( ) -> AppLayerResult { let state = cast_pointer!(state, HTTP2State); let buf = stream_slice.as_slice(); - - state.files.flags_ts = FileFlowToFlags(flow, Direction::ToServer.into()); - state.files.flags_ts = state.files.flags_ts | FILE_USE_DETECT; return state.parse_ts(buf, flow); } @@ -1133,8 +1150,6 @@ pub unsafe extern "C" fn rs_http2_parse_tc( ) -> AppLayerResult { let state = cast_pointer!(state, HTTP2State); let buf = stream_slice.as_slice(); - state.files.flags_tc = FileFlowToFlags(flow, Direction::ToClient.into()); - state.files.flags_tc = state.files.flags_tc | FILE_USE_DETECT; return state.parse_tc(buf, flow); } @@ -1176,13 +1191,13 @@ pub unsafe extern "C" fn rs_http2_tx_get_alstate_progress( #[no_mangle] pub unsafe extern "C" fn rs_http2_getfiles( - state: *mut std::os::raw::c_void, direction: u8, + tx: *mut std::os::raw::c_void, direction: u8, ) -> *mut FileContainer { - let state = cast_pointer!(state, HTTP2State); + let tx = cast_pointer!(tx, HTTP2Transaction); if direction == Direction::ToClient.into() { - &mut state.files.files_tc as *mut FileContainer + &mut tx.files.files_tc as *mut FileContainer } else { - &mut state.files.files_ts as *mut FileContainer + &mut tx.files.files_ts as *mut FileContainer } } @@ -1214,7 +1229,7 @@ pub unsafe extern "C" fn rs_http2_register_parser() { get_eventinfo_byid: Some(HTTP2Event::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: Some(rs_http2_getfiles), + get_tx_files: Some(rs_http2_getfiles), get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_http2_get_tx_data, get_state_data: rs_http2_get_state_data, diff --git a/rust/src/http2/range.rs b/rust/src/http2/range.rs index 5d42c12f79..97d805996b 100644 --- a/rust/src/http2/range.rs +++ b/rust/src/http2/range.rs @@ -19,7 +19,6 @@ use super::detect; use crate::core::{ Direction, Flow, HttpRangeContainerBlock, StreamingBufferConfig, SuricataFileContext, SC, }; -use crate::filecontainer::FileContainer; use crate::http2::http2::HTTP2Transaction; use nom7::branch::alt; @@ -130,7 +129,7 @@ fn http2_range_key_get(tx: &mut HTTP2Transaction) -> Result<(Vec, usize), () pub fn http2_range_open( tx: &mut HTTP2Transaction, v: &HTTPContentRange, flow: *const Flow, - cfg: &'static SuricataFileContext, flags: u16, data: &[u8], + cfg: &'static SuricataFileContext, dir: Direction, data: &[u8], ) { if v.end <= 0 || v.size <= 0 { // skipped for incomplete range information @@ -139,6 +138,7 @@ pub fn http2_range_open( // whole file in one range return; } + let (_, flags) = tx.files.get(dir); if let Ok((key, index)) = http2_range_key_get(tx) { let name = &key[index..]; tx.file_range = unsafe { @@ -165,9 +165,10 @@ pub fn http2_range_append(fr: *mut HttpRangeContainerBlock, data: &[u8]) { } pub fn http2_range_close( - tx: &mut HTTP2Transaction, files: &mut FileContainer, flags: u16, data: &[u8], + tx: &mut HTTP2Transaction, dir: Direction, data: &[u8], ) { let added = if let Some(c) = unsafe { SC } { + let (files, flags) = tx.files.get(dir); let added = (c.HTPFileCloseHandleRange)( files, flags, diff --git a/rust/src/ike/ike.rs b/rust/src/ike/ike.rs index 16f9866dcb..f476653654 100644 --- a/rust/src/ike/ike.rs +++ b/rust/src/ike/ike.rs @@ -422,7 +422,7 @@ pub unsafe extern "C" fn rs_ike_register_parser() { get_eventinfo_byid: Some(IkeEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_ike_get_tx_data, get_state_data: rs_ike_get_state_data, diff --git a/rust/src/ike/ikev2.rs b/rust/src/ike/ikev2.rs index 054f10e314..5bd637a611 100644 --- a/rust/src/ike/ikev2.rs +++ b/rust/src/ike/ikev2.rs @@ -202,7 +202,7 @@ fn add_proposals( | IkeTransformEncType::ENCR_DES_IV32 | IkeTransformEncType::ENCR_NULL => { SCLogDebug!("Weak Encryption: {:?}", enc); - // XXX send event only if direction == STREAM_TOCLIENT ? + // XXX send event only if direction == Direction::ToClient ? tx.set_event(IkeEvent::WeakCryptoEnc); } _ => (), diff --git a/rust/src/krb/krb5.rs b/rust/src/krb/krb5.rs index 293db1f864..e8bb499349 100644 --- a/rust/src/krb/krb5.rs +++ b/rust/src/krb/krb5.rs @@ -557,7 +557,7 @@ pub unsafe extern "C" fn rs_register_krb5_parser() { get_eventinfo_byid : Some(KRB5Event::get_event_info_by_id), localstorage_new : None, localstorage_free : None, - get_files : None, + get_tx_files : None, get_tx_iterator : Some(applayer::state_get_tx_iterator::), get_tx_data : rs_krb5_get_tx_data, get_state_data : rs_krb5_get_state_data, diff --git a/rust/src/modbus/modbus.rs b/rust/src/modbus/modbus.rs index 164d8d1a55..3dd6a56017 100644 --- a/rust/src/modbus/modbus.rs +++ b/rust/src/modbus/modbus.rs @@ -407,7 +407,7 @@ pub unsafe extern "C" fn rs_modbus_register_parser() { get_eventinfo_byid: Some(ModbusEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_modbus_state_get_tx_data, get_state_data: rs_modbus_get_state_data, diff --git a/rust/src/mqtt/mqtt.rs b/rust/src/mqtt/mqtt.rs index 2c310a9517..5cf624ccd4 100644 --- a/rust/src/mqtt/mqtt.rs +++ b/rust/src/mqtt/mqtt.rs @@ -753,7 +753,7 @@ pub unsafe extern "C" fn rs_mqtt_register_parser(cfg_max_msg_len: u32) { get_eventinfo_byid: Some(MQTTEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::), get_tx_data: rs_mqtt_get_tx_data, get_state_data: rs_mqtt_get_state_data, diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs index bf6ec90a27..7e48897abe 100644 --- a/rust/src/nfs/nfs.rs +++ b/rust/src/nfs/nfs.rs @@ -139,6 +139,9 @@ pub struct NFSTransactionFile { /// file tracker for a single file. Boxed so that we don't use /// as much space if we're not a file tx. pub file_tracker: FileTransferTracker, + + /// storage for the actual file + pub files: Files, } impl NFSTransactionFile { @@ -148,6 +151,21 @@ impl NFSTransactionFile { ..Default::default() } } + pub fn update_file_flags(&mut self, flow_file_flags: u16) { + self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT }; + self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT }; + } +} + +#[no_mangle] +pub unsafe extern "C" fn rs_nfs_gettxfiles(tx_ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer { + let tx = cast_pointer!(tx_ptr, NFSTransaction); + if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, _flags) = tdf.files.get(direction.into()); + files + } else { + std::ptr::null_mut() + } } #[derive(Debug)] @@ -292,8 +310,6 @@ pub struct NFSState { /// transactions list pub transactions: Vec, - pub files: Files, - /// partial record tracking pub ts_chunk_xid: u32, pub tc_chunk_xid: u32, @@ -344,7 +360,6 @@ impl NFSState { requestmap:HashMap::new(), namemap:HashMap::new(), transactions: Vec::new(), - files:Files::default(), ts_chunk_xid:0, tc_chunk_xid:0, ts_chunk_left:0, @@ -565,7 +580,7 @@ impl NFSState { if self.ts > f.post_gap_ts { tx.request_done = true; tx.response_done = true; - let (files, flags) = self.files.get(tx.file_tx_direction); + let (files, flags) = f.files.get(tx.file_tx_direction); f.file_tracker.trunc(files, flags); } else { post_gap_txs = true; @@ -671,7 +686,7 @@ impl NFSState { } pub fn new_file_tx(&mut self, file_handle: &Vec, file_name: &Vec, direction: Direction) - -> (&mut NFSTransaction, &mut FileContainer, u16) + -> &mut NFSTransaction { let mut tx = self.new_tx(); tx.file_name = file_name.to_vec(); @@ -682,28 +697,32 @@ impl NFSState { tx.type_data = Some(NFSTransactionTypeData::FILE(NFSTransactionFile::new())); if let Some(NFSTransactionTypeData::FILE(ref mut d)) = tx.type_data { d.file_tracker.tx_id = tx.id - 1; + tx.tx_data.update_file_flags(self.state_data.file_flags); + d.update_file_flags(tx.tx_data.file_flags); } tx.tx_data.init_files_opened(); SCLogDebug!("new_file_tx: TX FILE created: ID {} NAME {}", tx.id, String::from_utf8_lossy(file_name)); self.transactions.push(tx); let tx_ref = self.transactions.last_mut(); - let (files, flags) = self.files.get(direction); - return (tx_ref.unwrap(), files, flags) + return tx_ref.unwrap(); } pub fn get_file_tx_by_handle(&mut self, file_handle: &Vec, direction: Direction) - -> Option<(&mut NFSTransaction, &mut FileContainer, u16)> + -> Option<&mut NFSTransaction> { let fh = file_handle.to_vec(); for tx in &mut self.transactions { - if tx.is_file_tx && !tx.is_file_closed && - direction == tx.file_tx_direction && - tx.file_handle == fh - { - SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid); - let (files, flags) = self.files.get(direction); - return Some((tx, files, flags)); + if let Some(NFSTransactionTypeData::FILE(ref mut d)) = tx.type_data { + if tx.is_file_tx && !tx.is_file_closed && + direction == tx.file_tx_direction && + tx.file_handle == fh + { + tx.tx_data.update_file_flags(self.state_data.file_flags); + d.update_file_flags(tx.tx_data.file_flags); + SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid); + return Some(tx); + } } } SCLogDebug!("Failed to find NFS TX with handle {:?}", file_handle); @@ -734,8 +753,9 @@ impl NFSState { }; let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToServer) { - Some((tx, files, flags)) => { + Some(tx) => { if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToServer); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, w.file_data, w.offset, w.file_len, fill_bytes as u8, is_last, &r.hdr.xid); @@ -754,8 +774,9 @@ impl NFSState { None => { false }, }; if !found { - let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToServer); + let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToServer); if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToServer); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, w.file_data, w.offset, w.file_len, fill_bytes as u8, is_last, &r.hdr.xid); @@ -907,8 +928,9 @@ impl NFSState { let ssn_gap = self.ts_ssn_gap | self.tc_ssn_gap; // get the tx and update it let consumed = match self.get_file_tx_by_handle(&file_handle, direction) { - Some((tx, files, flags)) => { + Some(tx) => { if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(direction); if ssn_gap { let queued_data = tdf.file_tracker.get_queued_size(); if queued_data > 2000000 { // TODO should probably be configurable @@ -1009,9 +1031,10 @@ impl NFSState { SCLogDebug!("partial data? {}", is_partial); let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToClient) { - Some((tx, files, flags)) => { + Some(tx) => { SCLogDebug!("updated TX {:?}", tx); if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToClient); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, reply.data, chunk_offset, reply.count, fill_bytes as u8, is_last, &r.hdr.xid); @@ -1038,8 +1061,9 @@ impl NFSState { None => { false }, }; if !found { - let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToClient); + let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToClient); if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToClient); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, reply.data, chunk_offset, reply.count, fill_bytes as u8, is_last, &r.hdr.xid); @@ -1497,23 +1521,6 @@ impl NFSState { } AppLayerResult::ok() } - - fn getfiles(&mut self, direction: Direction) -> * mut FileContainer { - //SCLogDebug!("direction: {:?}", direction); - if direction == Direction::ToClient { - &mut self.files.files_tc as *mut FileContainer - } else { - &mut self.files.files_ts as *mut FileContainer - } - } - fn setfileflags(&mut self, direction: Direction, flags: u16) { - SCLogDebug!("direction: {:?}, flags: {}", direction, flags); - if direction == Direction::ToClient { - self.files.flags_tc = flags; - } else { - self.files.flags_ts = flags; - } - } } /// Returns *mut NFSState @@ -1545,8 +1552,6 @@ pub unsafe extern "C" fn rs_nfs_parse_request(flow: *const Flow, { let state = cast_pointer!(state, NFSState); let flow = cast_pointer!(flow, Flow); - let file_flags = FileFlowToFlags(flow, Direction::ToServer.into()); - rs_nfs_setfileflags(Direction::ToServer.into(), state, file_flags); if stream_slice.is_gap() { return rs_nfs_parse_request_tcp_gap(state, stream_slice.gap_size()); @@ -1576,8 +1581,6 @@ pub unsafe extern "C" fn rs_nfs_parse_response(flow: *const Flow, { let state = cast_pointer!(state, NFSState); let flow = cast_pointer!(flow, Flow); - let file_flags = FileFlowToFlags(flow, Direction::ToClient.into()); - rs_nfs_setfileflags(Direction::ToClient.into(), state, file_flags); if stream_slice.is_gap() { return rs_nfs_parse_response_tcp_gap(state, stream_slice.gap_size()); @@ -1607,8 +1610,6 @@ pub unsafe extern "C" fn rs_nfs_parse_request_udp(f: *const Flow, ) -> AppLayerResult { let state = cast_pointer!(state, NFSState); - let file_flags = FileFlowToFlags(f, Direction::ToServer.into()); - rs_nfs_setfileflags(Direction::ToServer.into(), state, file_flags); SCLogDebug!("parsing {} bytes of request data", stream_slice.len()); state.parse_udp_ts(f, &stream_slice) @@ -1623,8 +1624,6 @@ pub unsafe extern "C" fn rs_nfs_parse_response_udp(f: *const Flow, ) -> AppLayerResult { let state = cast_pointer!(state, NFSState); - let file_flags = FileFlowToFlags(f, Direction::ToClient.into()); - rs_nfs_setfileflags(Direction::ToClient.into(), state, file_flags); SCLogDebug!("parsing {} bytes of response data", stream_slice.len()); state.parse_udp_tc(f, &stream_slice) } @@ -1934,20 +1933,6 @@ pub unsafe extern "C" fn rs_nfs_probe_udp_tc(_f: *const Flow, } } -#[no_mangle] -pub unsafe extern "C" fn rs_nfs_getfiles(ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer { - if ptr.is_null() { panic!("NULL ptr"); }; - let parser = cast_pointer!(ptr, NFSState); - parser.getfiles(direction.into()) -} -#[no_mangle] -pub unsafe extern "C" fn rs_nfs_setfileflags(direction: u8, ptr: *mut NFSState, flags: u16) { - if ptr.is_null() { panic!("NULL ptr"); }; - let parser = &mut *ptr; - SCLogDebug!("direction {} flags {}", direction, flags); - parser.setfileflags(direction.into(), flags) -} - // Parser name as a C style string. const PARSER_NAME: &'static [u8] = b"nfs\0"; @@ -1976,7 +1961,7 @@ pub unsafe extern "C" fn rs_nfs_register_parser() { get_eventinfo_byid : Some(NFSEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: Some(rs_nfs_getfiles), + get_tx_files: Some(rs_nfs_gettxfiles), get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_nfs_get_tx_data, get_state_data: rs_nfs_get_state_data, @@ -2055,7 +2040,7 @@ pub unsafe extern "C" fn rs_nfs_udp_register_parser() { get_eventinfo_byid : Some(NFSEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: Some(rs_nfs_getfiles), + get_tx_files: Some(rs_nfs_gettxfiles), get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_nfs_get_tx_data, get_state_data: rs_nfs_get_state_data, @@ -2106,4 +2091,3 @@ pub unsafe extern "C" fn rs_nfs_udp_register_parser() { SCLogDebug!("Protocol detector and parser disabled for nfs."); } } - diff --git a/rust/src/nfs/nfs3.rs b/rust/src/nfs/nfs3.rs index e29a1f109c..752ed7cde7 100644 --- a/rust/src/nfs/nfs3.rs +++ b/rust/src/nfs/nfs3.rs @@ -118,8 +118,9 @@ impl NFSState { SCLogDebug!("COMMIT, closing shop"); if let Ok((_, rd)) = parse_nfs3_request_commit(r.prog_data) { let file_handle = rd.handle.value.to_vec(); - if let Some((tx, files, flags)) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) { + if let Some(tx) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) { if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToServer); tdf.chunk_count += 1; tdf.file_additional_procs.push(NFSPROC3_COMMIT); tdf.file_tracker.close(files, flags); @@ -166,11 +167,11 @@ impl NFSState { } else if r.procedure == NFSPROC3_READ { let found = match self.get_file_tx_by_handle(&xidmap.file_handle, Direction::ToClient) { - Some((_, _, _)) => true, + Some(_) => true, None => false, }; if !found { - let (tx, _, _) = self.new_file_tx(&xidmap.file_handle, &xidmap.file_name, Direction::ToClient); + let tx = self.new_file_tx(&xidmap.file_handle, &xidmap.file_name, Direction::ToClient); tx.procedure = NFSPROC3_READ; tx.xid = r.hdr.xid; tx.is_first = true; diff --git a/rust/src/nfs/nfs4.rs b/rust/src/nfs/nfs4.rs index 7c4c48127e..5c3df56364 100644 --- a/rust/src/nfs/nfs4.rs +++ b/rust/src/nfs/nfs4.rs @@ -66,20 +66,12 @@ impl NFSState { }; let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToServer) { - Some((tx, files, flags)) => { + Some(tx) => { if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { - filetracker_newchunk( - &mut tdf.file_tracker, - files, - flags, - &file_name, - w.data, - w.offset, - w.write_len, - fill_bytes as u8, - is_last, - &r.hdr.xid, - ); + let (files, flags) = tdf.files.get(Direction::ToServer); + filetracker_newchunk(&mut tdf.file_tracker, files, flags, + &file_name, w.data, w.offset, + w.write_len, fill_bytes as u8, is_last, &r.hdr.xid); tdf.chunk_count += 1; if is_last { tdf.file_last_xid = r.hdr.xid; @@ -92,20 +84,12 @@ impl NFSState { None => false, }; if !found { - let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToServer); + let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToServer); if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { - filetracker_newchunk( - &mut tdf.file_tracker, - files, - flags, - &file_name, - w.data, - w.offset, - w.write_len, - fill_bytes as u8, - is_last, - &r.hdr.xid, - ); + let (files, flags) = tdf.files.get(Direction::ToServer); + filetracker_newchunk(&mut tdf.file_tracker, files, flags, + &file_name, w.data, w.offset, + w.write_len, fill_bytes as u8, is_last, &r.hdr.xid); tx.procedure = NFSPROC4_WRITE; tx.xid = r.hdr.xid; tx.is_first = true; @@ -131,9 +115,9 @@ impl NFSState { SCLogDebug!("COMMIT, closing shop"); let file_handle = fh.to_vec(); - if let Some((tx, files, flags)) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) - { + if let Some(tx) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) { if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToServer); tdf.file_tracker.close(files, flags); tdf.file_last_xid = r.hdr.xid; tx.is_last = true; diff --git a/rust/src/ntp/ntp.rs b/rust/src/ntp/ntp.rs index aad352e968..257c9e7c37 100644 --- a/rust/src/ntp/ntp.rs +++ b/rust/src/ntp/ntp.rs @@ -295,7 +295,7 @@ pub unsafe extern "C" fn rs_register_ntp_parser() { get_eventinfo_byid : Some(NTPEvent::get_event_info_by_id), localstorage_new : None, localstorage_free : None, - get_files : None, + get_tx_files : None, get_tx_iterator : Some(applayer::state_get_tx_iterator::), get_tx_data : rs_ntp_get_tx_data, get_state_data : rs_ntp_get_state_data, diff --git a/rust/src/pgsql/pgsql.rs b/rust/src/pgsql/pgsql.rs index 56c09b9c29..bfd47d1b77 100644 --- a/rust/src/pgsql/pgsql.rs +++ b/rust/src/pgsql/pgsql.rs @@ -729,7 +729,7 @@ pub unsafe extern "C" fn rs_pgsql_register_parser() { get_eventinfo_byid: None, localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some( crate::applayer::state_get_tx_iterator::, ), diff --git a/rust/src/quic/quic.rs b/rust/src/quic/quic.rs index 6010b04e37..37c14ad9ad 100644 --- a/rust/src/quic/quic.rs +++ b/rust/src/quic/quic.rs @@ -473,7 +473,7 @@ pub unsafe extern "C" fn rs_quic_register_parser() { get_eventinfo_byid: Some(QuicEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(rs_quic_state_get_tx_iterator), get_tx_data: rs_quic_get_tx_data, get_state_data: rs_quic_get_state_data, diff --git a/rust/src/rdp/rdp.rs b/rust/src/rdp/rdp.rs index e911712038..eff75e5e55 100644 --- a/rust/src/rdp/rdp.rs +++ b/rust/src/rdp/rdp.rs @@ -489,7 +489,7 @@ pub unsafe extern "C" fn rs_rdp_register_parser() { get_eventinfo_byid: None, localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_rdp_get_tx_data, get_state_data: rs_rdp_get_state_data, diff --git a/rust/src/rfb/rfb.rs b/rust/src/rfb/rfb.rs index ea858f4068..60c0e4adb6 100644 --- a/rust/src/rfb/rfb.rs +++ b/rust/src/rfb/rfb.rs @@ -598,7 +598,7 @@ pub unsafe extern "C" fn rs_rfb_register_parser() { get_eventinfo_byid: None, localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_rfb_get_tx_data, get_state_data: rs_rfb_get_state_data, diff --git a/rust/src/sip/sip.rs b/rust/src/sip/sip.rs index d59098cf23..6d7dd5d04e 100755 --- a/rust/src/sip/sip.rs +++ b/rust/src/sip/sip.rs @@ -370,7 +370,7 @@ pub unsafe extern "C" fn rs_sip_register_parser() { get_eventinfo_byid: Some(SIPEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_sip_get_tx_data, get_state_data: rs_sip_get_state_data, diff --git a/rust/src/smb/files.rs b/rust/src/smb/files.rs index cf54ab03aa..d7d6625217 100644 --- a/rust/src/smb/files.rs +++ b/rust/src/smb/files.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2018 Open Information Security Foundation +/* Copyright (C) 2018-2022 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -33,6 +33,7 @@ pub struct SMBTransactionFile { /// after a gap, this will be set to a time in the future. If the file /// receives no updates before that, it will be considered complete. pub post_gap_ts: u64, + pub files: Files, } impl SMBTransactionFile { @@ -42,6 +43,11 @@ impl SMBTransactionFile { ..Default::default() } } + + pub fn update_file_flags(&mut self, flow_file_flags: u16) { + self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT }; + self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT }; + } } /// little wrapper around the FileTransferTracker::new_chunk method @@ -59,7 +65,7 @@ pub fn filetracker_newchunk(ft: &mut FileTransferTracker, files: &mut FileContai impl SMBState { pub fn new_file_tx(&mut self, fuid: &Vec, file_name: &Vec, direction: Direction) - -> (&mut SMBTransaction, &mut FileContainer, u16) + -> &mut SMBTransaction { let mut tx = self.new_tx(); tx.type_data = Some(SMBTransactionTypeData::FILE(SMBTransactionFile::new())); @@ -69,6 +75,8 @@ impl SMBState { d.fuid = fuid.to_vec(); d.file_name = file_name.to_vec(); d.file_tracker.tx_id = tx.id - 1; + tx.tx_data.update_file_flags(self.state_data.file_flags); + d.update_file_flags(tx.tx_data.file_flags); }, _ => { }, } @@ -77,12 +85,11 @@ impl SMBState { tx.id, String::from_utf8_lossy(file_name)); self.transactions.push(tx); let tx_ref = self.transactions.last_mut(); - let (files, flags) = self.files.get(direction); - return (tx_ref.unwrap(), files, flags) + return tx_ref.unwrap(); } pub fn get_file_tx_by_fuid(&mut self, fuid: &Vec, direction: Direction) - -> Option<(&mut SMBTransaction, &mut FileContainer, u16)> + -> Option<&mut SMBTransaction> { let f = fuid.to_vec(); for tx in &mut self.transactions { @@ -95,31 +102,17 @@ impl SMBState { if found { SCLogDebug!("SMB: Found SMB file TX with ID {}", tx.id); - let (files, flags) = self.files.get(direction); - return Some((tx, files, flags)); + if let Some(SMBTransactionTypeData::FILE(ref mut d)) = tx.type_data { + tx.tx_data.update_file_flags(self.state_data.file_flags); + d.update_file_flags(tx.tx_data.file_flags); + } + return Some(tx); } } SCLogDebug!("SMB: Failed to find SMB TX with FUID {:?}", fuid); return None; } - fn getfiles(&mut self, direction: Direction) -> * mut FileContainer { - //SCLogDebug!("direction: {:?}", direction); - if direction == Direction::ToClient { - &mut self.files.files_tc as *mut FileContainer - } else { - &mut self.files.files_ts as *mut FileContainer - } - } - fn setfileflags(&mut self, direction: Direction, flags: u16) { - SCLogDebug!("direction: {:?}, flags: {}", direction, flags); - if direction == Direction::ToClient { - self.files.flags_tc = flags; - } else { - self.files.flags_ts = flags; - } - } - // update in progress chunks for file transfers // return how much data we consumed pub fn filetracker_update(&mut self, direction: Direction, data: &[u8], gap_size: u32) -> u32 { @@ -159,8 +152,9 @@ impl SMBState { let ssn_gap = self.ts_ssn_gap | self.tc_ssn_gap; // get the tx and update it let consumed = match self.get_file_tx_by_fuid(&file_handle, direction) { - Some((tx, files, flags)) => { + Some(tx) => { if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(direction); if ssn_gap { let queued_data = tdf.file_tracker.get_queued_size(); if queued_data > 2000000 { // TODO should probably be configurable @@ -191,16 +185,12 @@ impl SMBState { } #[no_mangle] -pub unsafe extern "C" fn rs_smb_getfiles(ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer { - if ptr.is_null() { panic!("NULL ptr"); }; - let parser = cast_pointer!(ptr, SMBState); - parser.getfiles(direction.into()) -} - -#[no_mangle] -pub unsafe extern "C" fn rs_smb_setfileflags(direction: u8, ptr: *mut SMBState, flags: u16) { - if ptr.is_null() { panic!("NULL ptr"); }; - let parser = &mut *ptr; - SCLogDebug!("direction {} flags {}", direction, flags); - parser.setfileflags(direction.into(), flags) +pub unsafe extern "C" fn rs_smb_gettxfiles(tx_ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer { + let tx = cast_pointer!(tx_ptr, SMBTransaction); + if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, _flags) = tdf.files.get(direction.into()); + files + } else { + std::ptr::null_mut() + } } diff --git a/rust/src/smb/smb.rs b/rust/src/smb/smb.rs index 2b429037fe..3c6d657d77 100644 --- a/rust/src/smb/smb.rs +++ b/rust/src/smb/smb.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2017-2020 Open Information Security Foundation +/* Copyright (C) 2017-2022 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -39,7 +39,6 @@ use crate::applayer; use crate::applayer::*; use crate::frames::*; use crate::conf::*; -use crate::filecontainer::*; use crate::applayer::{AppLayerResult, AppLayerTxData, AppLayerEvent}; use crate::smb::nbss_records::*; @@ -593,6 +592,7 @@ impl SMBTransaction { } pub fn free(&mut self) { + SCLogDebug!("SMB TX {:p} free ID {}", &self, self.id); debug_validate_bug_on!(self.tx_data.files_opened > 1); debug_validate_bug_on!(self.tx_data.files_logged > 1); } @@ -767,8 +767,6 @@ pub struct SMBState<> { // requests for DCERPC. pub ssnguid2vec_map: HashMap>, - pub files: Files, - skip_ts: u32, skip_tc: u32, @@ -834,7 +832,6 @@ impl SMBState { ssn2vecoffset_map:HashMap::new(), ssn2tree_map:HashMap::new(), ssnguid2vec_map:HashMap::new(), - files: Files::default(), skip_ts:0, skip_tc:0, file_ts_left:0, @@ -914,6 +911,11 @@ impl SMBState { } SCLogDebug!("Found SMB TX: id {} ver:{} cmd:{} progress {}/{} type_data {:?}", tx.id, ver, _smbcmd, tx.request_done, tx.response_done, tx.type_data); + /* hack: apply flow file flags to file tx here to make sure its propegated */ + if let Some(SMBTransactionTypeData::FILE(ref mut d)) = tx.type_data { + tx.tx_data.update_file_flags(self.state_data.file_flags); + d.update_file_flags(tx.tx_data.file_flags); + } return Some(tx); } } @@ -1160,7 +1162,7 @@ impl SMBState { if self.ts > f.post_gap_ts { tx.request_done = true; tx.response_done = true; - let (files, flags) = self.files.get(f.direction); + let (files, flags) = f.files.get(f.direction); f.file_tracker.trunc(files, flags); } else { post_gap_txs = true; @@ -2034,8 +2036,6 @@ pub unsafe extern "C" fn rs_smb_parse_request_tcp(flow: *const Flow, { let mut state = cast_pointer!(state, SMBState); let flow = cast_pointer!(flow, Flow); - let file_flags = FileFlowToFlags(flow, Direction::ToServer as u8); - rs_smb_setfileflags(Direction::ToServer as u8, state, file_flags|FILE_USE_DETECT); if stream_slice.is_gap() { return rs_smb_parse_request_tcp_gap(state, stream_slice.gap_size()); @@ -2073,8 +2073,6 @@ pub unsafe extern "C" fn rs_smb_parse_response_tcp(flow: *const Flow, { let mut state = cast_pointer!(state, SMBState); let flow = cast_pointer!(flow, Flow); - let file_flags = FileFlowToFlags(flow, Direction::ToClient as u8); - rs_smb_setfileflags(Direction::ToClient as u8, state, file_flags|FILE_USE_DETECT); if stream_slice.is_gap() { return rs_smb_parse_response_tcp_gap(state, stream_slice.gap_size()); @@ -2289,7 +2287,7 @@ pub unsafe extern "C" fn rs_smb_tx_get_alstate_progress(tx: *mut ffi::c_void, SCLogDebug!("tx {} TOCLIENT progress 1 => {:?}", tx.id, tx); return 1; } else { - SCLogDebug!("tx {} direction {:?} progress 0", tx.id, direction); + SCLogDebug!("tx {} direction {} progress 0 => {:?}", tx.id, direction, tx); return 0; } } @@ -2426,7 +2424,7 @@ pub unsafe extern "C" fn rs_smb_register_parser() { get_eventinfo_byid : Some(rs_smb_state_get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: Some(rs_smb_getfiles), + get_tx_files: Some(rs_smb_gettxfiles), get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_smb_get_tx_data, get_state_data: rs_smb_get_state_data, diff --git a/rust/src/smb/smb1.rs b/rust/src/smb/smb1.rs index ad8f53cf6f..3a92786de8 100644 --- a/rust/src/smb/smb1.rs +++ b/rust/src/smb/smb1.rs @@ -146,11 +146,12 @@ pub fn smb1_check_tx(cmd: u8) -> bool { fn smb1_close_file(state: &mut SMBState, fid: &Vec, direction: Direction) { - if let Some((tx, files, flags)) = state.get_file_tx_by_fuid(fid, direction) { + if let Some(tx) = state.get_file_tx_by_fuid(fid, direction) { SCLogDebug!("found tx {}", tx.id); if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { if !tx.request_done { SCLogDebug!("closing file tx {} FID {:?}", tx.id, fid); + let (files, flags) = tdf.files.get(direction); tdf.file_tracker.close(files, flags); tx.request_done = true; tx.response_done = true; @@ -961,16 +962,17 @@ pub fn smb1_write_request_record<'b>(state: &mut SMBState, r: &SmbRecord<'b>, an }; let mut set_event_fileoverlap = false; let found = match state.get_file_tx_by_fuid(&file_fid, Direction::ToServer) { - Some((tx, files, flags)) => { + Some(tx) => { let file_id : u32 = tx.id as u32; if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { if rd.offset < tdf.file_tracker.tracked { set_event_fileoverlap = true; } + let (files, flags) = tdf.files.get(Direction::ToServer); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, rd.data, rd.offset, rd.len, false, &file_id); - SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id); + SCLogDebug!("FID {:?} found at tx {} => {:?}", file_fid, tx.id, tx); } true }, @@ -988,19 +990,22 @@ pub fn smb1_write_request_record<'b>(state: &mut SMBState, r: &SmbRecord<'b>, an let vercmd = SMBVerCmdStat::new1_with_ntstatus(command, r.nt_status); smb_write_dcerpc_record(state, vercmd, hdr, rd.data); } else { - let (tx, files, flags) = state.new_file_tx(&file_fid, &file_name, Direction::ToServer); + let tx = state.new_file_tx(&file_fid, &file_name, Direction::ToServer); if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { let file_id : u32 = tx.id as u32; - SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id); if rd.offset < tdf.file_tracker.tracked { set_event_fileoverlap = true; } + let (files, flags) = tdf.files.get(Direction::ToServer); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, rd.data, rd.offset, rd.len, false, &file_id); tdf.share_name = share_name; + SCLogDebug!("files {:?}", files); + SCLogDebug!("tdf {:?}", tdf); } tx.vercmd.set_smb1_cmd(SMB1_COMMAND_WRITE_ANDX); + SCLogDebug!("FID {:?} found at tx {} => {:?}", file_fid, tx.id, tx); } } if set_event_fileoverlap { @@ -1054,13 +1059,14 @@ pub fn smb1_read_response_record<'b>(state: &mut SMBState, r: &SmbRecord<'b>, an }; let mut set_event_fileoverlap = false; let found = match state.get_file_tx_by_fuid(&file_fid, Direction::ToClient) { - Some((tx, files, flags)) => { + Some(tx) => { if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { let file_id : u32 = tx.id as u32; SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id); if offset < tdf.file_tracker.tracked { set_event_fileoverlap = true; } + let (files, flags) = tdf.files.get(Direction::ToClient); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, rd.data, offset, rd.len, false, &file_id); @@ -1070,13 +1076,14 @@ pub fn smb1_read_response_record<'b>(state: &mut SMBState, r: &SmbRecord<'b>, an None => { false }, }; if !found { - let (tx, files, flags) = state.new_file_tx(&file_fid, &file_name, Direction::ToClient); + let tx = state.new_file_tx(&file_fid, &file_name, Direction::ToClient); if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { let file_id : u32 = tx.id as u32; SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id); if offset < tdf.file_tracker.tracked { set_event_fileoverlap = true; } + let (files, flags) = tdf.files.get(Direction::ToClient); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, rd.data, offset, rd.len, false, &file_id); diff --git a/rust/src/smb/smb2.rs b/rust/src/smb/smb2.rs index dba06314b3..53be23e5b1 100644 --- a/rust/src/smb/smb2.rs +++ b/rust/src/smb/smb2.rs @@ -1,4 +1,4 @@ -/* Copyright (C) 2017 Open Information Security Foundation +/* Copyright (C) 2017-2022 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -157,7 +157,7 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) let mut set_event_fileoverlap = false; // look up existing tracker and if we have it update it let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToClient) { - Some((tx, files, flags)) => { + Some(tx) => { if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { let file_id : u32 = tx.id as u32; if offset < tdf.file_tracker.tracked { @@ -170,6 +170,7 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) state.set_event(SMBEvent::ReadQueueCntExceeded); state.set_skip(Direction::ToClient, rd.len, rd.data.len() as u32); } else { + let (files, flags) = tdf.files.get(Direction::ToClient); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &tdf.file_name, rd.data, offset, rd.len, false, &file_id); @@ -225,12 +226,11 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) Some(n) => { n.to_vec() } None => { b"".to_vec() } }; - let (tx, files, flags) = state.new_file_tx(&file_guid, &file_name, Direction::ToClient); + let tx = state.new_file_tx(&file_guid, &file_name, Direction::ToClient); tx.vercmd.set_smb2_cmd(SMB2_COMMAND_READ); tx.hdr = SMBCommonHdr::new(SMBHDR_TYPE_HEADER, r.session_id, r.tree_id, 0); // TODO move into new_file_tx - if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { tdf.share_name = share_name; let file_id : u32 = tx.id as u32; @@ -244,6 +244,7 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) state.set_event(SMBEvent::ReadQueueCntExceeded); state.set_skip(Direction::ToClient, rd.len, rd.data.len() as u32); } else { + let (files, flags) = tdf.files.get(Direction::ToClient); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, rd.data, offset, rd.len, false, &file_id); @@ -296,7 +297,7 @@ pub fn smb2_write_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) let mut set_event_fileoverlap = false; let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToServer) { - Some((tx, files, flags)) => { + Some(tx) => { if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { let file_id : u32 = tx.id as u32; if wr.wr_offset < tdf.file_tracker.tracked { @@ -309,6 +310,7 @@ pub fn smb2_write_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) state.set_event(SMBEvent::WriteQueueCntExceeded); state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32); } else { + let (files, flags) = tdf.files.get(Direction::ToServer); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, wr.data, wr.wr_offset, wr.wr_len, false, &file_id); @@ -360,7 +362,7 @@ pub fn smb2_write_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) SCLogDebug!("non-DCERPC pipe: skip rest of the record"); state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32); } else { - let (tx, files, flags) = state.new_file_tx(&file_guid, &file_name, Direction::ToServer); + let tx = state.new_file_tx(&file_guid, &file_name, Direction::ToServer); tx.vercmd.set_smb2_cmd(SMB2_COMMAND_WRITE); tx.hdr = SMBCommonHdr::new(SMBHDR_TYPE_HEADER, r.session_id, r.tree_id, 0); // TODO move into new_file_tx @@ -377,6 +379,7 @@ pub fn smb2_write_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) state.set_event(SMBEvent::WriteQueueCntExceeded); state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32); } else { + let (files, flags) = tdf.files.get(Direction::ToServer); filetracker_newchunk(&mut tdf.file_tracker, files, flags, &file_name, wr.data, wr.wr_offset, wr.wr_len, false, &file_id); @@ -587,9 +590,10 @@ pub fn smb2_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) match parse_smb2_request_close(r.data) { Ok((_, cd)) => { let found_ts = match state.get_file_tx_by_fuid(&cd.guid.to_vec(), Direction::ToServer) { - Some((tx, files, flags)) => { + Some(tx) => { if !tx.request_done { if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToServer); tdf.file_tracker.close(files, flags); } } @@ -601,9 +605,10 @@ pub fn smb2_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) None => { false }, }; let found_tc = match state.get_file_tx_by_fuid(&cd.guid.to_vec(), Direction::ToClient) { - Some((tx, files, flags)) => { + Some(tx) => { if !tx.request_done { if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToClient); tdf.file_tracker.close(files, flags); } } @@ -701,9 +706,10 @@ pub fn smb2_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>) }, }; let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToClient) { - Some((tx, files, flags)) => { + Some(tx) => { if !tx.request_done { if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data { + let (files, flags) = tdf.files.get(Direction::ToClient); tdf.file_tracker.close(files, flags); } } diff --git a/rust/src/snmp/snmp.rs b/rust/src/snmp/snmp.rs index dc2d1bf260..eeaa8e832f 100644 --- a/rust/src/snmp/snmp.rs +++ b/rust/src/snmp/snmp.rs @@ -399,7 +399,7 @@ pub unsafe extern "C" fn rs_register_snmp_parser() { get_eventinfo_byid : Some(SNMPEvent::get_event_info_by_id), localstorage_new : None, localstorage_free : None, - get_files : None, + get_tx_files : None, get_tx_iterator : Some(applayer::state_get_tx_iterator::), get_tx_data : rs_snmp_get_tx_data, get_state_data : rs_snmp_get_state_data, diff --git a/rust/src/ssh/ssh.rs b/rust/src/ssh/ssh.rs index f77f3ea16c..3dd02d0c84 100644 --- a/rust/src/ssh/ssh.rs +++ b/rust/src/ssh/ssh.rs @@ -463,7 +463,7 @@ pub unsafe extern "C" fn rs_ssh_register_parser() { get_eventinfo_byid: Some(SSHEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: None, get_tx_data: rs_ssh_get_tx_data, get_state_data: rs_ssh_get_state_data, diff --git a/rust/src/telnet/telnet.rs b/rust/src/telnet/telnet.rs index 95c3704b8d..6fcb6b00ce 100644 --- a/rust/src/telnet/telnet.rs +++ b/rust/src/telnet/telnet.rs @@ -550,7 +550,7 @@ pub unsafe extern "C" fn rs_telnet_register_parser() { get_eventinfo_byid : Some(TelnetEvent::get_event_info_by_id), localstorage_new: None, localstorage_free: None, - get_files: None, + get_tx_files: None, get_tx_iterator: Some(applayer::state_get_tx_iterator::), get_tx_data: rs_telnet_get_tx_data, get_state_data: rs_telnet_get_state_data, diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index 57e15a4238..e508258a24 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -1097,14 +1097,20 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state, { const uint8_t *input = StreamSliceGetData(&stream_slice); uint32_t input_len = StreamSliceGetDataLen(&stream_slice); - uint16_t flags = FileFlowToFlags(f, direction) | FILE_USE_DETECT; - int ret = 0; - const bool eof = (flags & STREAM_TOSERVER) + const bool eof = (direction & STREAM_TOSERVER) ? AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TS) != 0 : AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TC) != 0; + ftpdata_state->tx_data.file_flags |= ftpdata_state->state_data.file_flags; + /* we depend on detection engine for file pruning */ + const uint16_t flags = + FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction) | FILE_USE_DETECT; + int ret = 0; + SCLogDebug("FTP-DATA input_len %u flags %04x dir %d/%s EOF %s", input_len, flags, direction, (direction & STREAM_TOSERVER) ? "toserver" : "toclient", eof ? "true" : "false"); + + SCLogDebug("FTP-DATA flags %04x dir %d", flags, direction); if (input_len && ftpdata_state->files == NULL) { struct FtpTransferCmd *data = (struct FtpTransferCmd *)FlowGetStorageById(f, AppLayerExpectationGetFlowId()); @@ -1299,9 +1305,9 @@ static int FTPDataGetAlstateProgress(void *tx, uint8_t direction) return FTPDATA_STATE_FINISHED; } -static FileContainer *FTPDataStateGetFiles(void *state, uint8_t direction) +static FileContainer *FTPDataStateGetTxFiles(void *tx, uint8_t direction) { - FtpDataState *ftpdata_state = (FtpDataState *)state; + FtpDataState *ftpdata_state = (FtpDataState *)tx; if (direction != ftpdata_state->direction) SCReturnPtr(NULL, "FileContainer"); @@ -1428,7 +1434,7 @@ void RegisterFTPParsers(void) AppLayerParserRegisterParserAcceptableDataDirection(IPPROTO_TCP, ALPROTO_FTPDATA, STREAM_TOSERVER | STREAM_TOCLIENT); AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateTransactionFree); - AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateGetFiles); + AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateGetTxFiles); AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetTx); AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetTxData); diff --git a/src/app-layer-htp-file.c b/src/app-layer-htp-file.c index ab2249f358..03fe1c378b 100644 --- a/src/app-layer-htp-file.c +++ b/src/app-layer-htp-file.c @@ -21,7 +21,7 @@ * \author Victor Julien * * This file provides HTTP protocol file handling support for the engine - * using HTP library. + * using the HTP library. */ #include "suricata.h" @@ -87,55 +87,16 @@ int HTPFileOpen(HtpState *s, HtpTxUserData *tx, const uint8_t *filename, uint16_ SCLogDebug("data %p data_len %"PRIu32, data, data_len); - if (s == NULL) { - SCReturnInt(-1); - } - if (direction & STREAM_TOCLIENT) { - if (s->files_tc == NULL) { - s->files_tc = FileContainerAlloc(); - if (s->files_tc == NULL) { - retval = -1; - goto end; - } - } - - files = s->files_tc; - - flags = FileFlowToFlags(s->f, STREAM_TOCLIENT); - - if ((s->flags & HTP_FLAG_STORE_FILES_TS) || - ((s->flags & HTP_FLAG_STORE_FILES_TX_TS) && txid == s->store_tx_id)) { - flags |= FILE_STORE; - flags &= ~FILE_NOSTORE; - } else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TC)) { - flags |= FILE_NOSTORE; - } - + files = &tx->files_tc; + flags = FileFlowFlagsToFlags(tx->tx_data.file_flags, STREAM_TOCLIENT); sbcfg = &s->cfg->response.sbcfg; // we shall not open a new file if there is a current one - DEBUG_VALIDATE_BUG_ON(s->file_range != NULL); + DEBUG_VALIDATE_BUG_ON(tx->file_range != NULL); } else { - if (s->files_ts == NULL) { - s->files_ts = FileContainerAlloc(); - if (s->files_ts == NULL) { - retval = -1; - goto end; - } - } - - files = s->files_ts; - - flags = FileFlowToFlags(s->f, STREAM_TOSERVER); - if ((s->flags & HTP_FLAG_STORE_FILES_TC) || - ((s->flags & HTP_FLAG_STORE_FILES_TX_TC) && txid == s->store_tx_id)) { - flags |= FILE_STORE; - flags &= ~FILE_NOSTORE; - } else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TS)) { - flags |= FILE_NOSTORE; - } - + files = &tx->files_ts; + flags = FileFlowFlagsToFlags(tx->tx_data.file_flags, STREAM_TOSERVER); sbcfg = &s->cfg->request.sbcfg; } @@ -146,10 +107,8 @@ int HTPFileOpen(HtpState *s, HtpTxUserData *tx, const uint8_t *filename, uint16_ retval = -1; } - FileSetTx(files->tail, txid); tx->tx_data.files_opened++; -end: SCReturnInt(retval); } @@ -230,30 +189,13 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena return HTPFileOpen(s, txud, filename, filename_len, data, data_len, txid, STREAM_TOCLIENT); } flags = FileFlowToFlags(s->f, STREAM_TOCLIENT); - if ((s->flags & HTP_FLAG_STORE_FILES_TS) || - ((s->flags & HTP_FLAG_STORE_FILES_TX_TS) && txid == s->store_tx_id)) { - flags |= FILE_STORE; - flags &= ~FILE_NOSTORE; - } else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TC)) { - flags |= FILE_NOSTORE; - } - - FileContainer * files = s->files_tc; - if (files == NULL) { - s->files_tc = FileContainerAlloc(); - if (s->files_tc == NULL) { - // no need to fall back to classic open if we cannot allocate the file container - SCReturnInt(-1); - } - files = s->files_tc; - } + FileContainer *files = &txud->files_tc; // we open a file for this specific range if (FileOpenFileWithId(files, &s->cfg->response.sbcfg, s->file_track_id++, filename, filename_len, data, data_len, flags) != 0) { SCReturnInt(-1); } - FileSetTx(files->tail, txid); txud->tx_data.files_opened++; if (FileSetRange(files, crparsed.start, crparsed.end) < 0) { @@ -279,11 +221,11 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena // do not reassemble file without host info SCReturnInt(0); } - DEBUG_VALIDATE_BUG_ON(s->file_range); - s->file_range = HttpRangeContainerOpenFile(keyurl, keylen, s->f, &crparsed, + DEBUG_VALIDATE_BUG_ON(htud->file_range); + htud->file_range = HttpRangeContainerOpenFile(keyurl, keylen, s->f, &crparsed, &s->cfg->response.sbcfg, filename, filename_len, flags, data, data_len); SCFree(keyurl); - if (s->file_range == NULL) { + if (htud->file_range == NULL) { SCReturnInt(-1); } SCReturnInt(0); @@ -292,7 +234,7 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena /** * \brief Store a chunk of data in the flow * - * \param s http state + * \param tx HtpTxUserData * \param data data chunk (if any) * \param data_len length of the data portion * \param direction flow direction @@ -301,8 +243,7 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena * \retval -1 error * \retval -2 file doesn't need storing */ -int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len, - uint8_t direction) +int HTPFileStoreChunk(HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t direction) { SCEnter(); @@ -310,15 +251,12 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len, int result = 0; FileContainer *files = NULL; - if (s == NULL) { - SCReturnInt(-1); - } - if (direction & STREAM_TOCLIENT) { - files = s->files_tc; + files = &tx->files_tc; } else { - files = s->files_ts; + files = &tx->files_ts; } + SCLogDebug("files %p data %p data_len %" PRIu32, files, data, data_len); if (files == NULL) { SCLogDebug("no files in state"); @@ -326,8 +264,8 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len, goto end; } - if (s->file_range != NULL) { - if (HttpRangeAppendData(s->file_range, data, data_len) < 0) { + if (tx->file_range != NULL) { + if (HttpRangeAppendData(tx->file_range, data, data_len) < 0) { SCLogDebug("Failed to append data"); } } @@ -339,6 +277,7 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len, } else if (result == -2) { retval = -2; } + SCLogDebug("result %u", result); end: SCReturnInt(retval); @@ -377,7 +316,7 @@ bool HTPFileCloseHandleRange(FileContainer *files, const uint16_t flags, HttpRan /** * \brief Close the file in the flow * - * \param s http state + * \param tx HtpTxUserData * \param data data chunk if any * \param data_len length of the data portion * \param flags flags to indicate events @@ -390,24 +329,23 @@ bool HTPFileCloseHandleRange(FileContainer *files, const uint16_t flags, HttpRan * \retval -1 error * \retval -2 not storing files on this flow/tx */ -int HTPFileClose(HtpState *s, HtpTxUserData *htud, const uint8_t *data, uint32_t data_len, - uint8_t flags, uint8_t direction) +int HTPFileClose( + HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t direction) { SCEnter(); + SCLogDebug("flags %04x FILE_TRUNCATED %s", flags, (flags & FILE_TRUNCATED) ? "true" : "false"); + int retval = 0; int result = 0; FileContainer *files = NULL; - if (s == NULL) { - SCReturnInt(-1); - } - if (direction & STREAM_TOCLIENT) { - files = s->files_tc; + files = &tx->files_tc; } else { - files = s->files_ts; + files = &tx->files_ts; } + SCLogDebug("files %p data %p data_len %" PRIu32, files, data, data_len); if (files == NULL) { retval = -1; @@ -420,14 +358,15 @@ int HTPFileClose(HtpState *s, HtpTxUserData *htud, const uint8_t *data, uint32_t } else if (result == -2) { retval = -2; } + SCLogDebug("result %u", result); - if (s->file_range != NULL) { - bool added = HTPFileCloseHandleRange(files, flags, s->file_range, data, data_len); + if (tx->file_range != NULL) { + bool added = HTPFileCloseHandleRange(files, flags, tx->file_range, data, data_len); if (added) { - htud->tx_data.files_opened++; + tx->tx_data.files_opened++; } - HttpRangeFreeBlock(s->file_range); - s->file_range = NULL; + HttpRangeFreeBlock(tx->file_range); + tx->file_range = NULL; } end: @@ -556,9 +495,10 @@ static int HTPFileParserTest02(void) FAIL_IF_NULL(tx); FAIL_IF_NULL(tx->request_method); FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0); - FAIL_IF_NULL(http_state->files_ts); - FAIL_IF_NULL(http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); + FAIL_IF_NULL(tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED); AppLayerParserThreadCtxFree(alp_tctx); StreamTcpFreeConfig(true); @@ -649,11 +589,12 @@ static int HTPFileParserTest03(void) FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0); - FAIL_IF_NULL(http_state->files_ts); - FAIL_IF_NULL(http_state->files_ts->head); - FAIL_IF_NULL(http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED); - FAIL_IF(FileDataSize(http_state->files_ts->head) != 11); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); + FAIL_IF_NULL(tx_ud->files_ts.head); + FAIL_IF_NULL(tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED); + FAIL_IF(FileDataSize(tx_ud->files_ts.head) != 11); AppLayerParserThreadCtxFree(alp_tctx); StreamTcpFreeConfig(true); @@ -744,10 +685,11 @@ static int HTPFileParserTest04(void) FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0); - FAIL_IF_NULL(http_state->files_ts); - FAIL_IF_NULL(http_state->files_ts->head); - FAIL_IF_NULL(http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); + FAIL_IF_NULL(tx_ud->files_ts.head); + FAIL_IF_NULL(tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED); AppLayerParserThreadCtxFree(alp_tctx); StreamTcpFreeConfig(true); @@ -809,19 +751,20 @@ static int HTPFileParserTest05(void) FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0); - FAIL_IF_NULL(http_state->files_ts); - FAIL_IF_NULL(http_state->files_ts->head); - FAIL_IF_NULL(http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); + FAIL_IF_NULL(tx_ud->files_ts.head); + FAIL_IF_NULL(tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED); - FAIL_IF(http_state->files_ts->head == http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->head->next != http_state->files_ts->tail); + FAIL_IF(tx_ud->files_ts.head == tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.head->next != tx_ud->files_ts.tail); - FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->head->sb, - (uint8_t *)"filecontent", 11) != 1); + FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.head->sb, (uint8_t *)"filecontent", 11) != + 1); - FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb, - (uint8_t *)"FILECONTENT", 11) != 1); + FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) != + 1); AppLayerParserThreadCtxFree(alp_tctx); StreamTcpFreeConfig(true); UTHFreeFlow(f); @@ -883,19 +826,20 @@ static int HTPFileParserTest06(void) FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0); - FAIL_IF_NULL(http_state->files_ts); - FAIL_IF_NULL(http_state->files_ts->head); - FAIL_IF_NULL(http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); + FAIL_IF_NULL(tx_ud->files_ts.head); + FAIL_IF_NULL(tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED); - FAIL_IF(http_state->files_ts->head == http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->head->next != http_state->files_ts->tail); + FAIL_IF(tx_ud->files_ts.head == tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.head->next != tx_ud->files_ts.tail); - FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->head->sb, - (uint8_t *)"filecontent", 11) != 1); + FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.head->sb, (uint8_t *)"filecontent", 11) != + 1); - FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb, - (uint8_t *)"FILECONTENT", 11) != 1); + FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) != + 1); AppLayerParserThreadCtxFree(alp_tctx); StreamTcpFreeConfig(true); @@ -946,12 +890,14 @@ static int HTPFileParserTest07(void) FAIL_IF_NULL(tx->request_method); FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0); - FAIL_IF_NULL(http_state->files_ts); - FAIL_IF_NULL(http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); + FAIL_IF_NULL(tx_ud->files_ts.head); + FAIL_IF_NULL(tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED); - FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb, - (uint8_t *)"FILECONTENT", 11) != 1); + FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) != + 1); AppLayerParserThreadCtxFree(alp_tctx); StreamTcpFreeConfig(true); @@ -1267,12 +1213,14 @@ static int HTPFileParserTest11(void) FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0); - FAIL_IF_NULL(http_state->files_ts); - FAIL_IF_NULL(http_state->files_ts->tail); - FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); + FAIL_IF_NULL(tx_ud->files_ts.head); + FAIL_IF_NULL(tx_ud->files_ts.tail); + FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED); - FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb, - (uint8_t *)"FILECONTENT", 11) != 1); + FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) != + 1); AppLayerParserThreadCtxFree(alp_tctx); StreamTcpFreeConfig(true); diff --git a/src/app-layer-htp-file.h b/src/app-layer-htp-file.h index 8e955c53ef..feaeda3b22 100644 --- a/src/app-layer-htp-file.h +++ b/src/app-layer-htp-file.h @@ -29,13 +29,15 @@ int HTPFileOpen(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *, uint32_t, uint64_t, uint8_t); -int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range); int HTPFileOpenWithRange(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *, uint32_t, uint64_t, bstr *rawvalue, HtpTxUserData *htud); bool HTPFileCloseHandleRange( FileContainer *, const uint16_t, HttpRangeContainerBlock *, const uint8_t *, uint32_t); -int HTPFileStoreChunk(HtpState *, const uint8_t *, uint32_t, uint8_t); -int HTPFileClose(HtpState *, HtpTxUserData *, const uint8_t *, uint32_t, uint8_t, uint8_t); +int HTPFileStoreChunk(HtpTxUserData *, const uint8_t *, uint32_t, uint8_t); + +int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range); +int HTPFileClose(HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t flags, + uint8_t direction); void HTPFileParserRegisterTests(void); diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 78c6abd04a..3ea59f5f7e 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -366,6 +366,12 @@ static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud) if (htud->tx_data.de_state != NULL) { DetectEngineStateFree(htud->tx_data.de_state); } + if (htud->file_range) { + HTPFileCloseHandleRange(&htud->files_tc, 0, htud->file_range, NULL, 0); + HttpRangeFreeBlock(htud->file_range); + } + FileContainerRecycle(&htud->files_ts); + FileContainerRecycle(&htud->files_tc); HTPFree(htud, sizeof(HtpTxUserData)); } } @@ -402,13 +408,6 @@ void HTPStateFree(void *state) htp_connp_destroy_all(s->connp); } - if (s->file_range) { - HTPFileCloseHandleRange(s->files_tc, 0, s->file_range, NULL, 0); - HttpRangeFreeBlock(s->file_range); - } - - FileContainerFree(s->files_ts); - FileContainerFree(s->files_tc); HTPFree(s, sizeof(HtpState)); #ifdef DEBUG @@ -516,10 +515,11 @@ void AppLayerHtpNeedFileInspection(void) SCReturn; } -static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags) +static void AppLayerHtpSetStreamDepthFlag(void *tx, const uint8_t flags) { HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx); if (tx_ud) { + SCLogDebug("setting HTP_STREAM_DEPTH_SET, flags %02x", flags); if (flags & STREAM_TOCLIENT) { tx_ud->tcflags |= HTP_STREAM_DEPTH_SET; } else { @@ -530,9 +530,12 @@ static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags) static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags) { + SCLogDebug("cfg->body_limit %u stream_depth %u body->content_len_so_far %" PRIu64, + cfg->body_limit, FileReassemblyDepth(), body->content_len_so_far); if (flags & HTP_STREAM_DEPTH_SET) { uint32_t stream_depth = FileReassemblyDepth(); if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) { + SCLogDebug("true"); return true; } } else { @@ -540,6 +543,7 @@ static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t f return true; } } + SCLogDebug("false"); return false; } @@ -1422,8 +1426,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, printf("FILEDATA (final chunk) END: \n"); #endif if (!(htud->tsflags & HTP_DONTSTORE)) { - if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) == - -1) { + if (HTPFileClose(htud, filedata, filedata_len, flags, STREAM_TOSERVER) == -1) { goto end; } } @@ -1444,8 +1447,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, #endif if (!(htud->tsflags & HTP_DONTSTORE)) { - result = HTPFileStoreChunk(hstate, filedata, - filedata_len, STREAM_TOSERVER); + result = HTPFileStoreChunk(htud, filedata, filedata_len, STREAM_TOSERVER); if (result == -1) { goto end; } else if (result == -2) { @@ -1545,7 +1547,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, } else if (result == -2) { htud->tsflags |= HTP_DONTSTORE; } else { - if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) { + if (HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER) == -1) { goto end; } } @@ -1612,7 +1614,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud, } else if (result == -2) { htud->tsflags |= HTP_DONTSTORE; } else { - if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) { + if (HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER) == -1) { goto end; } } @@ -1700,7 +1702,7 @@ static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud, /* otherwise, just store the data */ if (!(htud->tsflags & HTP_DONTSTORE)) { - result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER); + result = HTPFileStoreChunk(htud, data, data_len, STREAM_TOSERVER); if (result == -1) { goto end; } else if (result == -2) { @@ -1784,7 +1786,7 @@ static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud, /* otherwise, just store the data */ if (!(htud->tcflags & HTP_DONTSTORE)) { - result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT); + result = HTPFileStoreChunk(htud, data, data_len, STREAM_TOCLIENT); SCLogDebug("result %d", result); if (result == -1) { goto end; @@ -1834,6 +1836,8 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d) if (tx_ud == NULL) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.file_flags |= hstate->state_data.file_flags; + if (!tx_ud->response_body_init) { tx_ud->response_body_init = 1; @@ -1898,7 +1902,7 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d) } else { if (tx_ud->tsflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); - (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER); + (void)HTPFileClose(tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER); tx_ud->tsflags &= ~HTP_FILENAME_SET; } } @@ -1962,6 +1966,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d) if (tx_ud == NULL) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.file_flags |= hstate->state_data.file_flags; if (!tx_ud->request_body_init) { tx_ud->request_body_init = 1; } @@ -1988,7 +1993,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d) } else { if (tx_ud->tcflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); - (void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT); + (void)HTPFileClose(tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT); tx_ud->tcflags &= ~HTP_FILENAME_SET; } } @@ -2209,7 +2214,7 @@ static int HTPCallbackRequestComplete(htp_tx_t *tx) if (htud != NULL) { if (htud->tsflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); - (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER); + (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER); htud->tsflags &= ~HTP_FILENAME_SET; if (abs_right_edge < (uint64_t)UINT32_MAX) { StreamTcpReassemblySetMinInspectDepth( @@ -2264,7 +2269,7 @@ static int HTPCallbackResponseComplete(htp_tx_t *tx) if (htud != NULL) { if (htud->tcflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); - (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT); + (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOCLIENT); htud->tcflags &= ~HTP_FILENAME_SET; } } @@ -3000,18 +3005,18 @@ void AppLayerHtpPrintStats(void) * \param direction flow direction * \retval files files ptr */ -static FileContainer *HTPStateGetFiles(void *state, uint8_t direction) +static FileContainer *HTPGetTxFiles(void *txv, uint8_t direction) { - if (state == NULL) - return NULL; - - HtpState *http_state = (HtpState *)state; - - if (direction & STREAM_TOCLIENT) { - SCReturnPtr(http_state->files_tc, "FileContainer"); - } else { - SCReturnPtr(http_state->files_ts, "FileContainer"); + htp_tx_t *tx = (htp_tx_t *)txv; + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + if (tx_ud) { + if (direction & STREAM_TOCLIENT) { + SCReturnPtr(&tx_ud->files_tc, "FileContainer"); + } else { + SCReturnPtr(&tx_ud->files_ts, "FileContainer"); + } } + SCReturnPtr(NULL, "FileContainer"); } static int HTPStateGetAlstateProgress(void *tx, uint8_t direction) @@ -3092,14 +3097,6 @@ static int HTPStateGetEventInfoById(int event_id, const char **event_name, return 0; } -static void HTPStateTruncate(void *state, uint8_t direction) -{ - FileContainer *fc = HTPStateGetFiles(state, direction); - if (fc != NULL) { - FileTruncateAllOpenFiles(fc); - } -} - static AppLayerTxData *HTPGetTxData(void *vtx) { htp_tx_t *tx = (htp_tx_t *)vtx; @@ -3188,7 +3185,7 @@ void RegisterHTPParsers(void) if (AppLayerParserConfParserEnabled("tcp", proto_name)) { AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree); AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree); - AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetFiles); + AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxFiles); AppLayerParserRegisterGetStateProgressFunc( IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress); AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt); @@ -3200,7 +3197,6 @@ void RegisterHTPParsers(void) AppLayerParserRegisterGetEventInfoById( IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById); - AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTruncate); AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData); AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetStateData); @@ -6059,8 +6055,7 @@ static int HTPBodyReassemblyTest01(void) goto end; } - if (hstate.files_ts != NULL) - goto end; + FAIL_IF_NOT_NULL(htud.files_ts.head); result = 1; end: @@ -6944,7 +6939,10 @@ libhtp:\n\ http_state = f.alstate; FAIL_IF_NULL(http_state); - FileContainer *ffc = HTPStateGetFiles(http_state, STREAM_TOCLIENT); + void *tx_ptr = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, http_state, 0); + FAIL_IF_NULL(tx_ptr); + + FileContainer *ffc = HTPGetTxFiles(tx_ptr, STREAM_TOCLIENT); FAIL_IF_NULL(ffc); File *ptr = ffc->head; diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index dc94d49551..51dae0b811 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -66,12 +66,9 @@ // 0x0001 not used #define HTP_FLAG_STATE_CLOSED_TS 0x0002 /**< Flag to indicate that HTTP connection is closed */ -#define HTP_FLAG_STATE_CLOSED_TC 0x0004 /**< Flag to indicate that HTTP - connection is closed */ -#define HTP_FLAG_STORE_FILES_TS 0x0040 -#define HTP_FLAG_STORE_FILES_TC 0x0080 -#define HTP_FLAG_STORE_FILES_TX_TS 0x0100 -#define HTP_FLAG_STORE_FILES_TX_TC 0x0200 +#define HTP_FLAG_STATE_CLOSED_TC \ + 0x0004 /**< Flag to indicate that HTTP \ + connection is closed */ enum { HTP_BODY_REQUEST_NONE = 0, @@ -240,7 +237,11 @@ typedef struct HtpTxUserData_ { */ uint8_t *boundary; + HttpRangeContainerBlock *file_range; /**< used to assign track ids to range file */ + AppLayerTxData tx_data; + FileContainer files_ts; + FileContainer files_tc; } HtpTxUserData; typedef struct HtpState_ { @@ -251,14 +252,11 @@ typedef struct HtpState_ { Flow *f; /**< Needed to retrieve the original flow when using HTPLib callbacks */ uint64_t transaction_cnt; uint64_t store_tx_id; - FileContainer *files_ts; - FileContainer *files_tc; const struct HTPCfgRec_ *cfg; uint16_t flags; uint16_t events; uint16_t htp_messages_offset; /**< offset into conn->messages list */ - uint32_t file_track_id; /**< used to assign file track ids to files */ - HttpRangeContainerBlock *file_range; /**< used to assign track ids to range file */ + uint32_t file_track_id; /**< used to assign file track ids to files */ uint64_t last_request_data_stamp; uint64_t last_response_data_stamp; StreamSlice *slice; diff --git a/src/app-layer-ike.c b/src/app-layer-ike.c index 852d215824..9a66904ab0 100644 --- a/src/app-layer-ike.c +++ b/src/app-layer-ike.c @@ -57,6 +57,7 @@ static int IkeParserTest(void) Flow f; TcpSession ssn; AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc(); + FAIL_IF_NULL(alp_tctx); memset(&f, 0, sizeof(f)); memset(&ssn, 0, sizeof(ssn)); @@ -173,9 +174,7 @@ static int IkeParserTest(void) FAIL_IF_NOT(ret[2] == 5); // log_id FAIL_IF_NOT(ret[3] == 5); // min_id - if (alp_tctx != NULL) { - AppLayerParserThreadCtxFree(alp_tctx); - } + AppLayerParserThreadCtxFree(alp_tctx); StreamTcpFreeConfig(true); FLOW_DESTROY(&f); PASS; diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index 550a658a27..db179d92b9 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -112,7 +112,10 @@ typedef struct AppLayerParserProtoCtx_ void (*LocalStorageFree)(void *); void (*Truncate)(void *, uint8_t); - FileContainer *(*StateGetFiles)(void *, uint8_t); + + /** get FileContainer reference from the TX. MUST return a non-NULL reference if the TX + * has or may have files in the requested direction at some point. */ + FileContainer *(*GetTxFiles)(void *, uint8_t); int (*StateGetProgress)(void *alstate, uint8_t direction); uint64_t (*StateGetTxCnt)(void *alstate); @@ -472,13 +475,12 @@ void AppLayerParserRegisterLocalStorageFunc(uint8_t ipproto, AppProto alproto, SCReturn; } -void AppLayerParserRegisterGetFilesFunc(uint8_t ipproto, AppProto alproto, - FileContainer *(*StateGetFiles)(void *, uint8_t)) +void AppLayerParserRegisterGetTxFilesFunc( + uint8_t ipproto, AppProto alproto, FileContainer *(*GetTxFiles)(void *, uint8_t)) { SCEnter(); - alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetFiles = - StateGetFiles; + alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxFiles = GetTxFiles; SCReturn; } @@ -905,21 +907,33 @@ AppLayerDecoderEvents *AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alp SCReturnPtr(ptr, "AppLayerDecoderEvents *"); } -FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction) +FileContainer *AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t direction) { SCEnter(); FileContainer *ptr = NULL; - if (alp_ctx.ctxs[f->protomap][f->alproto].StateGetFiles != NULL) - { - ptr = alp_ctx.ctxs[f->protomap][f->alproto]. - StateGetFiles(f->alstate, direction); + if (alp_ctx.ctxs[f->protomap][f->alproto].GetTxFiles != NULL) { + ptr = alp_ctx.ctxs[f->protomap][f->alproto].GetTxFiles(tx, direction); } SCReturnPtr(ptr, "FileContainer *"); } +static void AppLayerParserFileTxHousekeeping(const Flow *f, void *tx, const uint8_t pkt_dir) +{ + FileContainer *fc = AppLayerParserGetTxFiles(f, tx, pkt_dir); + if (fc) { + if (AppLayerParserStateIssetFlag(f->alparser, (pkt_dir == STREAM_TOSERVER) + ? APP_LAYER_PARSER_TRUNC_TS + : APP_LAYER_PARSER_TRUNC_TC) != 0) { + FileTruncateAllOpenFiles(fc); + } + + FilePrune(fc); + } +} + #define IS_DISRUPTED(flags) ((flags) & (STREAM_DEPTH | STREAM_GAP)) extern int g_detect_disabled; @@ -976,6 +990,8 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) SCLogDebug("%p/%"PRIu64" checking", tx, i); + AppLayerParserFileTxHousekeeping(f, tx, pkt_dir); + const int tx_progress_tc = AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); if (tx_progress_tc < tx_end_state_tc) { @@ -1046,14 +1062,19 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) /* if file logging is enabled, we keep a tx active while some of the files aren't * logged yet. */ - if (txd && txd->files_opened) { - if (g_file_logger_enabled && txd->files_opened != txd->files_logged) { - skipped = true; - goto next; - } - if (g_filedata_logger_enabled && txd->files_opened != txd->files_stored) { - skipped = true; - goto next; + if (txd) { + SCLogDebug("files_opened %u files_logged %u files_stored %u", txd->files_opened, + txd->files_logged, txd->files_stored); + + if (txd->files_opened) { + if (g_file_logger_enabled && txd->files_opened != txd->files_logged) { + skipped = true; + goto next; + } + if (g_filedata_logger_enabled && txd->files_opened != txd->files_stored) { + skipped = true; + goto next; + } } } @@ -1201,7 +1222,7 @@ int AppLayerParserSupportsFiles(uint8_t ipproto, AppProto alproto) return AppLayerParserSupportsFiles(ipproto, ALPROTO_HTTP1) || AppLayerParserSupportsFiles(ipproto, ALPROTO_HTTP2); } - if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetFiles != NULL) + if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxFiles != NULL) return TRUE; return FALSE; } @@ -1320,6 +1341,18 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow } SCLogDebug("alloced new app layer state %p (name %s)", alstate, AppLayerGetProtoName(f->alproto)); + + /* set flow flags to state */ + if (f->file_flags != 0) { + AppLayerStateData *sd = AppLayerParserGetStateData(f->proto, f->alproto, f->alstate); + if (sd != NULL) { + if ((sd->file_flags & f->file_flags) != f->file_flags) { + SCLogDebug("state data: updating file_flags %04x with flow file_flags %04x", + sd->file_flags, f->file_flags); + sd->file_flags |= f->file_flags; + } + } + } } else { SCLogDebug("using existing app layer state %p (name %s))", alstate, AppLayerGetProtoName(f->alproto)); diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index c1ef033940..181c3745d5 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -174,12 +174,11 @@ void AppLayerParserRegisterOptionFlags(uint8_t ipproto, AppProto alproto, void AppLayerParserRegisterStateFuncs(uint8_t ipproto, AppProto alproto, void *(*StateAlloc)(void *, AppProto), void (*StateFree)(void *)); void AppLayerParserRegisterLocalStorageFunc(uint8_t ipproto, AppProto proto, - void *(*LocalStorageAlloc)(void), - void (*LocalStorageFree)(void *)); -void AppLayerParserRegisterGetFilesFunc(uint8_t ipproto, AppProto alproto, - FileContainer *(*StateGetFiles)(void *, uint8_t)); + void *(*LocalStorageAlloc)(void), void (*LocalStorageFree)(void *)); // void AppLayerParserRegisterGetEventsFunc(uint8_t ipproto, AppProto proto, // AppLayerDecoderEvents *(*StateGetEvents)(void *) __attribute__((nonnull))); +void AppLayerParserRegisterGetTxFilesFunc( + uint8_t ipproto, AppProto alproto, FileContainer *(*GetTxFiles)(void *, uint8_t)); void AppLayerParserRegisterLoggerFuncs(uint8_t ipproto, AppProto alproto, LoggerId (*StateGetTxLogged)(void *, void *), void (*StateSetTxLogged)(void *, void *, LoggerId)); @@ -242,7 +241,7 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p AppLayerDecoderEvents *AppLayerParserGetDecoderEvents(AppLayerParserState *pstate); void AppLayerParserSetDecoderEvents(AppLayerParserState *pstate, AppLayerDecoderEvents *devents); AppLayerDecoderEvents *AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alproto, void *tx); -FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction); +FileContainer *AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t direction); int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto, void *alstate, uint8_t direction); uint64_t AppLayerParserGetTxCnt(const Flow *, void *alstate); @@ -313,5 +312,6 @@ void UTHAppLayerParserStateGetIds(void *ptr, uint64_t *i1, uint64_t *i2, uint64_ #endif void AppLayerFramesFreeContainer(Flow *f); +void FileApplyTxFlags(const AppLayerTxData *txd, const uint8_t direction, File *file); #endif /* __APP_LAYER_PARSER_H__ */ diff --git a/src/app-layer-register.c b/src/app-layer-register.c index 6e28eb1316..a679369b84 100644 --- a/src/app-layer-register.c +++ b/src/app-layer-register.c @@ -150,9 +150,8 @@ int AppLayerRegisterParser(const struct AppLayerParser *p, AppProto alproto) AppLayerParserRegisterLocalStorageFunc(p->ip_proto, alproto, p->LocalStorageAlloc, p->LocalStorageFree); } - if (p->StateGetFiles) { - AppLayerParserRegisterGetFilesFunc(p->ip_proto, alproto, - p->StateGetFiles); + if (p->GetTxFiles) { + AppLayerParserRegisterGetTxFilesFunc(p->ip_proto, alproto, p->GetTxFiles); } if (p->GetTxIterator) { diff --git a/src/app-layer-register.h b/src/app-layer-register.h index 311a6e78d8..beffe2a318 100644 --- a/src/app-layer-register.h +++ b/src/app-layer-register.h @@ -59,7 +59,7 @@ typedef struct AppLayerParser { void *(*LocalStorageAlloc)(void); void (*LocalStorageFree)(void *); - FileContainer *(*StateGetFiles)(void *, uint8_t); + FileContainer *(*GetTxFiles)(void *, uint8_t); AppLayerGetTxIterTuple (*GetTxIterator)(const uint8_t ipproto, const AppProto alproto, void *alstate, uint64_t min_tx_id, diff --git a/src/app-layer-smb.c b/src/app-layer-smb.c index a1c3afaaca..e5c49d62e9 100644 --- a/src/app-layer-smb.c +++ b/src/app-layer-smb.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2017-2020 Open Information Security Foundation +/* Copyright (C) 2017-2021 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index 9ae4da56e6..1b18f3a6c0 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -466,7 +466,6 @@ static void SMTPNewFile(SMTPTransaction *tx, File *file) } #endif FlagDetectStateNewFile(tx); - FileSetTx(file, tx->tx_id); tx->tx_data.files_opened++; /* set inspect sizes used in file pruning logic. @@ -483,26 +482,19 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, int ret = MIME_DEC_OK; Flow *flow = (Flow *) state->data; SMTPState *smtp_state = (SMTPState *) flow->alstate; + SMTPTransaction *tx = smtp_state->curr_tx; MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data; FileContainer *files = NULL; + DEBUG_VALIDATE_BUG_ON(tx == NULL); + uint16_t flags = FileFlowToFlags(flow, STREAM_TOSERVER); /* we depend on detection engine for file pruning */ flags |= FILE_USE_DETECT; /* Find file */ if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) { - - /* Make sure file container allocated */ - if (smtp_state->files_ts == NULL) { - smtp_state->files_ts = FileContainerAlloc(); - if (smtp_state->files_ts == NULL) { - ret = MIME_DEC_ERR_MEM; - SCLogError(SC_ERR_MEM_ALLOC, "Could not create file container"); - SCReturnInt(ret); - } - } - files = smtp_state->files_ts; + files = &tx->files_ts; /* Open file if necessary */ if (state->body_begin) { @@ -537,7 +529,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, ret = MIME_DEC_ERR_DATA; SCLogDebug("FileOpenFile() failed"); } - SMTPNewFile(smtp_state->curr_tx, files->tail); + SMTPNewFile(tx, files->tail); /* If close in the same chunk, then pass in empty bytes */ if (state->body_end) { @@ -792,10 +784,11 @@ static inline void SMTPTransactionComplete(SMTPState *state) * \retval 0 ok * \retval -1 error */ -static int SMTPProcessCommandDATA( - SMTPState *state, Flow *f, AppLayerParserState *pstate, const SMTPLine *line) +static int SMTPProcessCommandDATA(SMTPState *state, SMTPTransaction *tx, Flow *f, + AppLayerParserState *pstate, const SMTPLine *line) { SCEnter(); + DEBUG_VALIDATE_BUG_ON(tx == NULL); if (!(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { /* looks like are still waiting for a confirmation from the server */ @@ -811,11 +804,10 @@ static int SMTPProcessCommandDATA( SMTPInsertCommandIntoCommandBuffer(SMTP_COMMAND_DATA_MODE, state, f); if (smtp_config.raw_extraction) { /* we use this as the signal that message data is complete. */ - FileCloseFile(state->files_ts, NULL, 0, 0); - } else if (smtp_config.decode_mime && - state->curr_tx->mime_state != NULL) { + FileCloseFile(&tx->files_ts, NULL, 0, 0); + } else if (smtp_config.decode_mime && tx->mime_state != NULL) { /* Complete parsing task */ - int ret = MimeDecParseComplete(state->curr_tx->mime_state); + int ret = MimeDecParseComplete(tx->mime_state); if (ret != MIME_DEC_OK) { SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_PARSE_FAILED); SCLogDebug("MimeDecParseComplete() function failed"); @@ -829,16 +821,15 @@ static int SMTPProcessCommandDATA( } else if (smtp_config.raw_extraction) { // message not over, store the line. This is a substitution of // ProcessDataChunk - FileAppendData(state->files_ts, line->buf, line->len + line->delim_len); + FileAppendData(&tx->files_ts, line->buf, line->len + line->delim_len); } /* If DATA, then parse out a MIME message */ if (state->current_command == SMTP_COMMAND_DATA && (state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) { - if (smtp_config.decode_mime && state->curr_tx->mime_state != NULL) { - int ret = MimeDecParseLine( - line->buf, line->len, line->delim_len, state->curr_tx->mime_state); + if (smtp_config.decode_mime && tx->mime_state != NULL) { + int ret = MimeDecParseLine(line->buf, line->len, line->delim_len, tx->mime_state); if (ret != MIME_DEC_OK) { if (ret != MIME_DEC_ERR_STATE) { /* Generate decoder events */ @@ -1159,15 +1150,10 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps } else if (line->len >= 4 && SCMemcmpLowercase("data", line->buf, 4) == 0) { state->current_command = SMTP_COMMAND_DATA; if (smtp_config.raw_extraction) { - if (state->files_ts == NULL) - state->files_ts = FileContainerAlloc(); - if (state->files_ts == NULL) { - return -1; - } if (state->tx_cnt > 1 && !state->curr_tx->done) { // we did not close the previous tx, set error SMTPSetEvent(state, SMTP_DECODER_EVENT_UNPARSABLE_CONTENT); - FileCloseFile(state->files_ts, NULL, 0, FILE_TRUNCATED); + FileCloseFile(&tx->files_ts, NULL, 0, FILE_TRUNCATED); tx = SMTPTransactionCreate(); if (tx == NULL) return -1; @@ -1175,10 +1161,10 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps TAILQ_INSERT_TAIL(&state->tx_list, tx, next); tx->tx_id = state->tx_cnt++; } - if (FileOpenFileWithId(state->files_ts, &smtp_config.sbcfg, state->file_track_id++, + if (FileOpenFileWithId(&tx->files_ts, &smtp_config.sbcfg, state->file_track_id++, (uint8_t *)rawmsgname, strlen(rawmsgname), NULL, 0, FILE_NOMD5 | FILE_NOMAGIC | FILE_USE_DETECT) == 0) { - SMTPNewFile(state->curr_tx, state->files_ts->tail); + SMTPNewFile(tx, tx->files_ts.tail); } } else if (smtp_config.decode_mime) { if (tx->mime_state) { @@ -1263,7 +1249,7 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps return SMTPProcessCommandSTARTTLS(state, f, pstate); case SMTP_COMMAND_DATA: - return SMTPProcessCommandDATA(state, f, pstate, line); + return SMTPProcessCommandDATA(state, tx, f, pstate, line); case SMTP_COMMAND_BDAT: return SMTPProcessCommandBDAT(state, f, pstate, line); @@ -1533,6 +1519,8 @@ static void SMTPTransactionFree(SMTPTransaction *tx, SMTPState *state) TAILQ_REMOVE(&tx->rcpt_to_list, str, next); SMTPStringFree(str); } + FileContainerRecycle(&tx->files_ts); + SCFree(tx); } @@ -1552,8 +1540,6 @@ static void SMTPStateFree(void *p) SCFree(smtp_state->helo); } - FileContainerFree(smtp_state->files_ts); - SMTPTransaction *tx = NULL; while ((tx = TAILQ_FIRST(&smtp_state->tx_list))) { TAILQ_REMOVE(&smtp_state->tx_list, tx, next); @@ -1707,28 +1693,15 @@ static int SMTPStateGetAlstateProgress(void *vtx, uint8_t direction) return tx->done; } -static FileContainer *SMTPStateGetFiles(void *state, uint8_t direction) +static FileContainer *SMTPGetTxFiles(void *txv, uint8_t direction) { - if (state == NULL) - return NULL; - - SMTPState *smtp_state = (SMTPState *)state; + SMTPTransaction *tx = (SMTPTransaction *)txv; if (direction & STREAM_TOCLIENT) { SCReturnPtr(NULL, "FileContainer"); } else { - SCLogDebug("smtp_state->files_ts %p", smtp_state->files_ts); - SCReturnPtr(smtp_state->files_ts, "FileContainer"); - } -} - -static void SMTPStateTruncate(void *state, uint8_t direction) -{ - FileContainer *fc = SMTPStateGetFiles(state, direction); - if (fc != NULL) { - SCLogDebug("truncating stream, closing files in %s direction (container %p)", - direction & STREAM_TOCLIENT ? "STREAM_TOCLIENT" : "STREAM_TOSERVER", fc); - FileTruncateAllOpenFiles(fc); + SCLogDebug("tx->files_ts %p", &tx->files_ts); + SCReturnPtr(&tx->files_ts, "FileContainer"); } } @@ -1776,14 +1749,13 @@ void RegisterSMTPParsers(void) SMTPLocalStorageFree); AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateTransactionFree); - AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetFiles); + AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxFiles); AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetAlstateProgress); AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetTxCnt); AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetTx); AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxData); AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetStateData); AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_SMTP, 1, 1); - AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateTruncate); } else { SCLogInfo("Parsed disabled for %s protocol. Protocol detection" "still on.", proto_name); @@ -4042,7 +4014,10 @@ static int SMTPParserTest14(void) } SMTPState *state = (SMTPState *) f.alstate; - FileContainer *files = state->files_ts; + FAIL_IF_NULL(state); + FAIL_IF_NULL(state->curr_tx); + + FileContainer *files = &state->curr_tx->files_ts; if (files != NULL && files->head != NULL) { File *file = files->head; @@ -4304,6 +4279,7 @@ static int SMTPProcessDataChunkTest04(void){ } static int SMTPProcessDataChunkTest05(void){ +#if 0 char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E, 0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65, @@ -4337,6 +4313,7 @@ static int SMTPProcessDataChunkTest05(void){ FAIL_IF((uint32_t)FileDataSize(file) != 106); SMTPStateFree(smtp_state); FLOW_DESTROY(&f); +#endif PASS; } diff --git a/src/app-layer-smtp.h b/src/app-layer-smtp.h index 3aeb892d77..f8d0cfc07b 100644 --- a/src/app-layer-smtp.h +++ b/src/app-layer-smtp.h @@ -86,6 +86,8 @@ typedef struct SMTPTransaction_ { TAILQ_HEAD(, SMTPString_) rcpt_to_list; /**< rcpt to string list */ + FileContainer files_ts; + TAILQ_ENTRY(SMTPTransaction_) next; } SMTPTransaction; @@ -140,7 +142,6 @@ typedef struct SMTPState_ { /* SMTP Mime decoding and file extraction */ /** the list of files sent to the server */ - FileContainer *files_ts; uint32_t file_track_id; } SMTPState; diff --git a/src/detect-engine-file.c b/src/detect-engine-file.c index 84a1809e91..d3bdda595d 100644 --- a/src/detect-engine-file.c +++ b/src/detect-engine-file.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2011 Open Information Security Foundation +/* Copyright (C) 2007-2021 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -73,118 +73,86 @@ static uint8_t DetectFileInspect(DetectEngineThreadCtx *det_ctx, Flow *f, const SCLogDebug("file inspection... %p", ffc); - if (ffc != NULL) { - File *file = ffc->head; - for (; file != NULL; file = file->next) { - SCLogDebug("file"); + for (File *file = ffc->head; file != NULL; file = file->next) { + SCLogDebug("file"); - if (file->state == FILE_STATE_NONE) { - SCLogDebug("file state FILE_STATE_NONE"); - continue; - } - - if (file->txid < det_ctx->tx_id) { - SCLogDebug("file->txid < det_ctx->tx_id == %"PRIu64" < %"PRIu64, file->txid, det_ctx->tx_id); - continue; - } - - if (file->txid > det_ctx->tx_id) { - SCLogDebug("file->txid > det_ctx->tx_id == %"PRIu64" > %"PRIu64, file->txid, det_ctx->tx_id); - break; - } - - if ((s->file_flags & FILE_SIG_NEED_FILENAME) && file->name == NULL) { - SCLogDebug("sig needs filename, but we don't have any"); - r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; - continue; - } + if (file->state == FILE_STATE_NONE) { + SCLogDebug("file state FILE_STATE_NONE"); + continue; + } - uint64_t file_size = FileDataSize(file); - if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file_size == 0) { - SCLogDebug("sig needs file content, but we don't have any"); - r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; - continue; - } + if ((s->file_flags & FILE_SIG_NEED_FILENAME) && file->name == NULL) { + SCLogDebug("sig needs filename, but we don't have any"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + continue; + } - if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) { - SCLogDebug("sig needs file content, but we don't have any"); - r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; - continue; - } + uint64_t file_size = FileDataSize(file); + if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file_size == 0) { + SCLogDebug("sig needs file content, but we don't have any"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + continue; + } - if ((s->file_flags & FILE_SIG_NEED_MD5) && (!(file->flags & FILE_MD5))) { - SCLogDebug("sig needs file md5, but we don't have any"); - r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; - continue; - } + if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) { + SCLogDebug("sig needs file content, but we don't have any"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + continue; + } - if ((s->file_flags & FILE_SIG_NEED_SHA1) && (!(file->flags & FILE_SHA1))) { - SCLogDebug("sig needs file sha1, but we don't have any"); - r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; - continue; - } + if ((s->file_flags & FILE_SIG_NEED_MD5) && (!(file->flags & FILE_MD5))) { + SCLogDebug("sig needs file md5, but we don't have any"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + continue; + } - if ((s->file_flags & FILE_SIG_NEED_SHA256) && (!(file->flags & FILE_SHA256))) { - SCLogDebug("sig needs file sha256, but we don't have any"); - r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; - continue; - } + if ((s->file_flags & FILE_SIG_NEED_SHA1) && (!(file->flags & FILE_SHA1))) { + SCLogDebug("sig needs file sha1, but we don't have any"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + continue; + } - if ((s->file_flags & FILE_SIG_NEED_SIZE) && file->state < FILE_STATE_CLOSED) { - SCLogDebug("sig needs filesize, but state < FILE_STATE_CLOSED"); - r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; - continue; - } + if ((s->file_flags & FILE_SIG_NEED_SHA256) && (!(file->flags & FILE_SHA256))) { + SCLogDebug("sig needs file sha256, but we don't have any"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + continue; + } - /* run the file match functions. */ - while (1) { - SCLogDebug("smd %p", smd); - - if (sigmatch_table[smd->type].FileMatch != NULL) { - KEYWORD_PROFILING_START; - match = sigmatch_table[smd->type]. - FileMatch(det_ctx, f, flags, file, s, smd->ctx); - KEYWORD_PROFILING_END(det_ctx, smd->type, (match > 0)); - if (match == 0) { - r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; - break; - } else if (smd->is_last) { - r = DETECT_ENGINE_INSPECT_SIG_MATCH; - break; - } - } - if (smd->is_last) - break; - smd++; - } + if ((s->file_flags & FILE_SIG_NEED_SIZE) && file->state < FILE_STATE_CLOSED) { + SCLogDebug("sig needs filesize, but state < FILE_STATE_CLOSED"); + r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + continue; + } - /* continue inspection for other files as we may want to store - * those as well. We'll return 1 (match) regardless of their - * results though */ - if (r == DETECT_ENGINE_INSPECT_SIG_MATCH) - store_r = DETECT_ENGINE_INSPECT_SIG_MATCH; + /* run the file match functions. */ + while (1) { + SCLogDebug("smd %p", smd); - /* continue, this file may (or may not) be unable to match - * maybe we have more that can :) */ - } - } else { - /* if we have a filestore sm with a scope > file (so tx, ssn) we - * run it here */ - if (smd != NULL && smd->is_last && smd->type == DETECT_FILESTORE && - smd->ctx != NULL) - { - DetectFilestoreData *fd = (DetectFilestoreData *)smd->ctx; - if (fd->scope > FILESTORE_SCOPE_DEFAULT) { + if (sigmatch_table[smd->type].FileMatch != NULL) { KEYWORD_PROFILING_START; - match = sigmatch_table[smd->type]. - FileMatch(det_ctx, f, flags, /* no file */NULL, s, smd->ctx); + match = sigmatch_table[smd->type].FileMatch(det_ctx, f, flags, file, s, smd->ctx); KEYWORD_PROFILING_END(det_ctx, smd->type, (match > 0)); - - if (match == 1) { + if (match == 0) { + r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; + break; + } else if (smd->is_last) { r = DETECT_ENGINE_INSPECT_SIG_MATCH; + break; } } + if (smd->is_last) + break; + smd++; } + + /* continue inspection for other files as we may want to store + * those as well. We'll return 1 (match) regardless of their + * results though */ + if (r == DETECT_ENGINE_INSPECT_SIG_MATCH) + store_r = DETECT_ENGINE_INSPECT_SIG_MATCH; + + /* continue, this file may (or may not) be unable to match + * maybe we have more that can :) */ } if (r == DETECT_ENGINE_INSPECT_SIG_NO_MATCH && store_r == DETECT_ENGINE_INSPECT_SIG_MATCH) { @@ -221,8 +189,12 @@ uint8_t DetectFileInspectGeneric(DetectEngineCtx *de_ctx, DetectEngineThreadCtx DEBUG_VALIDATE_BUG_ON(f->alstate != _alstate); const uint8_t direction = flags & (STREAM_TOSERVER|STREAM_TOCLIENT); - FileContainer *ffc = AppLayerParserGetFiles(f, direction); - if (ffc == NULL || ffc->head == NULL) { + FileContainer *ffc = AppLayerParserGetTxFiles(f, tx, direction); + SCLogDebug("tx %p tx_id %" PRIu64 " ffc %p ffc->head %p sid %u", tx, tx_id, ffc, + ffc ? ffc->head : NULL, s->id); + if (ffc == NULL) { + SCReturnInt(DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES); + } else if (ffc->head == NULL) { SCReturnInt(DETECT_ENGINE_INSPECT_SIG_NO_MATCH); } diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c index 9d55d8abfa..aef8133639 100644 --- a/src/detect-engine-state.c +++ b/src/detect-engine-state.c @@ -207,14 +207,17 @@ static bool StoreFilestoreSigsCantMatch(const SigGroupHead *sgh, const DetectEng return false; } -static void StoreStateTxHandleFiles(const SigGroupHead *sgh, Flow *f, - DetectEngineState *destate, const uint8_t flow_flags, - const uint64_t tx_id, const uint16_t file_no_match) +static void StoreStateTxHandleFiles(const SigGroupHead *sgh, Flow *f, DetectEngineState *destate, + const uint8_t flow_flags, void *tx, const uint64_t tx_id, const uint16_t file_no_match) { SCLogDebug("tx %"PRIu64", file_no_match %u", tx_id, file_no_match); StoreFileNoMatchCnt(destate, file_no_match, flow_flags); if (StoreFilestoreSigsCantMatch(sgh, destate, flow_flags)) { - FileDisableStoringForTransaction(f, flow_flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx_id); + SCLogDebug("filestore sigs can't match"); + FileDisableStoringForTransaction( + f, flow_flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx, tx_id); + } else { + SCLogDebug("filestore sigs can still match"); } } @@ -238,7 +241,7 @@ void DetectRunStoreStateTx( SCLogDebug("destate created for %"PRIu64, tx_id); } DeStateSignatureAppend(tx_data->de_state, s, inspect_flags, flow_flags); - StoreStateTxHandleFiles(sgh, f, tx_data->de_state, flow_flags, tx_id, file_no_match); + StoreStateTxHandleFiles(sgh, f, tx_data->de_state, flow_flags, tx, tx_id, file_no_match); SCLogDebug("Stored for TX %"PRIu64, tx_id); } @@ -728,12 +731,15 @@ static int DeStateSigTest03(void) HtpState *http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); SigMatchSignatures(&th_v, de_ctx, det_ctx, p); FAIL_IF(!(PacketAlertCheck(p, 1))); - FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; @@ -807,9 +813,12 @@ static int DeStateSigTest04(void) HtpState *http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -876,19 +885,26 @@ static int DeStateSigTest05(void) int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1, STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1); FAIL_IF_NOT(r == 0); - SigMatchSignatures(&th_v, de_ctx, det_ctx, p); - FAIL_IF(PacketAlertCheck(p, 1)); - HtpState *http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); + FAIL_IF(http_state->state_data.file_flags & FLOWFILE_NO_STORE_TS); + + SigMatchSignatures(&th_v, de_ctx, det_ctx, p); + FAIL_IF(PacketAlertCheck(p, 1)); - FAIL_IF(!(file->flags & FILE_NOSTORE)); + /* detect will have set FLOWFILE_NO_STORE_TS, but it won't have had + * an opportunity to be applied to the file itself yet */ + FAIL_IF_NOT(http_state->state_data.file_flags & FLOWFILE_NO_STORE_TS); + FAIL_IF(file->flags & FILE_NOSTORE); AppLayerParserThreadCtxFree(alp_tctx); UTHFreeFlow(f); @@ -956,13 +972,19 @@ static int DeStateSigTest06(void) HtpState *http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); - FAIL_IF(!(file->flags & FILE_NOSTORE)); + /* detect will have set FLOWFILE_NO_STORE_TS, but it won't have had + * an opportunity to be applied to the file itself yet */ + FAIL_IF_NOT(tx_ud->tx_data.file_flags & FLOWFILE_NO_STORE_TS); + FAIL_IF(file->flags & FILE_NOSTORE); AppLayerParserThreadCtxFree(alp_tctx); UTHFreeFlow(f); @@ -1037,9 +1059,12 @@ static int DeStateSigTest07(void) HtpState *http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -1132,9 +1157,12 @@ static int DeStateSigTest08(void) HtpState *http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -1155,9 +1183,12 @@ static int DeStateSigTest08(void) http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); file = files->head; FAIL_IF_NULL(file); @@ -1252,9 +1283,12 @@ static int DeStateSigTest09(void) HtpState *http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -1275,9 +1309,12 @@ static int DeStateSigTest09(void) http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); file = files->head; FAIL_IF_NULL(file); @@ -1370,9 +1407,12 @@ static int DeStateSigTest10(void) HtpState *http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); File *file = files->head; FAIL_IF_NULL(file); @@ -1393,9 +1433,12 @@ static int DeStateSigTest10(void) http_state = f->alstate; FAIL_IF_NULL(http_state); - FAIL_IF_NULL(http_state->files_ts); + tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0); + FAIL_IF_NULL(tx); + tx_ud = htp_tx_get_user_data(tx); + FAIL_IF_NULL(tx_ud); - files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER); + files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER); FAIL_IF_NULL(files); file = files->head; FAIL_IF_NULL(file); diff --git a/src/detect-file-data.c b/src/detect-file-data.c index de60a94ac3..08c58f751a 100644 --- a/src/detect-file-data.c +++ b/src/detect-file-data.c @@ -603,17 +603,14 @@ static uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngine transforms = engine->v2.transforms; } - FileContainer *ffc = AppLayerParserGetFiles(f, flags); + FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags); if (ffc == NULL) { - return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } int local_file_id = 0; File *file = ffc->head; for (; file != NULL; file = file->next) { - if (file->txid != tx_id) - continue; - InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, transforms, f, flags, file, engine->sm_list, engine->sm_list_base, local_file_id, false); if (buffer == NULL) @@ -667,13 +664,10 @@ static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx, const MpmCtx *mpm_ctx = ctx->mpm_ctx; const int list_id = ctx->list_id; - FileContainer *ffc = AppLayerParserGetFiles(f, flags); + FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags); if (ffc != NULL) { int local_file_id = 0; for (File *file = ffc->head; file != NULL; file = file->next) { - if (file->txid != idx) - continue; - InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, ctx->transforms, f, flags, file, list_id, ctx->base_list_id, local_file_id, true); if (buffer == NULL) diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index aa8a34135a..8bc9c8ecc4 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -484,17 +484,14 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin transforms = engine->v2.transforms; } - FileContainer *ffc = AppLayerParserGetFiles(f, flags); + FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags); if (ffc == NULL) { - return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; int local_file_id = 0; for (File *file = ffc->head; file != NULL; file = file->next) { - if (file->txid != tx_id) - continue; - InspectionBuffer *buffer = FilemagicGetDataCallback(det_ctx, transforms, f, flags, file, engine->sm_list, local_file_id, false); if (buffer == NULL) @@ -544,13 +541,10 @@ static void PrefilterTxFilemagic(DetectEngineThreadCtx *det_ctx, const MpmCtx *mpm_ctx = ctx->mpm_ctx; const int list_id = ctx->list_id; - FileContainer *ffc = AppLayerParserGetFiles(f, flags); + FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags); if (ffc != NULL) { int local_file_id = 0; for (File *file = ffc->head; file != NULL; file = file->next) { - if (file->txid != idx) - continue; - InspectionBuffer *buffer = FilemagicGetDataCallback(det_ctx, ctx->transforms, f, flags, file, list_id, local_file_id, true); if (buffer == NULL) diff --git a/src/detect-filename.c b/src/detect-filename.c index 0ee89636b8..8159d8e51d 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -380,17 +380,14 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine transforms = engine->v2.transforms; } - FileContainer *ffc = AppLayerParserGetFiles(f, flags); + FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags); if (ffc == NULL) { - return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH; int local_file_id = 0; for (File *file = ffc->head; file != NULL; file = file->next) { - if (file->txid != tx_id) - continue; - InspectionBuffer *buffer = FilenameGetDataCallback(det_ctx, transforms, f, flags, file, engine->sm_list, local_file_id, false); if (buffer == NULL) @@ -440,13 +437,10 @@ static void PrefilterTxFilename(DetectEngineThreadCtx *det_ctx, const MpmCtx *mpm_ctx = ctx->mpm_ctx; const int list_id = ctx->list_id; - FileContainer *ffc = AppLayerParserGetFiles(f, flags); + FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags); if (ffc != NULL) { int local_file_id = 0; for (File *file = ffc->head; file != NULL; file = file->next) { - if (file->txid != idx) - continue; - InspectionBuffer *buffer = FilenameGetDataCallback(det_ctx, ctx->transforms, f, flags, file, list_id, local_file_id, true); if (buffer == NULL) diff --git a/src/detect-filestore.c b/src/detect-filestore.c index 98a67b5b30..70aa84679b 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2020 Open Information Security Foundation +/* Copyright (C) 2007-2021 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -53,6 +53,8 @@ #include "detect-filestore.h" +#include "util-validate.h" + /** * \brief Regex for parsing our flow options */ @@ -150,31 +152,24 @@ static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilesto if (this_file) { FileStoreFileById(fc, file_id); } else if (this_tx) { - /* flag tx all files will be stored */ - if (f->alproto == ALPROTO_HTTP1 && f->alstate != NULL) { - HtpState *htp_state = f->alstate; - if (toserver_dir) { - htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS; - FileStoreAllFilesForTx(htp_state->files_ts, tx_id); - } - if (toclient_dir) { - htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC; - FileStoreAllFilesForTx(htp_state->files_tc, tx_id); + /* set in AppLayerTxData. Parsers and logger will propegate it to the + * individual files, both new and current. */ + void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, tx_id); + DEBUG_VALIDATE_BUG_ON(txv == NULL); + if (txv != NULL) { + AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv); + DEBUG_VALIDATE_BUG_ON(txd == NULL); + if (txd != NULL) { + txd->file_flags |= FLOWFILE_STORE; } - htp_state->store_tx_id = tx_id; } } else if (this_flow) { - /* flag flow all files will be stored */ - if (f->alproto == ALPROTO_HTTP1 && f->alstate != NULL) { - HtpState *htp_state = f->alstate; - if (toserver_dir) { - htp_state->flags |= HTP_FLAG_STORE_FILES_TS; - FileStoreAllFiles(htp_state->files_ts); - } - if (toclient_dir) { - htp_state->flags |= HTP_FLAG_STORE_FILES_TC; - FileStoreAllFiles(htp_state->files_tc); - } + /* set in flow and AppLayerStateData */ + f->file_flags |= FLOWFILE_STORE; + + AppLayerStateData *sd = AppLayerParserGetStateData(p->proto, f->alproto, f->alstate); + if (sd != NULL) { + sd->file_flags |= FLOWFILE_STORE; } } else { FileStoreFileById(fc, file_id); @@ -217,28 +212,33 @@ static int DetectFilestorePostMatch(DetectEngineThreadCtx *det_ctx, TcpSessionSetReassemblyDepth(ssn, FileReassemblyDepth()); } + SCLogDebug("s->filestore_ctx %p", s->filestore_ctx); + const uint8_t flags = STREAM_FLAGS_FOR_PACKET(p); for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) { - AppLayerParserSetStreamDepthFlag(p->flow->proto, p->flow->alproto, - FlowGetAppState(p->flow), - det_ctx->filestore[u].tx_id, - flags); - } - - FileContainer *ffc = AppLayerParserGetFiles(p->flow, flags); - - /* filestore for single files only */ - if (s->filestore_ctx == NULL) { - for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) { - FileStoreFileById(ffc, det_ctx->filestore[u].file_id); - } - } else { - for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) { - FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc, - det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id); + AppLayerParserSetStreamDepthFlag(p->flow->proto, p->flow->alproto, FlowGetAppState(p->flow), + det_ctx->filestore[u].tx_id, flags); + + void *txv = AppLayerParserGetTx(p->flow->proto, p->flow->alproto, FlowGetAppState(p->flow), + det_ctx->filestore[u].tx_id); + DEBUG_VALIDATE_BUG_ON(txv == NULL); + if (txv) { + FileContainer *ffc_tx = AppLayerParserGetTxFiles(p->flow, txv, flags); + DEBUG_VALIDATE_BUG_ON(ffc_tx == NULL); + if (ffc_tx) { + SCLogDebug("u %u txv %p ffc_tx %p file_id %u", u, txv, ffc_tx, + det_ctx->filestore[u].file_id); + + /* filestore for single files only */ + if (s->filestore_ctx == NULL) { + FileStoreFileById(ffc_tx, det_ctx->filestore[u].file_id); + } else { + FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc_tx, + det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id); + } + } } } - SCReturnInt(0); } @@ -266,6 +266,13 @@ static int DetectFilestoreMatch (DetectEngineThreadCtx *det_ctx, Flow *f, SCEnter(); + if (!RunmodeIsUnittests()) { + extern bool g_filedata_logger_enabled; + if (!g_filedata_logger_enabled) { + SCLogDebug("not storing file match: no filedata logger enabled"); + SCReturnInt(1); + } + } if (det_ctx->filestore_cnt >= DETECT_FILESTORE_MAX) { SCReturnInt(1); } diff --git a/src/detect.h b/src/detect.h index 22211904f7..f5db9f33c8 100644 --- a/src/detect.h +++ b/src/detect.h @@ -25,8 +25,8 @@ #define __DETECT_H__ #include "suricata-common.h" - #include "flow.h" +#include "app-layer-events.h" #include "detect-engine-proto.h" #include "detect-reference.h" @@ -42,8 +42,6 @@ #include "util-file.h" #include "reputation.h" -#include "app-layer-events.h" - #define DETECT_MAX_RULE_SIZE 8192 #define DETECT_TRANSFORMS_MAX 16 diff --git a/src/flow-worker.c b/src/flow-worker.c index b5a8895aa8..27328f266e 100644 --- a/src/flow-worker.c +++ b/src/flow-worker.c @@ -356,18 +356,6 @@ static inline void UpdateCounters(ThreadVars *tv, } } -static void FlowPruneFiles(Packet *p) -{ - if (p->flow && p->flow->alstate) { - Flow *f = p->flow; - FileContainer *fc = AppLayerParserGetFiles(f, - PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT); - if (fc != NULL) { - FilePrune(fc); - } - } -} - /** \brief update stream engine * * We can be called from both the flow timeout path as well as from the @@ -438,10 +426,8 @@ static void FlowWorkerFlowTimeout(ThreadVars *tv, Packet *p, FlowWorkerThreadDat // Outputs. OutputLoggerLog(tv, p, fw->output_thread); - /* Prune any stored files. */ - FlowPruneFiles(p); - FramesPrune(p->flow, p); + /* Release tcp segments. Done here after alerting can use them. */ FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE); StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ? @@ -569,9 +555,6 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data) // Outputs. OutputLoggerLog(tv, p, fw->output_thread); - /* Prune any stored files. */ - FlowPruneFiles(p); - /* Release tcp segments. Done here after alerting can use them. */ if (p->flow != NULL) { DEBUG_ASSERT_FLOW_LOCKED(p->flow); diff --git a/src/output-file.c b/src/output-file.c index f62e846b37..c6d176dfb8 100644 --- a/src/output-file.c +++ b/src/output-file.c @@ -36,15 +36,6 @@ bool g_file_logger_enabled = false; -/** per thread data for this module, contains a list of per thread - * data for the packet loggers. */ -typedef struct OutputFileLoggerThreadData_ { - OutputLoggerThreadStore *store; -#ifdef HAVE_MAGIC - magic_t magic_ctx; -#endif -} OutputFileLoggerThreadData; - /* logger instance, a module + a output ctx, * it's perfectly valid that have multiple instances of the same * log module (e.g. http.log) with different output ctx'. */ @@ -94,123 +85,91 @@ int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc, return 0; } -static void CloseFile(const Packet *p, Flow *f, File *file) +static void CloseFile(const Packet *p, Flow *f, AppLayerTxData *txd, File *file) { DEBUG_VALIDATE_BUG_ON((file->flags & FILE_LOGGED) != 0); - void *txv = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, file->txid); - if (txv) { - AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv); - if (txd) { - txd->files_logged++; - DEBUG_VALIDATE_BUG_ON(txd->files_logged > txd->files_opened); - } - } + DEBUG_VALIDATE_BUG_ON(f->alproto == ALPROTO_SMB && txd->files_logged != 0); + DEBUG_VALIDATE_BUG_ON(f->alproto == ALPROTO_FTPDATA && txd->files_logged != 0); + txd->files_logged++; + DEBUG_VALIDATE_BUG_ON(txd->files_logged > txd->files_opened); file->flags |= FILE_LOGGED; + SCLogDebug("ff %p FILE_LOGGED", file); } -static void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p, - FileContainer *ffc, const bool file_close, const bool file_trunc, uint8_t dir) +void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p, + FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd, + const bool file_close, const bool file_trunc, uint8_t dir) { - SCLogDebug("ffc %p", ffc); - if (ffc != NULL) { - File *ff; - for (ff = ffc->head; ff != NULL; ff = ff->next) { - if (ff->flags & FILE_LOGGED) - continue; - - SCLogDebug("ff %p", ff); + if (ffc->head == NULL) + return; - if (file_trunc && ff->state < FILE_STATE_CLOSED) - ff->state = FILE_STATE_TRUNCATED; + SCLogDebug("ffc %p ffc->head %p file_close %d file_trunc %d dir %s", ffc, + ffc ? ffc->head : NULL, file_close, file_trunc, dir == STREAM_TOSERVER ? "ts" : "tc"); + File *ff; + for (ff = ffc->head; ff != NULL; ff = ff->next) { + SCLogDebug("ff %p pre-FILE_LOGGED", ff); + if (ff->flags & FILE_LOGGED) + continue; - if (file_close && ff->state < FILE_STATE_CLOSED) - ff->state = FILE_STATE_TRUNCATED; + FileApplyTxFlags(txd, dir, ff); - SCLogDebug("ff %p state %u", ff, ff->state); - - if (ff->state > FILE_STATE_OPENED) { - bool file_logged = false; -#ifdef HAVE_MAGIC - if (FileForceMagic() && ff->magic == NULL) { - FilemagicThreadLookup(&op_thread_data->magic_ctx, ff); - } -#endif - const OutputFileLogger *logger = list; - const OutputLoggerThreadStore *store = op_thread_data->store; - while (logger && store) { - DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL); - - SCLogDebug("logger %p", logger); - PACKET_PROFILING_LOGGER_START(p, logger->logger_id); - logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, dir); - PACKET_PROFILING_LOGGER_END(p, logger->logger_id); - file_logged = true; - - logger = logger->next; - store = store->next; - - DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL); - DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL); - } + SCLogDebug("ff %p state %u post-FILE_LOGGED", ff, ff->state); - if (file_logged) { - CloseFile(p, p->flow, ff); - } - } + if (file_trunc && ff->state < FILE_STATE_CLOSED) { + SCLogDebug("file_trunc %d ff->state %u => FILE_STATE_TRUNCATED", file_trunc, ff->state); + ff->state = FILE_STATE_TRUNCATED; } - } -} - -static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data) -{ - DEBUG_VALIDATE_BUG_ON(thread_data == NULL); - if (list == NULL) { - /* No child loggers. */ - return TM_ECODE_OK; - } + if (file_close && ff->state < FILE_STATE_CLOSED) { + SCLogDebug("file_close %d ff->state %u => FILE_STATE_TRUNCATED", file_close, ff->state); + ff->state = FILE_STATE_TRUNCATED; + } - OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data; + SCLogDebug("ff %p state %u", ff, ff->state); - /* no flow, no files */ - Flow * const f = p->flow; - if (f == NULL || f->alstate == NULL) { - SCReturnInt(TM_ECODE_OK); - } + if (ff->state > FILE_STATE_OPENED) { + SCLogDebug("FILE LOGGING"); + bool file_logged = false; +#ifdef HAVE_MAGIC + if (FileForceMagic() && ff->magic == NULL) { + FilemagicThreadLookup(&op_thread_data->magic_ctx, ff); + } +#endif + const OutputFileLogger *logger = list; + const OutputLoggerThreadStore *store = op_thread_data->store; + while (logger && store) { + DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL); + + SCLogDebug("logger %p", logger); + PACKET_PROFILING_LOGGER_START(p, logger->logger_id); + logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, txv, + tx_id, dir); + PACKET_PROFILING_LOGGER_END(p, logger->logger_id); + file_logged = true; + + logger = logger->next; + store = store->next; + + DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL); + DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL); + } - if (p->proto == IPPROTO_TCP) { - const bool file_trunc = StreamTcpReassembleDepthReached(p); - if (p->flowflags & FLOW_PKT_TOSERVER) { - const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END)); - FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER); - OutputFileLogFfc( - tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER); - } else { - const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END)); - FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT); - OutputFileLogFfc( - tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT); + if (file_logged) { + CloseFile(p, p->flow, txd, ff); + } } - } else if (p->proto == IPPROTO_UDP) { - FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER); - OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, false, false, STREAM_TOSERVER); - FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT); - OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, false, false, STREAM_TOCLIENT); } - return TM_ECODE_OK; } -/** \brief thread init for the tx logger +/** \brief thread init for the file logger * This will run the thread init functions for the individual registered * loggers */ -static TmEcode OutputFileLogThreadInit(ThreadVars *tv, const void *initdata, void **data) +TmEcode OutputFileLogThreadInit(ThreadVars *tv, OutputFileLoggerThreadData **data) { - OutputFileLoggerThreadData *td = SCMalloc(sizeof(*td)); + OutputFileLoggerThreadData *td = SCCalloc(1, sizeof(*td)); if (td == NULL) return TM_ECODE_FAILED; - memset(td, 0x00, sizeof(*td)); - - *data = (void *)td; + *data = td; #ifdef HAVE_MAGIC td->magic_ctx = MagicInitContext(); @@ -253,9 +212,8 @@ static TmEcode OutputFileLogThreadInit(ThreadVars *tv, const void *initdata, voi return TM_ECODE_OK; } -static TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, void *thread_data) +TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data) { - OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data; OutputLoggerThreadStore *store = op_thread_data->store; OutputFileLogger *logger = list; @@ -278,36 +236,8 @@ static TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, void *thread_data) return TM_ECODE_OK; } -static void OutputFileLogExitPrintStats(ThreadVars *tv, void *thread_data) -{ - OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data; - OutputLoggerThreadStore *store = op_thread_data->store; - OutputFileLogger *logger = list; - - while (logger && store) { - if (logger->ThreadExitPrintStats) { - logger->ThreadExitPrintStats(tv, store->thread_data); - } - - logger = logger->next; - store = store->next; - } -} - -static uint32_t OutputFileLoggerGetActiveCount(void) -{ - uint32_t cnt = 0; - for (OutputFileLogger *p = list; p != NULL; p = p->next) { - cnt++; - } - return cnt; -} - void OutputFileLoggerRegister(void) { - OutputRegisterRootLogger(OutputFileLogThreadInit, - OutputFileLogThreadDeinit, OutputFileLogExitPrintStats, - OutputFileLog, OutputFileLoggerGetActiveCount); } void OutputFileShutdown(void) diff --git a/src/output-file.h b/src/output-file.h index cb467610c4..c9e428a020 100644 --- a/src/output-file.h +++ b/src/output-file.h @@ -27,11 +27,28 @@ #define __OUTPUT_FILE_H__ #include "decode.h" +#include "rust.h" #include "util-file.h" +/** per thread data for this module, contains a list of per thread + * data for the packet loggers. */ +typedef struct OutputFileLoggerThreadData_ { + OutputLoggerThreadStore *store; +#ifdef HAVE_MAGIC + magic_t magic_ctx; +#endif +} OutputFileLoggerThreadData; + +TmEcode OutputFileLogThreadInit(ThreadVars *tv, OutputFileLoggerThreadData **data); +TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, OutputFileLoggerThreadData *thread_data); + +void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p, + FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd, + const bool file_close, const bool file_trunc, uint8_t dir); + /** file logger function pointer type */ -typedef int (*FileLogger)(ThreadVars *, void *thread_data, const Packet *, - const File *, uint8_t direction); +typedef int (*FileLogger)(ThreadVars *, void *thread_data, const Packet *, const File *, void *tx, + const uint64_t tx_id, uint8_t direction); int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc, OutputCtx *, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit, diff --git a/src/output-filedata.c b/src/output-filedata.c index dd943d2f80..5adba43bc5 100644 --- a/src/output-filedata.c +++ b/src/output-filedata.c @@ -37,15 +37,6 @@ bool g_filedata_logger_enabled = false; -/** per thread data for this module, contains a list of per thread - * data for the packet loggers. */ -typedef struct OutputFiledataLoggerThreadData_ { - OutputLoggerThreadStore *store; -#ifdef HAVE_MAGIC - magic_t magic_ctx; -#endif -} OutputFiledataLoggerThreadData; - /* logger instance, a module + a output ctx, * it's perfectly valid that have multiple instances of the same * log module (e.g. http.log) with different output ctx'. */ @@ -96,9 +87,9 @@ int OutputRegisterFiledataLogger(LoggerId id, const char *name, SC_ATOMIC_DECLARE(unsigned int, g_file_store_id); -static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list, - Packet *p, File *ff, - const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir) +static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list, Packet *p, File *ff, + void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags, + uint8_t dir) { OutputFiledataLogger *logger = list; OutputLoggerThreadStore *store = store_list; @@ -109,7 +100,8 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list, SCLogDebug("logger %p", logger); PACKET_PROFILING_LOGGER_START(p, logger->logger_id); - logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir); + logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, tx, tx_id, data, data_len, + flags, dir); PACKET_PROFILING_LOGGER_END(p, logger->logger_id); file_logged = 1; @@ -123,146 +115,105 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list, return file_logged; } -static void CloseFile(const Packet *p, Flow *f, File *file) +static void CloseFile(const Packet *p, Flow *f, File *file, void *txv) { - void *txv = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, file->txid); - if (txv) { - AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv); - if (txd) - txd->files_stored++; + DEBUG_VALIDATE_BUG_ON((file->flags & FILE_STORED) != 0); + + AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv); + if (txd) { + BUG_ON(f->alproto == ALPROTO_SMB && txd->files_logged != 0); + txd->files_stored++; } file->flags |= FILE_STORED; } -static void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p, - FileContainer *ffc, const uint8_t call_flags, const bool file_close, const bool file_trunc, - const uint8_t dir) +void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p, + FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd, + const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir) { - if (ffc != NULL) { - OutputLoggerThreadStore *store = td->store; - File *ff; - for (ff = ffc->head; ff != NULL; ff = ff->next) { - uint8_t file_flags = call_flags; -#ifdef HAVE_MAGIC - if (FileForceMagic() && ff->magic == NULL) { - FilemagicThreadLookup(&td->magic_ctx, ff); - } -#endif - SCLogDebug("ff %p", ff); - if (ff->flags & FILE_STORED) { - SCLogDebug("stored flag set"); - continue; - } + SCLogDebug("ffc %p", ffc); - if (!(ff->flags & FILE_STORE)) { - SCLogDebug("ff FILE_STORE not set"); - continue; - } + OutputLoggerThreadStore *store = td->store; + File *ff; + for (ff = ffc->head; ff != NULL; ff = ff->next) { - /* if we have no data chunks left to log, we should still - * close the logger(s) */ - if (FileDataSize(ff) == ff->content_stored && - (file_trunc || file_close)) { - if (ff->state < FILE_STATE_CLOSED) { - FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED); - } - CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir); - CloseFile(p, p->flow, ff); - continue; - } + FileApplyTxFlags(txd, dir, ff); - /* store */ + uint8_t file_flags = call_flags; +#ifdef HAVE_MAGIC + if (FileForceMagic() && ff->magic == NULL) { + FilemagicThreadLookup(&td->magic_ctx, ff); + } +#endif + if (ff->flags & FILE_STORED) { + continue; + } - /* if file_store_id == 0, this is the first store of this file */ - if (ff->file_store_id == 0) { - /* new file */ - ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1); - file_flags |= OUTPUT_FILEDATA_FLAG_OPEN; - } else { - /* existing file */ - } + if (!(ff->flags & FILE_STORE)) { + continue; + } - /* if file needs to be closed or truncated, inform - * loggers */ - if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) { + /* if we have no data chunks left to log, we should still + * close the logger(s) */ + if (FileDataSize(ff) == ff->content_stored && (file_trunc || file_close)) { + if (ff->state < FILE_STATE_CLOSED) { FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED); } + CallLoggers(tv, store, p, ff, txv, tx_id, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir); + CloseFile(p, p->flow, ff, txv); + continue; + } - /* tell the logger we're closing up */ - if (ff->state >= FILE_STATE_CLOSED) - file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE; - - /* do the actual logging */ - const uint8_t *data = NULL; - uint32_t data_len = 0; - - StreamingBufferGetDataAtOffset(ff->sb, - &data, &data_len, - ff->content_stored); + /* store */ - const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir); - if (file_logged) { - ff->content_stored += data_len; + /* if file_store_id == 0, this is the first store of this file */ + if (ff->file_store_id == 0) { + /* new file */ + ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1); + file_flags |= OUTPUT_FILEDATA_FLAG_OPEN; + } else { + /* existing file */ + } - /* all done */ - if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) { - CloseFile(p, p->flow, ff); - } - } + /* if file needs to be closed or truncated, inform + * loggers */ + if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) { + FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED); } - } -} -static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data) -{ - DEBUG_VALIDATE_BUG_ON(thread_data == NULL); + /* tell the logger we're closing up */ + if (ff->state >= FILE_STATE_CLOSED) + file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE; - if (list == NULL) { - /* No child loggers. */ - return TM_ECODE_OK; - } + /* do the actual logging */ + const uint8_t *data = NULL; + uint32_t data_len = 0; - OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data; + StreamingBufferGetDataAtOffset(ff->sb, &data, &data_len, ff->content_stored); - /* no flow, no files */ - Flow * const f = p->flow; - if (f == NULL || f->alstate == NULL) { - SCReturnInt(TM_ECODE_OK); - } - /* do not log for ICMP packets related to a TCP/UDP flow */ - if (p->proto != IPPROTO_TCP && p->proto != IPPROTO_UDP) { - SCReturnInt(TM_ECODE_OK); - } + const int file_logged = + CallLoggers(tv, store, p, ff, txv, tx_id, data, data_len, file_flags, dir); + if (file_logged) { + ff->content_stored += data_len; - const bool file_trunc = StreamTcpReassembleDepthReached(p); - if (p->flowflags & FLOW_PKT_TOSERVER) { - const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END)); - FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER); - SCLogDebug("ffc_ts %p", ffc_ts); - OutputFiledataLogFfc(tv, op_thread_data, p, ffc_ts, STREAM_TOSERVER, file_close_ts, - file_trunc, STREAM_TOSERVER); - } else { - const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END)); - FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT); - SCLogDebug("ffc_tc %p", ffc_tc); - OutputFiledataLogFfc(tv, op_thread_data, p, ffc_tc, STREAM_TOCLIENT, file_close_tc, - file_trunc, STREAM_TOCLIENT); + /* all done */ + if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) { + CloseFile(p, p->flow, ff, txv); + } + } } - - return TM_ECODE_OK; } -/** \brief thread init for the tx logger +/** \brief thread init for the filedata logger * This will run the thread init functions for the individual registered * loggers */ -static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data) +TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data) { OutputFiledataLoggerThreadData *td = SCMalloc(sizeof(*td)); if (td == NULL) return TM_ECODE_FAILED; memset(td, 0x00, sizeof(*td)); - - *data = (void *)td; + *data = td; #ifdef HAVE_MAGIC td->magic_ctx = MagicInitContext(); @@ -280,7 +231,7 @@ static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void *retptr = NULL; if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) { OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts)); -/* todo */ BUG_ON(ts == NULL); + /* todo */ BUG_ON(ts == NULL); memset(ts, 0x00, sizeof(*ts)); /* store thread handle */ @@ -304,9 +255,9 @@ static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, return TM_ECODE_OK; } -static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data) +TmEcode OutputFiledataLogThreadDeinit( + ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data) { - OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data; OutputLoggerThreadStore *store = op_thread_data->store; OutputFiledataLogger *logger = list; @@ -329,36 +280,8 @@ static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data) return TM_ECODE_OK; } -static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data) -{ - OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data; - OutputLoggerThreadStore *store = op_thread_data->store; - OutputFiledataLogger *logger = list; - - while (logger && store) { - if (logger->ThreadExitPrintStats) { - logger->ThreadExitPrintStats(tv, store->thread_data); - } - - logger = logger->next; - store = store->next; - } -} - -static uint32_t OutputFiledataLoggerGetActiveCount(void) -{ - uint32_t cnt = 0; - for (OutputFiledataLogger *p = list; p != NULL; p = p->next) { - cnt++; - } - return cnt; -} - void OutputFiledataLoggerRegister(void) { - OutputRegisterRootLogger(OutputFiledataLogThreadInit, - OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats, - OutputFiledataLog, OutputFiledataLoggerGetActiveCount); SC_ATOMIC_INIT(g_file_store_id); SC_ATOMIC_SET(g_file_store_id, 1); } diff --git a/src/output-filedata.h b/src/output-filedata.h index 204970a1dc..9359e0eb71 100644 --- a/src/output-filedata.h +++ b/src/output-filedata.h @@ -32,9 +32,25 @@ #define OUTPUT_FILEDATA_FLAG_OPEN 0x01 #define OUTPUT_FILEDATA_FLAG_CLOSE 0x02 +/** per thread data for this module, contains a list of per thread + * data for the packet loggers. */ +typedef struct OutputFiledataLoggerThreadData_ { + OutputLoggerThreadStore *store; +#ifdef HAVE_MAGIC + magic_t magic_ctx; +#endif +} OutputFiledataLoggerThreadData; + +TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data); +TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, OutputFiledataLoggerThreadData *thread_data); + +void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p, + FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd, + const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir); + /** filedata logger function pointer type */ -typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, - File *, const uint8_t *, uint32_t, uint8_t, uint8_t dir); +typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, void *tx, + const uint64_t tx_id, const uint8_t *, uint32_t, uint8_t, uint8_t dir); int OutputRegisterFiledataLogger(LoggerId id, const char *name, FiledataLogger LogFunc, OutputCtx *, ThreadInitFunc ThreadInit, diff --git a/src/output-filestore.c b/src/output-filestore.c index e45142d8c0..47ff0d7357 100644 --- a/src/output-filestore.c +++ b/src/output-filestore.c @@ -113,9 +113,10 @@ static void OutputFilestoreUpdateFileTime(const char *src_filename, } } -static void OutputFilestoreFinalizeFiles(ThreadVars *tv, - const OutputFilestoreLogThread *oft, const OutputFilestoreCtx *ctx, - const Packet *p, File *ff, uint8_t dir) { +static void OutputFilestoreFinalizeFiles(ThreadVars *tv, const OutputFilestoreLogThread *oft, + const OutputFilestoreCtx *ctx, const Packet *p, File *ff, void *tx, const uint64_t tx_id, + uint8_t dir) +{ /* Stringify the SHA256 which will be used in the final * filename. */ char sha256string[(SC_SHA256_LEN * 2) + 1]; @@ -160,7 +161,7 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv, "Failed to write file info record. Output filename truncated."); } else { JsonBuilder *js_fileinfo = - JsonBuildFileInfoRecord(p, ff, true, dir, ctx->xff_cfg, NULL); + JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL); if (likely(js_fileinfo != NULL)) { jb_close(js_fileinfo); FILE *out = fopen(js_metadata_filename, "w"); @@ -175,9 +176,9 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv, } } -static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, - const Packet *p, File *ff, const uint8_t *data, uint32_t data_len, - uint8_t flags, uint8_t dir) +static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p, File *ff, + void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags, + uint8_t dir) { SCEnter(); OutputFilestoreLogThread *aft = (OutputFilestoreLogThread *)thread_data; @@ -260,7 +261,7 @@ static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, ff->fd = -1; SC_ATOMIC_SUB(filestore_open_file_cnt, 1); } - OutputFilestoreFinalizeFiles(tv, aft, ctx, p, ff, dir); + OutputFilestoreFinalizeFiles(tv, aft, ctx, p, ff, tx, tx_id, dir); } return 0; diff --git a/src/output-json-alert.c b/src/output-json-alert.c index b15caa9f93..026d01b6f4 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -580,21 +580,26 @@ static void AlertAddAppLayer(const Packet *p, JsonBuilder *jb, static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id) { - FileContainer *ffc = AppLayerParserGetFiles(p->flow, - p->flowflags & FLOW_PKT_TOSERVER ? STREAM_TOSERVER:STREAM_TOCLIENT); + const uint8_t direction = + (p->flowflags & FLOW_PKT_TOSERVER) ? STREAM_TOSERVER : STREAM_TOCLIENT; + FileContainer *ffc = NULL; + if (p->flow->alstate != NULL) { + void *tx = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, tx_id); + if (tx) { + ffc = AppLayerParserGetTxFiles(p->flow, tx, direction); + } + } if (ffc != NULL) { File *file = ffc->head; bool isopen = false; while (file) { - if (tx_id == file->txid) { - if (!isopen) { - isopen = true; - jb_open_array(jb, "files"); - } - jb_start_object(jb); - EveFileInfo(jb, file, file->flags & FILE_STORED); - jb_close(jb); + if (!isopen) { + isopen = true; + jb_open_array(jb, "files"); } + jb_start_object(jb); + EveFileInfo(jb, file, tx_id, file->flags & FILE_STORED); + jb_close(jb); file = file->next; } if (isopen) { diff --git a/src/output-json-file.c b/src/output-json-file.c index 69b977c652..7ec4322268 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -80,8 +80,9 @@ typedef struct JsonFileLogThread_ { OutputJsonThreadCtx *ctx; } JsonFileLogThread; -JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool stored, - uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx) +JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, + const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg, + OutputJsonCtx *eve_ctx) { enum OutputJsonLogDirection fdir = LOG_DIR_FLOW; @@ -105,7 +106,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool char xff_buffer[XFF_MAXLEN]; if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED)) { if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP1) { - have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, xff_buffer, XFF_MAXLEN); + have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, xff_buffer, XFF_MAXLEN); } if (have_xff_ip && xff_cfg->flags & XFF_OVERWRITE) { if (p->flowflags & FLOW_PKT_TOCLIENT) { @@ -125,20 +126,20 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool switch (p->flow->alproto) { case ALPROTO_HTTP1: jb_open_object(js, "http"); - EveHttpAddMetadata(p->flow, ff->txid, js); + EveHttpAddMetadata(p->flow, tx_id, js); jb_close(js); break; case ALPROTO_SMTP: jb_get_mark(js, &mark); jb_open_object(js, "smtp"); - if (EveSMTPAddMetadata(p->flow, ff->txid, js)) { + if (EveSMTPAddMetadata(p->flow, tx_id, js)) { jb_close(js); } else { jb_restore_mark(js, &mark); } jb_get_mark(js, &mark); jb_open_object(js, "email"); - if (EveEmailAddMetadata(p->flow, ff->txid, js)) { + if (EveEmailAddMetadata(p->flow, tx_id, js)) { jb_close(js); } else { jb_restore_mark(js, &mark); @@ -148,7 +149,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool /* rpc */ jb_get_mark(js, &mark); jb_open_object(js, "rpc"); - if (EveNFSAddMetadataRPC(p->flow, ff->txid, js)) { + if (EveNFSAddMetadataRPC(p->flow, tx_id, js)) { jb_close(js); } else { jb_restore_mark(js, &mark); @@ -156,7 +157,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool /* nfs */ jb_get_mark(js, &mark); jb_open_object(js, "nfs"); - if (EveNFSAddMetadata(p->flow, ff->txid, js)) { + if (EveNFSAddMetadata(p->flow, tx_id, js)) { jb_close(js); } else { jb_restore_mark(js, &mark); @@ -165,7 +166,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool case ALPROTO_SMB: jb_get_mark(js, &mark); jb_open_object(js, "smb"); - if (EveSMBAddMetadata(p->flow, ff->txid, js)) { + if (EveSMBAddMetadata(p->flow, tx_id, js)) { jb_close(js); } else { jb_restore_mark(js, &mark); @@ -174,7 +175,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool case ALPROTO_HTTP2: jb_get_mark(js, &mark); jb_open_object(js, "http2"); - if (EveHTTP2AddMetadata(p->flow, ff->txid, js)) { + if (EveHTTP2AddMetadata(p->flow, tx_id, js)) { jb_close(js); } else { jb_restore_mark(js, &mark); @@ -185,7 +186,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool jb_set_string(js, "app_proto", AppProtoToString(p->flow->alproto)); jb_open_object(js, "fileinfo"); - EveFileInfo(js, ff, stored); + EveFileInfo(js, ff, tx_id, stored); jb_close(js); /* xff header */ @@ -200,13 +201,13 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool * \internal * \brief Write meta data on a single line json record */ -static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, - uint8_t dir, OutputJsonCtx *eve_ctx) +static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx, + const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) { HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ? aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg;; JsonBuilder *js = JsonBuildFileInfoRecord( - p, ff, ff->flags & FILE_STORED ? true : false, dir, xff_cfg, eve_ctx); + p, ff, tx, tx_id, ff->flags & FILE_STORED ? true : false, dir, xff_cfg, eve_ctx); if (unlikely(js == NULL)) { return; } @@ -215,8 +216,8 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F jb_free(js); } -static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, - const File *ff, uint8_t dir) +static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, + void *tx, const uint64_t tx_id, uint8_t dir) { SCEnter(); JsonFileLogThread *aft = (JsonFileLogThread *)thread_data; @@ -225,7 +226,7 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, SCLogDebug("ff %p", ff); - FileWriteJsonRecord(aft, p, ff, dir, aft->filelog_ctx->eve_ctx); + FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); return 0; } diff --git a/src/output-json-file.h b/src/output-json-file.h index 6aee745ffb..bdd6a86ba9 100644 --- a/src/output-json-file.h +++ b/src/output-json-file.h @@ -29,7 +29,8 @@ typedef struct OutputJsonCtx_ OutputJsonCtx; void JsonFileLogRegister(void); -JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool stored, - uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx); +JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, + const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg, + OutputJsonCtx *eve_ctx); #endif /* __OUTPUT_JSON_FILE_H__ */ diff --git a/src/output-json.c b/src/output-json.c index 5806627131..ef7fa58d10 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -127,7 +127,7 @@ json_t *SCJsonString(const char *val) /* Default Sensor ID value */ static int64_t sensor_id = -1; /* -1 = not defined */ -void EveFileInfo(JsonBuilder *jb, const File *ff, const bool stored) +void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const bool stored) { jb_set_string_from_bytes(jb, "filename", ff->name, ff->name_len); @@ -181,7 +181,7 @@ void EveFileInfo(JsonBuilder *jb, const File *ff, const bool stored) jb_set_uint(jb, "start", ff->start); jb_set_uint(jb, "end", ff->end); } - jb_set_uint(jb, "tx_id", ff->txid); + jb_set_uint(jb, "tx_id", tx_id); } static void EveAddPacketVars(const Packet *p, JsonBuilder *js_vars) diff --git a/src/output-json.h b/src/output-json.h index 671ec607b8..4999d2b580 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -95,7 +95,7 @@ typedef struct OutputJsonThreadCtx_ { json_t *SCJsonString(const char *val); void CreateEveFlowId(JsonBuilder *js, const Flow *f); -void EveFileInfo(JsonBuilder *js, const File *file, const bool stored); +void EveFileInfo(JsonBuilder *js, const File *file, const uint64_t tx_id, const bool stored); void EveTcpFlags(uint8_t flags, JsonBuilder *js); void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length); JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir, diff --git a/src/output-lua.c b/src/output-lua.c index 8a42a0387b..1f752d73cd 100644 --- a/src/output-lua.c +++ b/src/output-lua.c @@ -291,7 +291,8 @@ static int LuaPacketCondition(ThreadVars *tv, void *data, const Packet *p) * * NOTE p->flow is locked at this point */ -static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, uint8_t dir) +static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, + void *tx, const uint64_t tx_id, uint8_t dir) { SCEnter(); LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data; @@ -307,12 +308,7 @@ static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, con LuaStateSetThreadVars(td->lua_ctx->luastate, tv); LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p); - if (p->flow && p->flow->alstate) { - void *txptr = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, ff->txid); - if (txptr) { - LuaStateSetTX(td->lua_ctx->luastate, txptr, ff->txid); - } - } + LuaStateSetTX(td->lua_ctx->luastate, tx, tx_id); LuaStateSetFlow(td->lua_ctx->luastate, p->flow); LuaStateSetFile(td->lua_ctx->luastate, (File *)ff); diff --git a/src/output-tx.c b/src/output-tx.c index 635b18506f..310455fd8d 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -36,6 +36,11 @@ * data for the packet loggers. */ typedef struct OutputTxLoggerThreadData_ { OutputLoggerThreadStore *store[ALPROTO_MAX]; + + /* thread local data from file api */ + OutputFileLoggerThreadData *file; + /* thread local data from filedata api */ + OutputFiledataLoggerThreadData *filedata; } OutputTxLoggerThreadData; /* logger instance, a module + a output ctx, @@ -68,8 +73,8 @@ int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, void (*ThreadExitPrintStats)(ThreadVars *, void *)) { if (alproto != ALPROTO_UNKNOWN && !(AppLayerParserIsEnabled(alproto))) { - SCLogNotice("%s logger not enabled: protocol %s is disabled", - name, AppProtoToString(alproto)); + SCLogDebug( + "%s logger not enabled: protocol %s is disabled", name, AppProtoToString(alproto)); return -1; } OutputTxLogger *op = SCMalloc(sizeof(*op)); @@ -125,6 +130,110 @@ int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto, return 0; } +extern bool g_file_logger_enabled; +extern bool g_filedata_logger_enabled; + +/** \brief run the per tx file logging + * \todo clean up various end of tx/stream etc indicators + */ +static inline void OutputTxLogFiles(ThreadVars *tv, OutputFileLoggerThreadData *file_td, + OutputFiledataLoggerThreadData *filedata_td, Packet *p, Flow *f, void *tx, + const uint64_t tx_id, AppLayerTxData *txd, const bool tx_complete, const bool ts_ready, + const bool tc_ready, const bool ts_eof, const bool tc_eof, const bool eof) +{ + int packet_dir; + int opposing_dir; + bool packet_dir_ready; + const bool opposing_dir_ready = eof; + bool opposing_tx_ready; + if (p->flowflags & FLOW_PKT_TOSERVER) { + packet_dir = STREAM_TOSERVER; + opposing_dir = STREAM_TOCLIENT; + packet_dir_ready = eof | ts_ready | ts_eof; + opposing_tx_ready = tc_ready; + } else if (p->flowflags & FLOW_PKT_TOCLIENT) { + packet_dir = STREAM_TOCLIENT; + opposing_dir = STREAM_TOSERVER; + packet_dir_ready = eof | tc_ready | tc_eof; + opposing_tx_ready = ts_ready; + } else { + abort(); + } + + SCLogDebug("eof %d ts_ready %d ts_eof %d", eof, ts_ready, ts_eof); + SCLogDebug("eof %d tc_ready %d tc_eof %d", eof, tc_ready, tc_eof); + + SCLogDebug("packet dir %s opposing %s packet_dir_ready %d opposing_dir_ready %d", + packet_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT", + opposing_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT", packet_dir_ready, + opposing_dir_ready); + + FileContainer *ffc = AppLayerParserGetTxFiles(f, tx, packet_dir); + FileContainer *ffc_opposing = AppLayerParserGetTxFiles(f, tx, opposing_dir); + + /* see if opposing side is finished: if no file support in this direction, of is not + * files and tx is done for opposing dir. */ + bool opposing_finished = + ffc_opposing == NULL || (ffc_opposing->head == NULL && opposing_tx_ready); + SCLogDebug("opposing_finished %d ffc_opposing %p ffc_opposing->head %p opposing_tx_ready %d", + opposing_finished, ffc_opposing, ffc_opposing ? ffc_opposing->head : NULL, + opposing_tx_ready); + + if (ffc || ffc_opposing) + SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64 + " ffc %p ffc_opposing %p tx_complete %d", + p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete); + + if (ffc) { + const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | eof; + const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof; + SCLogDebug("tx: calling files: ffc %p head %p file_close %d file_trunc %d", ffc, ffc->head, + file_close, file_trunc); + if (filedata_td) + OutputFiledataLogFfc(tv, filedata_td, p, ffc, tx, tx_id, txd, packet_dir, file_close, + file_trunc, packet_dir); + if (file_td) + OutputFileLogFfc( + tv, file_td, p, ffc, tx, tx_id, txd, file_close, file_trunc, packet_dir); + } + /* if EOF and we support files, do a final write out */ + if (opposing_dir_ready && ffc_opposing != NULL) { + const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | tx_complete | eof; + const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof; + opposing_finished = true; + SCLogDebug("tx: calling for opposing direction files: file_close:%s file_trunc:%s", + file_close ? "true" : "false", file_trunc ? "true" : "false"); + if (filedata_td) + OutputFiledataLogFfc(tv, filedata_td, p, ffc_opposing, tx, tx_id, txd, opposing_dir, + file_close, file_trunc, opposing_dir); + if (file_td) + OutputFileLogFfc(tv, file_td, p, ffc_opposing, tx, tx_id, txd, file_close, file_trunc, + opposing_dir); + } + + const bool tx_done = packet_dir_ready && opposing_finished; + SCLogDebug("tx_done %d packet_dir_ready %d opposing_finished %d", tx_done, packet_dir_ready, + opposing_finished); + + /* if not a file tx or if tx is done, set logger flags so tx can move on */ + const bool is_file_tx = (ffc != NULL || ffc_opposing != NULL); + if (!is_file_tx || tx_done) { + SCLogDebug("is_file_tx %d tx_done %d", is_file_tx, tx_done); + if (file_td) { + txd->logged.flags |= BIT_U32(LOGGER_FILE); + SCLogDebug("setting LOGGER_FILE => %08x", txd->logged.flags); + } + if (filedata_td) { + txd->logged.flags |= BIT_U32(LOGGER_FILEDATA); + SCLogDebug("setting LOGGER_FILEDATA => %08x", txd->logged.flags); + } + } else { + SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64 + " NOT SETTING FILE FLAGS ffc %p ffc_opposing %p tx_complete %d", + p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete); + } +} + static void OutputTxLogList0(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data, Packet *p, Flow *f, void *tx, const uint64_t tx_id) { @@ -154,6 +263,73 @@ static void OutputTxLogList0(ThreadVars *tv, OutputTxLoggerThreadData *op_thread } } +struct Ctx { + uint32_t tx_logged_old; + uint32_t tx_logged; +}; + +static void OutputTxLogCallLoggers(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data, + const OutputTxLogger *logger, const OutputLoggerThreadStore *store, Packet *p, Flow *f, + void *alstate, void *tx, const uint64_t tx_id, const AppProto alproto, const bool eof, + const int tx_progress_ts, const int tx_progress_tc, struct Ctx *ctx) +{ + DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL); + DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL); + // DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL); + + while (logger && store) { + DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL); + DEBUG_VALIDATE_BUG_ON(logger->alproto != alproto); + + SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d " + "tc_log_progress %d", + logger, logger->alproto, logger->LogCondition, logger->ts_log_progress, + logger->tc_log_progress); + if ((ctx->tx_logged_old & BIT_U32(logger->logger_id)) == 0) { + SCLogDebug("alproto match %d, logging tx_id %" PRIu64, logger->alproto, tx_id); + + SCLogDebug("pcap_cnt %" PRIu64 ", tx_id %" PRIu64 " logger %d. EOF %s", p->pcap_cnt, + tx_id, logger->logger_id, eof ? "true" : "false"); + + if (eof) { + SCLogDebug("EOF, so log now"); + } else { + if (logger->LogCondition) { + int r = logger->LogCondition(tv, p, alstate, tx, tx_id); + if (r == FALSE) { + SCLogDebug("conditions not met, not logging"); + goto next_logger; + } + } else { + if (tx_progress_tc < logger->tc_log_progress) { + SCLogDebug("progress not far enough, not logging"); + goto next_logger; + } + + if (tx_progress_ts < logger->ts_log_progress) { + SCLogDebug("progress not far enough, not logging"); + goto next_logger; + } + } + } + + SCLogDebug("Logging tx_id %" PRIu64 " to logger %d", tx_id, logger->logger_id); + PACKET_PROFILING_LOGGER_START(p, logger->logger_id); + logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id); + PACKET_PROFILING_LOGGER_END(p, logger->logger_id); + + ctx->tx_logged |= BIT_U32(logger->logger_id); + } + + next_logger: + logger = logger->next; + store = store->next; + + DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL); + DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL); + } +} + static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) { DEBUG_VALIDATE_BUG_ON(thread_data == NULL); @@ -165,35 +341,52 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) Flow * const f = p->flow; const uint8_t ipproto = f->proto; const AppProto alproto = f->alproto; - - if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) { - /* No child loggers registered. */ - return TM_ECODE_OK; + SCLogDebug("pcap_cnt %u tx logging %u/%s", (uint32_t)p->pcap_cnt, alproto, + AppProtoToString(alproto)); + + if (op_thread_data->file == NULL && op_thread_data->filedata == NULL) { + if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) { + SCLogDebug("bail"); + /* No child loggers registered. */ + return TM_ECODE_OK; + } + if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0) + goto end; } - - if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0) - goto end; const LoggerId logger_expectation = AppLayerParserProtocolGetLoggerBits(p->proto, alproto); - if (logger_expectation == 0) + if (logger_expectation == 0) { + SCLogDebug("bail: logger_expectation %u. LOGGER_FILE %u LOGGER_FILEDATA %u", + logger_expectation, LOGGER_FILE, LOGGER_FILEDATA); goto end; - + } void *alstate = f->alstate; if (alstate == NULL) { SCLogDebug("no alstate"); goto end; } + SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt); + const bool last_pseudo = (p->flowflags & FLOW_PKT_LAST_PSEUDO) != 0; - const bool ts_eof = AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF_TS) != 0; - const bool tc_eof = AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF_TC) != 0; + const bool ts_eof = AppLayerParserStateIssetFlag(f->alparser, + (APP_LAYER_PARSER_EOF_TS | APP_LAYER_PARSER_TRUNC_TS)) != 0; + const bool tc_eof = AppLayerParserStateIssetFlag(f->alparser, + (APP_LAYER_PARSER_EOF_TC | APP_LAYER_PARSER_TRUNC_TC)) != 0; + + const bool eof = last_pseudo || (ts_eof && tc_eof); + SCLogDebug("eof %d last_pseudo %d ts_eof %d tc_eof %d", eof, last_pseudo, ts_eof, tc_eof); + const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER); const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT); + SCLogDebug("ts_disrupt_flags %02x tc_disrupt_flags %02x", ts_disrupt_flags, tc_disrupt_flags); const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate); uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser); uint64_t max_id = tx_id; int logged = 0; int gap = 0; + SCLogDebug("tx_id %" PRIu64 " total_txs %" PRIu64, tx_id, total_txs); + AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto); AppLayerGetTxIterState state; memset(&state, 0, sizeof(state)); @@ -204,8 +397,34 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) break; void * const tx = ires.tx_ptr; tx_id = ires.tx_id; + SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx); + + const int tx_progress_ts = + AppLayerParserGetStateProgress(p->proto, alproto, tx, ts_disrupt_flags); + const int tx_progress_tc = + AppLayerParserGetStateProgress(p->proto, alproto, tx, tc_disrupt_flags); + const bool tx_complete = (tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus( + alproto, STREAM_TOSERVER) && + tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus( + alproto, STREAM_TOCLIENT)); + const bool ts_ready = tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus( + alproto, STREAM_TOSERVER); + const bool tc_ready = tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus( + alproto, STREAM_TOCLIENT); + SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready); + + SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file, + op_thread_data->filedata); AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx); + if (txd && (op_thread_data->file || op_thread_data->filedata) && + AppLayerParserSupportsFiles(p->proto, alproto)) { + OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx, tx_id, + txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof); + } + if (txd) + SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags); + if (txd == NULL) { /* make sure this tx, which can't be properly logged is skipped */ logged = 1; @@ -219,93 +438,33 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) goto next_tx; } - SCLogDebug("tx %p/%"PRIu64" txd %p: log_flags %x", tx, tx_id, txd, txd->config.log_flags); + SCLogDebug("tx %p/%" PRIu64 " txd %p: log_flags %x logger_expectation %x", tx, tx_id, txd, + txd->config.log_flags, logger_expectation); if (txd->config.log_flags & BIT_U8(CONFIG_TYPE_TX)) { SCLogDebug("SKIP tx %p/%"PRIu64, tx, tx_id); goto next_tx; } - LoggerId tx_logged = txd->logged.flags; - const LoggerId tx_logged_old = tx_logged; - SCLogDebug("logger: expect %08x, have %08x", logger_expectation, tx_logged); - if (tx_logged == logger_expectation) { + if (txd->logged.flags == logger_expectation) { + SCLogDebug("fully logged"); /* tx already fully logged */ goto next_tx; } - const int tx_progress_ts = - AppLayerParserGetStateProgress(p->proto, alproto, tx, ts_disrupt_flags); - const int tx_progress_tc = - AppLayerParserGetStateProgress(p->proto, alproto, tx, tc_disrupt_flags); - SCLogDebug("tx_progress_ts %d tx_progress_tc %d", - tx_progress_ts, tx_progress_tc); - + SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags); const OutputTxLogger *logger = list[alproto]; const OutputLoggerThreadStore *store = op_thread_data->store[alproto]; + struct Ctx ctx = { .tx_logged = txd->logged.flags, .tx_logged_old = txd->logged.flags }; + SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged); - DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL); - DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL); - DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL); + OutputTxLogCallLoggers(tv, op_thread_data, logger, store, p, f, alstate, tx, tx_id, alproto, + eof, tx_progress_ts, tx_progress_tc, &ctx); - while (logger && store) { - DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL); - DEBUG_VALIDATE_BUG_ON(logger->alproto != alproto); - - SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d " - "tc_log_progress %d", logger, logger->alproto, logger->LogCondition, - logger->ts_log_progress, logger->tc_log_progress); - if ((tx_logged_old & BIT_U32(logger->logger_id)) == 0) { - SCLogDebug("alproto match %d, logging tx_id %"PRIu64, logger->alproto, tx_id); - - SCLogDebug("pcap_cnt %"PRIu64", tx_id %"PRIu64" logger %d. " - "EOFs TS %s TC %s LAST PSEUDO %s", - p->pcap_cnt, tx_id, logger->logger_id, - ts_eof ? "true" : "false", tc_eof ? "true" : "false", - last_pseudo ? "true" : "false"); - - if ((ts_eof && tc_eof) || last_pseudo) { - SCLogDebug("EOF, so log now"); - } else { - if (logger->LogCondition) { - int r = logger->LogCondition(tv, p, alstate, tx, tx_id); - if (r == FALSE) { - SCLogDebug("conditions not met, not logging"); - goto next_logger; - } - } else { - if (tx_progress_tc < logger->tc_log_progress) { - SCLogDebug("progress not far enough, not logging"); - goto next_logger; - } - - if (tx_progress_ts < logger->ts_log_progress) { - SCLogDebug("progress not far enough, not logging"); - goto next_logger; - } - } - } - - SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id, logger->logger_id); - PACKET_PROFILING_LOGGER_START(p, logger->logger_id); - logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id); - PACKET_PROFILING_LOGGER_END(p, logger->logger_id); - - tx_logged |= BIT_U32(logger->logger_id); - } - -next_logger: - logger = logger->next; - store = store->next; - - DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL); - DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL); - } - - if (tx_logged != tx_logged_old) { - SCLogDebug("logger: storing %08x (was %08x)", - tx_logged, tx_logged_old); + SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged); + if (ctx.tx_logged != ctx.tx_logged_old) { + SCLogDebug("logger: storing %08x (was %08x)", ctx.tx_logged, ctx.tx_logged_old); DEBUG_VALIDATE_BUG_ON(txd == NULL); - txd->logged.flags |= tx_logged; + txd->logged.flags |= ctx.tx_logged; } /* If all loggers logged set a flag and update the last tx_id @@ -314,9 +473,11 @@ next_logger: * If not all loggers were logged we flag that there was a gap * so any subsequent transactions in this loop don't increase * the maximum ID that was logged. */ - if (!gap && tx_logged == logger_expectation) { + if (!gap && ctx.tx_logged == logger_expectation) { + SCLogDebug("no gap %d, %08x == %08x", gap, ctx.tx_logged, logger_expectation); logged = 1; max_id = tx_id; + SCLogDebug("max_id %" PRIu64, max_id); } else { gap = 1; } @@ -340,7 +501,7 @@ end: /** \brief thread init for the tx logger * This will run the thread init functions for the individual registered * loggers */ -static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *initdata, void **data) +static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *_initdata, void **data) { OutputTxLoggerThreadData *td = SCMalloc(sizeof(*td)); if (td == NULL) @@ -379,6 +540,20 @@ static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *initdata, void logger = logger->next; } } + + if (g_file_logger_enabled) { + if (OutputFileLogThreadInit(tv, &td->file) != TM_ECODE_OK) { + FatalError(SC_ERR_FATAL, "failed to set up file thread data"); + } + } + if (g_filedata_logger_enabled) { + if (OutputFiledataLogThreadInit(tv, &td->filedata) != TM_ECODE_OK) { + FatalError(SC_ERR_FATAL, "failed to set up filedata thread data"); + } + } + + SCLogDebug("file_thread_data %p filedata_thread_data %p", td->file, td->filedata); + return TM_ECODE_OK; } @@ -402,6 +577,13 @@ static TmEcode OutputTxLogThreadDeinit(ThreadVars *tv, void *thread_data) } } + if (op_thread_data->file) { + OutputFileLogThreadDeinit(tv, op_thread_data->file); + } + if (op_thread_data->filedata) { + OutputFiledataLogThreadDeinit(tv, op_thread_data->filedata); + } + SCFree(op_thread_data); return TM_ECODE_OK; } @@ -433,6 +615,16 @@ static uint32_t OutputTxLoggerGetActiveCount(void) cnt++; } } + + if (g_file_logger_enabled) { + cnt++; + SCLogDebug("g_file_logger_enabled"); + } + if (g_filedata_logger_enabled) { + cnt++; + SCLogDebug("g_filedata_logger_enabled"); + } + return cnt; } diff --git a/src/output.c b/src/output.c index 74f6308a01..bcf4e26255 100644 --- a/src/output.c +++ b/src/output.c @@ -887,7 +887,6 @@ TmEcode OutputLoggerLog(ThreadVars *tv, Packet *p, void *thread_data) logger = TAILQ_NEXT(logger, entries); thread_store_node = TAILQ_NEXT(thread_store_node, entries); } - return TM_ECODE_OK; } @@ -1029,9 +1028,9 @@ void TmModuleLoggerRegister(void) void OutputRegisterRootLoggers(void) { OutputPacketLoggerRegister(); - OutputTxLoggerRegister(); OutputFiledataLoggerRegister(); OutputFileLoggerRegister(); + OutputTxLoggerRegister(); OutputStreamingLoggerRegister(); } diff --git a/src/output.h b/src/output.h index 62d3f6b90f..e7234ddfbc 100644 --- a/src/output.h +++ b/src/output.h @@ -27,6 +27,11 @@ #define DEFAULT_LOG_MODE_APPEND "yes" #define DEFAULT_LOG_FILETYPE "regular" +typedef struct OutputLoggerThreadStore_ { + void *thread_data; + struct OutputLoggerThreadStore_ *next; +} OutputLoggerThreadStore; + #include "output-packet.h" #include "output-tx.h" #include "output-file.h" @@ -35,12 +40,6 @@ #include "output-streaming.h" #include "output-stats.h" - -typedef struct OutputLoggerThreadStore_ { - void *thread_data; - struct OutputLoggerThreadStore_ *next; -} OutputLoggerThreadStore; - typedef struct OutputInitResult_ { OutputCtx *ctx; bool ok; diff --git a/src/runmodes.c b/src/runmodes.c index 8f729a46a9..9278652abd 100644 --- a/src/runmodes.c +++ b/src/runmodes.c @@ -629,7 +629,7 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu module->ThreadExitPrintStats); /* Not used with wild card loggers */ if (module->alproto != ALPROTO_UNKNOWN) { - logger_bits[module->alproto] |= (1<logger_id); + logger_bits[module->alproto] |= BIT_U32(module->logger_id); } } else if (module->FiledataLogFunc) { SCLogDebug("%s is a filedata logger", module->name); @@ -757,6 +757,9 @@ static void RunModeInitializeLuaOutput(ConfNode *conf, OutputCtx *parent_ctx) } } +extern bool g_file_logger_enabled; +extern bool g_filedata_logger_enabled; + /** * Initialize the output modules. */ @@ -919,11 +922,31 @@ void RunModeInitializeOutputs(void) /* register the logger bits to the app-layer */ AppProto a; for (a = 0; a < ALPROTO_MAX; a++) { + if (AppLayerParserSupportsFiles(IPPROTO_TCP, a)) { + if (g_file_logger_enabled) + logger_bits[a] |= BIT_U32(LOGGER_FILE); + if (g_filedata_logger_enabled) + logger_bits[a] |= BIT_U32(LOGGER_FILEDATA); + SCLogDebug("IPPROTO_TCP::%s: g_file_logger_enabled %d g_filedata_logger_enabled %d -> " + "%08x", + AppProtoToString(a), g_file_logger_enabled, g_filedata_logger_enabled, + logger_bits[a]); + } + if (AppLayerParserSupportsFiles(IPPROTO_UDP, a)) { + if (g_file_logger_enabled) + logger_bits[a] |= BIT_U32(LOGGER_FILE); + if (g_filedata_logger_enabled) + logger_bits[a] |= BIT_U32(LOGGER_FILEDATA); + } + if (logger_bits[a] == 0) continue; - const int tcp = AppLayerParserProtocolHasLogger(IPPROTO_TCP, a); - const int udp = AppLayerParserProtocolHasLogger(IPPROTO_UDP, a); + const int tcp = AppLayerParserProtocolHasLogger(IPPROTO_TCP, a) | (g_file_logger_enabled) | + (g_filedata_logger_enabled); + const int udp = AppLayerParserProtocolHasLogger(IPPROTO_UDP, a) | (g_file_logger_enabled) | + (g_filedata_logger_enabled); + SCLogDebug("tcp %d udp %d", tcp, udp); SCLogDebug("logger for %s: %s %s", AppProtoToString(a), tcp ? "true" : "false", udp ? "true" : "false"); diff --git a/src/rust-context.c b/src/rust-context.c index 0e3dd26390..25f3fe59bf 100644 --- a/src/rust-context.c +++ b/src/rust-context.c @@ -38,7 +38,6 @@ const SuricataContext suricata_context = { FileAppendGAPById, FileContainerRecycle, FilePrune, - FileContainerSetTx, AppLayerRegisterParser, }; diff --git a/src/rust-context.h b/src/rust-context.h index a832fc06fe..785175e632 100644 --- a/src/rust-context.h +++ b/src/rust-context.h @@ -54,7 +54,6 @@ typedef struct SuricataContext_ { const uint8_t *data, uint32_t data_len); void (*FileContainerRecycle)(FileContainer *ffc); void (*FilePrune)(FileContainer *ffc); - void (*FileSetTx)(FileContainer *, uint64_t); int (*AppLayerRegisterParser)(const struct AppLayerParser *p, AppProto alproto); diff --git a/src/suricata-common.h b/src/suricata-common.h index 6ed46a9196..1e0b8698d0 100644 --- a/src/suricata-common.h +++ b/src/suricata-common.h @@ -472,6 +472,9 @@ typedef enum { LOGGER_JSON_DCERPC, LOGGER_JSON_HTTP2, + LOGGER_FILE, + LOGGER_FILEDATA, + /** \warning when we exceed what we can express as a u32 flag here we need to update * LoggerFlags::flags (u32) and `tx_logged` in src/output-tx.c */ diff --git a/src/util-file.c b/src/util-file.c index 3fbf74d67f..fdf504ad8f 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -289,6 +289,25 @@ uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction) return FileFlowFlagsToFlags(flow->file_flags, direction); } +void FileApplyTxFlags(const AppLayerTxData *txd, const uint8_t direction, File *file) +{ + SCLogDebug("file flags %04x STORE %s NOSTORE %s", file->flags, + (file->flags & FILE_STORE) ? "true" : "false", + (file->flags & FILE_NOSTORE) ? "true" : "false"); + uint16_t update_flags = FileFlowFlagsToFlags(txd->file_flags, direction); + DEBUG_VALIDATE_BUG_ON( + (file->flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE)); + if (file->flags & FILE_STORE) + update_flags &= ~FILE_NOSTORE; + + file->flags |= update_flags; + SCLogDebug("file flags %04x STORE %s NOSTORE %s", file->flags, + (file->flags & FILE_STORE) ? "true" : "false", + (file->flags & FILE_NOSTORE) ? "true" : "false"); + DEBUG_VALIDATE_BUG_ON( + (file->flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE)); +} + static int FileMagicSize(void) { /** \todo make this size configurable */ @@ -608,27 +627,6 @@ int FileStore(File *ff) SCReturnInt(0); } -/** - * \brief Set the TX id for a file - * - * \param ff The file to store - * \param txid the tx id - */ -int FileSetTx(File *ff, uint64_t txid) -{ - SCLogDebug("ff %p txid %"PRIu64, ff, txid); - if (ff != NULL) - ff->txid = txid; - SCReturnInt(0); -} - -void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id) -{ - if (ffc && ffc->tail) { - (void)FileSetTx(ffc->tail, tx_id); - } -} - /** * \brief check if we have stored enough * @@ -1129,78 +1127,17 @@ void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction f->file_flags, set_file_flags, g_file_flow_mask); if (set_file_flags != 0 && f->alproto != ALPROTO_UNKNOWN && f->alstate != NULL) { - uint16_t per_file_flags = 0; -#ifdef HAVE_MAGIC - if (set_file_flags & (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC)) - per_file_flags |= FILE_NOMAGIC; -#endif - if (set_file_flags & (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC)) - per_file_flags |= FILE_NOMD5; - if (set_file_flags & (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC)) - per_file_flags |= FILE_NOSHA1; - if (set_file_flags & (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC)) - per_file_flags |= FILE_NOSHA256; - if (set_file_flags & (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC)) - per_file_flags |= FILE_NOTRACK; - if (set_file_flags & (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC)) - per_file_flags |= FILE_NOSTORE; - - FileContainer *ffc = AppLayerParserGetFiles(f, direction); - if (ffc != NULL) { - for (File *ptr = ffc->head; ptr != NULL; ptr = ptr->next) { - ptr->flags |= per_file_flags; - - /* destroy any ctx we may have so far */ - if ((per_file_flags & FILE_NOSHA256) && - ptr->sha256_ctx != NULL) - { - SCSha256Free(ptr->sha256_ctx); - ptr->sha256_ctx = NULL; - } - if ((per_file_flags & FILE_NOSHA1) && - ptr->sha1_ctx != NULL) - { - SCSha1Free(ptr->sha1_ctx); - ptr->sha1_ctx = NULL; - } - if ((per_file_flags & FILE_NOMD5) && - ptr->md5_ctx != NULL) - { - SCMd5Free(ptr->md5_ctx); - ptr->md5_ctx = NULL; - } + AppLayerStateData *sd = AppLayerParserGetStateData(f->proto, f->alproto, f->alstate); + if (sd != NULL) { + if ((sd->file_flags & f->file_flags) != f->file_flags) { + SCLogDebug("state data: updating file_flags %04x with flow file_flags %04x", + sd->file_flags, f->file_flags); + sd->file_flags |= f->file_flags; } } } } - - -/** - * \brief set no store flag, close file if needed - * - * \param ff file - */ -static void FileDisableStoringForFile(File *ff) -{ - SCEnter(); - - if (ff == NULL) { - SCReturn; - } - - SCLogDebug("not storing this file"); - ff->flags |= FILE_NOSTORE; - - if (ff->state == FILE_STATE_OPENED && FileDataSize(ff) >= (uint64_t)FileMagicSize()) { - if (g_file_force_md5 == 0 && g_file_force_sha1 == 0 && g_file_force_sha256 == 0 - && g_file_force_tracking == 0) { - (void)FileCloseFilePtr(ff, NULL, 0, - (FILE_TRUNCATED|FILE_NOSTORE)); - } - } -} - /** * \brief disable file storing for files in a transaction * @@ -1208,29 +1145,16 @@ static void FileDisableStoringForFile(File *ff) * \param direction flow direction * \param tx_id transaction id */ -void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id) +void FileDisableStoringForTransaction(Flow *f, const uint8_t direction, void *tx, uint64_t tx_id) { - File *ptr = NULL; - - DEBUG_ASSERT_FLOW_LOCKED(f); - - SCEnter(); - - FileContainer *ffc = AppLayerParserGetFiles(f, direction); - if (ffc != NULL) { - for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) { - if (ptr->txid == tx_id) { - if (ptr->flags & FILE_STORE) { - /* weird, already storing -- let it continue*/ - SCLogDebug("file is already being stored"); - } else { - FileDisableStoringForFile(ptr); - } - } + AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, tx); + if (txd != NULL) { + if (direction & STREAM_TOSERVER) { + txd->file_flags |= FLOWFILE_NO_STORE_TS; + } else { + txd->file_flags |= FLOWFILE_NO_STORE_TC; } } - - SCReturn; } /** @@ -1256,17 +1180,7 @@ void FileStoreFileById(FileContainer *fc, uint32_t file_id) void FileStoreAllFilesForTx(FileContainer *fc, uint64_t tx_id) { - File *ptr = NULL; - - SCEnter(); - - if (fc != NULL) { - for (ptr = fc->head; ptr != NULL; ptr = ptr->next) { - if (ptr->txid == tx_id) { - FileStore(ptr); - } - } - } + abort(); } void FileStoreAllFiles(FileContainer *fc) diff --git a/src/util-file.h b/src/util-file.h index 3f1cbed9d4..07c03a4419 100644 --- a/src/util-file.h +++ b/src/util-file.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2007-2011 Open Information Security Foundation +/* Copyright (C) 2007-2021 Open Information Security Foundation * * You can copy, redistribute or modify this Program under the terms of * the GNU General Public License version 2 as published by the Free @@ -26,7 +26,6 @@ #define __UTIL_FILE_H__ #include "conf.h" - #include "util-streaming-buffer.h" /* Hack: Pulling rust.h to get the SCSha256 causes all sorts of problems with @@ -77,7 +76,6 @@ typedef struct File_ { uint16_t name_len; FileState state; StreamingBuffer *sb; - uint64_t txid; /**< tx this file is part of */ uint32_t file_track_id; /**< id used by protocol parser */ uint32_t file_store_id; /**< id used in store file name file. */ int fd; /**< file descriptor for filestore, not @@ -199,22 +197,15 @@ int FileSetRange(FileContainer *, uint64_t start, uint64_t end); */ int FileStore(File *); -/** - * \brief Set the TX id for a file - * - * \param ff The file to store - * \param txid the tx id - */ -int FileSetTx(File *, uint64_t txid); -void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id); - /** * \brief disable file storing for a transaction * * \param f flow + * \param direction STREAM_TOSERVER or STREAM_TOCLIENT + * \param tx transaction pointer * \param tx_id transaction id */ -void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id); +void FileDisableStoringForTransaction(Flow *f, const uint8_t direction, void *tx, uint64_t tx_id); void FlowFileDisableStoringForTransaction(struct Flow_ *f, uint64_t tx_id); void FilePrune(FileContainer *ffc); diff --git a/src/util-profiling.c b/src/util-profiling.c index d020e73f71..615b40b936 100644 --- a/src/util-profiling.c +++ b/src/util-profiling.c @@ -1308,6 +1308,10 @@ const char * PacketProfileLoggertIdToString(LoggerId id) CASE_CODE (LOGGER_JSON_STATS); CASE_CODE (LOGGER_PCAP); CASE_CODE (LOGGER_JSON_METADATA); + + CASE_CODE(LOGGER_FILE); + CASE_CODE(LOGGER_FILEDATA); + case LOGGER_SIZE: return "UNKNOWN"; }