aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

app-layer: move files into transactions

Browse Source 
Update APIs to store files in transactions instead of the per flow state.

Goal is to avoid the overhead of matching up files and transactions in
cases where there are many of both.

Update all protocol implementations to support this.

Update file logging logic to account for having files in transactions. Instead
of it acting separately on file containers, it is now tied into the
transaction logging.

Update the filestore keyword to consider a match if filestore output not
enabled.
pull/7954/head
Victor Julien 5 years ago
parent 01e64d80da
commit 79499e4769
74 changed files with 1240 additions and 1303 deletions
Show Stats Download Patch File Download Diff File

15
rust/src/applayer.rs
Unescape Escape View File

@ -109,6 +109,8 @@ pub struct AppLayerTxData {
pub files_logged: u32,
pub files_stored: u32,
pub file_flags: u16,
/// detection engine flags for use by detection engine
detect_flags_ts: u64,
detect_flags_tc: u64,
@ -142,6 +144,7 @@ impl AppLayerTxData {
files_opened: 0,
files_logged: 0,
files_stored: 0,
file_flags: 0,
detect_flags_ts: 0,
detect_flags_tc: 0,
de_state: std::ptr::null_mut(),
@ -158,6 +161,13 @@ impl AppLayerTxData {
pub fn set_event(&mut self, event: u8) {
core::sc_app_layer_decoder_events_set_event_raw(&mut self.events, event as u8);
}
pub fn update_file_flags(&mut self, state_flags: u16) {
if (self.file_flags & state_flags) != state_flags {
SCLogDebug!("updating tx file_flags {:04x} with state flags {:04x}", self.file_flags, state_flags);
self.file_flags |= state_flags;
}
}
}
#[macro_export]
@ -322,7 +332,7 @@ pub struct RustParser {
pub localstorage_free: Option<LocalStorageFreeFn>,
/// Function to get files
pub get_files: Option<GetFilesFn>,
pub get_tx_files: Option<GetTxFilesFn>,
/// Function to get the TX iterator
pub get_tx_iterator: Option<GetTxIteratorFn>,
@ -378,8 +388,7 @@ pub type GetEventInfoFn = unsafe extern "C" fn (*const c_char, *mut c_int, *
pub type GetEventInfoByIdFn = unsafe extern "C" fn (c_int, *mut *const c_char, *mut AppLayerEventType) -> i8;
pub type LocalStorageNewFn = extern "C" fn () -> *mut c_void;
pub type LocalStorageFreeFn = extern "C" fn (*mut c_void);
pub type GetFilesFn = unsafe
extern "C" fn (*mut c_void, u8) -> *mut FileContainer;
pub type GetTxFilesFn = unsafe extern "C" fn (*mut c_void, u8) -> *mut FileContainer;
pub type GetTxIteratorFn = unsafe extern "C" fn (ipproto: u8, alproto: AppProto,
state: *mut c_void,
min_tx_id: u64,

2
rust/src/applayertemplate/template.rs
Unescape Escape View File

@ -455,7 +455,7 @@ pub unsafe extern "C" fn rs_template_register_parser() {
get_eventinfo_byid : Some(TemplateEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<TemplateState, TemplateTransaction>),
get_tx_data: rs_template_get_tx_data,
get_state_data: rs_template_get_state_data,

8
rust/src/core.rs
Unescape Escape View File

@ -94,10 +94,11 @@ pub static mut ALPROTO_FAILED : AppProto = 0; // updated during init
pub const IPPROTO_TCP : u8 = 6;
pub const IPPROTO_UDP : u8 = 17;
/*
macro_rules!BIT_U8 {
($x:expr) => (1 << $x);
}
*/
macro_rules!BIT_U16 {
($x:expr) => (1 << $x);
}
@ -181,10 +182,6 @@ pub type SCFilePrune = extern "C" fn (
pub type SCFileContainerRecycle = extern "C" fn (
file_container: &FileContainer);
pub type SCFileSetTx = extern "C" fn (
file: &FileContainer,
tx_id: u64);
// A Suricata context that is passed in from C. This is alternative to
// using functions from Suricata directly, so they can be wrapped so
// Rust unit tests will still compile when they are not linked
@ -210,7 +207,6 @@ pub struct SuricataContext {
pub FileAppendGAP: SCFileAppendGAPById,
pub FileContainerRecycle: SCFileContainerRecycle,
pub FilePrune: SCFilePrune,
pub FileSetTx: SCFileSetTx,
pub AppLayerRegisterParser: extern fn(parser: *const crate::applayer::RustParser, alproto: AppProto) -> std::os::raw::c_int,
}

2
rust/src/dcerpc/dcerpc.rs
Unescape Escape View File

@ -1367,7 +1367,7 @@ pub unsafe extern "C" fn rs_dcerpc_register_parser() {
get_eventinfo_byid : None,
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<DCERPCState, DCERPCTransaction>),
get_tx_data: rs_dcerpc_get_tx_data,
get_state_data: rs_dcerpc_get_state_data,

2
rust/src/dcerpc/dcerpc_udp.rs
Unescape Escape View File

@ -360,7 +360,7 @@ pub unsafe extern "C" fn rs_dcerpc_udp_register_parser() {
get_eventinfo_byid: None,
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<DCERPCUDPState, DCERPCTransaction>),
get_tx_data: rs_dcerpc_udp_get_tx_data,
get_state_data: rs_dcerpc_udp_get_state_data,

2
rust/src/dhcp/dhcp.rs
Unescape Escape View File

@ -296,7 +296,7 @@ pub unsafe extern "C" fn rs_dhcp_register_parser() {
get_eventinfo_byid : Some(DHCPEvent::get_event_info_by_id),
localstorage_new : None,
localstorage_free : None,
get_files : None,
get_tx_files : None,
get_tx_iterator : Some(applayer::state_get_tx_iterator::<DHCPState, DHCPTransaction>),
get_tx_data : rs_dhcp_get_tx_data,
get_state_data : rs_dhcp_get_state_data,

4
rust/src/dns/dns.rs
Unescape Escape View File

@ -980,7 +980,7 @@ pub unsafe extern "C" fn rs_dns_udp_register_parser() {
get_eventinfo_byid: Some(DNSEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::<DNSState, DNSTransaction>),
get_tx_data: rs_dns_state_get_tx_data,
get_state_data: rs_dns_get_state_data,
@ -1026,7 +1026,7 @@ pub unsafe extern "C" fn rs_dns_tcp_register_parser() {
get_eventinfo_byid: Some(DNSEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::<DNSState, DNSTransaction>),
get_tx_data: rs_dns_state_get_tx_data,
get_state_data: rs_dns_get_state_data,

20
rust/src/filecontainer.rs
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2017 Open Information Security Foundation
/* Copyright (C) 2017-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -22,7 +22,7 @@ use crate::core::*;
// Defined in util-file.h
extern {
pub fn FileFlowToFlags(flow: *const Flow, flags: u8) -> u16;
pub fn FileFlowFlagsToFlags(flow_file_flags: u16, flags: u8) -> u16;
}
pub const FILE_USE_DETECT: u16 = BIT_U16!(13);
@ -74,11 +74,8 @@ impl FileContainer {
}
pub fn free(&mut self) {
SCLogDebug!("freeing self");
match unsafe {SC} {
None => panic!("BUG no suricata_config"),
Some(c) => {
(c.FileContainerRecycle)(self);
},
if let Some(c) = unsafe {SC} {
(c.FileContainerRecycle)(self);
}
}
@ -145,13 +142,4 @@ impl FileContainer {
}
}
}
pub fn file_set_txid_on_last_file(&mut self, tx_id: u64) {
match unsafe {SC} {
None => panic!("BUG no suricata_config"),
Some(c) => {
(c.FileSetTx)(self, tx_id);
}
}
}
}

1
rust/src/filetracker.rs
Unescape Escape View File

@ -87,7 +87,6 @@ impl FileTransferTracker {
{
let r = files.file_open(config, &self.track_id, name, flags);
if r == 0 {
files.file_set_txid_on_last_file(self.tx_id);
self.file_open = true;
}
r

63
rust/src/http2/http2.rs
Unescape Escape View File

@ -142,6 +142,8 @@ pub struct HTTP2Transaction {
//temporary escaped header for detection
//must be attached to transaction for memory management (be freed at the right time)
pub escaped: Vec<Vec<u8>>,
pub files: Files,
}
impl Transaction for HTTP2Transaction {
@ -165,6 +167,7 @@ impl HTTP2Transaction {
ft_tc: FileTransferTracker::new(),
ft_ts: FileTransferTracker::new(),
escaped: Vec::with_capacity(16),
files: Files::default(),
}
}
@ -200,9 +203,13 @@ impl HTTP2Transaction {
}
}
pub fn update_file_flags(&mut self, flow_file_flags: u16) {
self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT };
self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT };
}
fn decompress<'a>(
&'a mut self, input: &'a [u8], dir: Direction, sfcm: &'static SuricataFileContext,
over: bool, files: &mut FileContainer, flags: u16, flow: *const Flow,
&'a mut self, input: &'a [u8], dir: Direction, sfcm: &'static SuricataFileContext, over: bool, flow: *const Flow,
) -> io::Result<()> {
let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE);
let decompressed = self.decoder.decompress(input, &mut output, dir)?;
@ -220,9 +227,9 @@ impl HTTP2Transaction {
) {
match range::http2_parse_check_content_range(&value) {
Ok((_, v)) => {
range::http2_range_open(self, &v, flow, sfcm, flags, decompressed);
range::http2_range_open(self, &v, flow, sfcm, Direction::ToClient, decompressed);
if over && !self.file_range.is_null() {
range::http2_range_close(self, files, flags, &[])
range::http2_range_close(self, Direction::ToClient, &[])
}
}
_ => {
@ -233,12 +240,13 @@ impl HTTP2Transaction {
} else {
if !self.file_range.is_null() {
if over {
range::http2_range_close(self, files, flags, decompressed)
range::http2_range_close(self, Direction::ToClient, decompressed)
} else {
range::http2_range_append(self.file_range, decompressed)
}
}
}
let (files, flags) = self.files.get(Direction::ToClient);
self.ft_tc.new_chunk(
sfcm,
files,
@ -256,6 +264,7 @@ impl HTTP2Transaction {
if !self.ft_ts.file_open {
self.tx_data.incr_files_opened();
}
let (files, flags) = self.files.get(Direction::ToServer);
self.ft_ts.new_chunk(
sfcm,
files,
@ -401,7 +410,6 @@ pub struct HTTP2State {
dynamic_headers_tc: HTTP2DynTable,
transactions: VecDeque<HTTP2Transaction>,
progress: HTTP2ConnectionState,
pub files: Files,
}
impl State<HTTP2Transaction> for HTTP2State {
@ -428,7 +436,6 @@ impl HTTP2State {
dynamic_headers_tc: HTTP2DynTable::new(),
transactions: VecDeque::new(),
progress: HTTP2ConnectionState::Http2StateInit,
files: Files::default(),
}
}
@ -441,7 +448,7 @@ impl HTTP2State {
None => panic!("BUG no suricata_config"),
Some(c) => {
(c.HTPFileCloseHandleRange)(
&mut self.files.files_tc,
&mut tx.files.files_tc,
0,
tx.file_range,
std::ptr::null_mut(),
@ -482,7 +489,7 @@ impl HTTP2State {
None => panic!("BUG no suricata_config"),
Some(c) => {
(c.HTPFileCloseHandleRange)(
&mut self.files.files_tc,
&mut tx.files.files_tc,
0,
tx.file_range,
std::ptr::null_mut(),
@ -504,6 +511,8 @@ impl HTTP2State {
pub fn get_tx(&mut self, tx_id: u64) -> Option<&HTTP2Transaction> {
for tx in &mut self.transactions {
if tx.tx_id == tx_id + 1 {
tx.tx_data.update_file_flags(self.state_data.file_flags);
tx.update_file_flags(tx.tx_data.file_flags);
return Some(tx);
}
}
@ -541,6 +550,8 @@ impl HTTP2State {
self.tx_id += 1;
tx.tx_id = self.tx_id;
tx.state = HTTP2TransactionState::HTTP2StateGlobal;
tx.tx_data.update_file_flags(self.state_data.file_flags);
tx.update_file_flags(tx.tx_data.file_flags);
self.transactions.push_back(tx);
return self.transactions.back_mut().unwrap();
}
@ -575,7 +586,11 @@ impl HTTP2State {
self.set_event(HTTP2Event::StreamIdReuse);
}
}
return &mut self.transactions[index - 1];
let tx = &mut self.transactions[index - 1];
tx.tx_data.update_file_flags(self.state_data.file_flags);
tx.update_file_flags(tx.tx_data.file_flags);
return tx;
} else {
let mut tx = HTTP2Transaction::new();
self.tx_id += 1;
@ -594,6 +609,8 @@ impl HTTP2State {
}
}
}
tx.tx_data.update_file_flags(self.state_data.file_flags);
tx.update_file_flags(tx.tx_data.file_flags);
self.transactions.push_back(tx);
return self.transactions.back_mut().unwrap();
}
@ -934,15 +951,18 @@ impl HTTP2State {
//borrow checker forbids to reuse directly tx
let index = self.find_tx_index(sid);
if index > 0 {
let tx_same = &mut self.transactions[index - 1];
let (files, flags) = self.files.get(dir);
let mut tx_same = &mut self.transactions[index - 1];
if dir == Direction::ToServer {
tx_same.ft_tc.tx_id = tx_same.tx_id - 1;
} else {
tx_same.ft_ts.tx_id = tx_same.tx_id - 1;
};
match tx_same.decompress(
&rem[..hlsafe],
dir,
sfcm,
over,
files,
flags,
flow,
) {
Err(_e) => {
@ -1120,9 +1140,6 @@ pub unsafe extern "C" fn rs_http2_parse_ts(
) -> AppLayerResult {
let state = cast_pointer!(state, HTTP2State);
let buf = stream_slice.as_slice();
state.files.flags_ts = FileFlowToFlags(flow, Direction::ToServer.into());
state.files.flags_ts = state.files.flags_ts | FILE_USE_DETECT;
return state.parse_ts(buf, flow);
}
@ -1133,8 +1150,6 @@ pub unsafe extern "C" fn rs_http2_parse_tc(
) -> AppLayerResult {
let state = cast_pointer!(state, HTTP2State);
let buf = stream_slice.as_slice();
state.files.flags_tc = FileFlowToFlags(flow, Direction::ToClient.into());
state.files.flags_tc = state.files.flags_tc | FILE_USE_DETECT;
return state.parse_tc(buf, flow);
}
@ -1176,13 +1191,13 @@ pub unsafe extern "C" fn rs_http2_tx_get_alstate_progress(
#[no_mangle]
pub unsafe extern "C" fn rs_http2_getfiles(
state: *mut std::os::raw::c_void, direction: u8,
tx: *mut std::os::raw::c_void, direction: u8,
) -> *mut FileContainer {
let state = cast_pointer!(state, HTTP2State);
let tx = cast_pointer!(tx, HTTP2Transaction);
if direction == Direction::ToClient.into() {
&mut state.files.files_tc as *mut FileContainer
&mut tx.files.files_tc as *mut FileContainer
} else {
&mut state.files.files_ts as *mut FileContainer
&mut tx.files.files_ts as *mut FileContainer
}
}
@ -1214,7 +1229,7 @@ pub unsafe extern "C" fn rs_http2_register_parser() {
get_eventinfo_byid: Some(HTTP2Event::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: Some(rs_http2_getfiles),
get_tx_files: Some(rs_http2_getfiles),
get_tx_iterator: Some(applayer::state_get_tx_iterator::<HTTP2State, HTTP2Transaction>),
get_tx_data: rs_http2_get_tx_data,
get_state_data: rs_http2_get_state_data,

7
rust/src/http2/range.rs
Unescape Escape View File

@ -19,7 +19,6 @@ use super::detect;
use crate::core::{
Direction, Flow, HttpRangeContainerBlock, StreamingBufferConfig, SuricataFileContext, SC,
};
use crate::filecontainer::FileContainer;
use crate::http2::http2::HTTP2Transaction;
use nom7::branch::alt;
@ -130,7 +129,7 @@ fn http2_range_key_get(tx: &mut HTTP2Transaction) -> Result<(Vec<u8>, usize), ()
pub fn http2_range_open(
tx: &mut HTTP2Transaction, v: &HTTPContentRange, flow: *const Flow,
cfg: &'static SuricataFileContext, flags: u16, data: &[u8],
cfg: &'static SuricataFileContext, dir: Direction, data: &[u8],
) {
if v.end <= 0 || v.size <= 0 {
// skipped for incomplete range information
@ -139,6 +138,7 @@ pub fn http2_range_open(
// whole file in one range
return;
}
let (_, flags) = tx.files.get(dir);
if let Ok((key, index)) = http2_range_key_get(tx) {
let name = &key[index..];
tx.file_range = unsafe {
@ -165,9 +165,10 @@ pub fn http2_range_append(fr: *mut HttpRangeContainerBlock, data: &[u8]) {
}
pub fn http2_range_close(
tx: &mut HTTP2Transaction, files: &mut FileContainer, flags: u16, data: &[u8],
tx: &mut HTTP2Transaction, dir: Direction, data: &[u8],
) {
let added = if let Some(c) = unsafe { SC } {
let (files, flags) = tx.files.get(dir);
let added = (c.HTPFileCloseHandleRange)(
files,
flags,

2
rust/src/ike/ike.rs
Unescape Escape View File

@ -422,7 +422,7 @@ pub unsafe extern "C" fn rs_ike_register_parser() {
get_eventinfo_byid: Some(IkeEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<IKEState, IKETransaction>),
get_tx_data: rs_ike_get_tx_data,
get_state_data: rs_ike_get_state_data,

2
rust/src/ike/ikev2.rs
Unescape Escape View File

@ -202,7 +202,7 @@ fn add_proposals(
| IkeTransformEncType::ENCR_DES_IV32
| IkeTransformEncType::ENCR_NULL => {
SCLogDebug!("Weak Encryption: {:?}", enc);
// XXX send event only if direction == STREAM_TOCLIENT ?
// XXX send event only if direction == Direction::ToClient ?
tx.set_event(IkeEvent::WeakCryptoEnc);
}
_ => (),

2
rust/src/krb/krb5.rs
Unescape Escape View File

@ -557,7 +557,7 @@ pub unsafe extern "C" fn rs_register_krb5_parser() {
get_eventinfo_byid : Some(KRB5Event::get_event_info_by_id),
localstorage_new : None,
localstorage_free : None,
get_files : None,
get_tx_files : None,
get_tx_iterator : Some(applayer::state_get_tx_iterator::<KRB5State, KRB5Transaction>),
get_tx_data : rs_krb5_get_tx_data,
get_state_data : rs_krb5_get_state_data,

2
rust/src/modbus/modbus.rs
Unescape Escape View File

@ -407,7 +407,7 @@ pub unsafe extern "C" fn rs_modbus_register_parser() {
get_eventinfo_byid: Some(ModbusEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<ModbusState, ModbusTransaction>),
get_tx_data: rs_modbus_state_get_tx_data,
get_state_data: rs_modbus_get_state_data,

2
rust/src/mqtt/mqtt.rs
Unescape Escape View File

@ -753,7 +753,7 @@ pub unsafe extern "C" fn rs_mqtt_register_parser(cfg_max_msg_len: u32) {
get_eventinfo_byid: Some(MQTTEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(crate::applayer::state_get_tx_iterator::<MQTTState, MQTTTransaction>),
get_tx_data: rs_mqtt_get_tx_data,
get_state_data: rs_mqtt_get_state_data,

108
rust/src/nfs/nfs.rs
Unescape Escape View File

@ -139,6 +139,9 @@ pub struct NFSTransactionFile {
/// file tracker for a single file. Boxed so that we don't use
/// as much space if we're not a file tx.
pub file_tracker: FileTransferTracker,
/// storage for the actual file
pub files: Files,
}
impl NFSTransactionFile {
@ -148,6 +151,21 @@ impl NFSTransactionFile {
..Default::default()
}
}
pub fn update_file_flags(&mut self, flow_file_flags: u16) {
self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT };
self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT };
}
}
#[no_mangle]
pub unsafe extern "C" fn rs_nfs_gettxfiles(tx_ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer {
let tx = cast_pointer!(tx_ptr, NFSTransaction);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, _flags) = tdf.files.get(direction.into());
files
} else {
std::ptr::null_mut()
}
}
#[derive(Debug)]
@ -292,8 +310,6 @@ pub struct NFSState {
/// transactions list
pub transactions: Vec<NFSTransaction>,
pub files: Files,
/// partial record tracking
pub ts_chunk_xid: u32,
pub tc_chunk_xid: u32,
@ -344,7 +360,6 @@ impl NFSState {
requestmap:HashMap::new(),
namemap:HashMap::new(),
transactions: Vec::new(),
files:Files::default(),
ts_chunk_xid:0,
tc_chunk_xid:0,
ts_chunk_left:0,
@ -565,7 +580,7 @@ impl NFSState {
if self.ts > f.post_gap_ts {
tx.request_done = true;
tx.response_done = true;
let (files, flags) = self.files.get(tx.file_tx_direction);
let (files, flags) = f.files.get(tx.file_tx_direction);
f.file_tracker.trunc(files, flags);
} else {
post_gap_txs = true;
@ -671,7 +686,7 @@ impl NFSState {
}
pub fn new_file_tx(&mut self, file_handle: &Vec<u8>, file_name: &Vec<u8>, direction: Direction)
-> (&mut NFSTransaction, &mut FileContainer, u16)
-> &mut NFSTransaction
{
let mut tx = self.new_tx();
tx.file_name = file_name.to_vec();
@ -682,28 +697,32 @@ impl NFSState {
tx.type_data = Some(NFSTransactionTypeData::FILE(NFSTransactionFile::new()));
if let Some(NFSTransactionTypeData::FILE(ref mut d)) = tx.type_data {
d.file_tracker.tx_id = tx.id - 1;
tx.tx_data.update_file_flags(self.state_data.file_flags);
d.update_file_flags(tx.tx_data.file_flags);
}
tx.tx_data.init_files_opened();
SCLogDebug!("new_file_tx: TX FILE created: ID {} NAME {}",
tx.id, String::from_utf8_lossy(file_name));
self.transactions.push(tx);
let tx_ref = self.transactions.last_mut();
let (files, flags) = self.files.get(direction);
return (tx_ref.unwrap(), files, flags)
return tx_ref.unwrap();
}
pub fn get_file_tx_by_handle(&mut self, file_handle: &Vec<u8>, direction: Direction)
-> Option<(&mut NFSTransaction, &mut FileContainer, u16)>
-> Option<&mut NFSTransaction>
{
let fh = file_handle.to_vec();
for tx in &mut self.transactions {
if tx.is_file_tx && !tx.is_file_closed &&
direction == tx.file_tx_direction &&
tx.file_handle == fh
{
SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid);
let (files, flags) = self.files.get(direction);
return Some((tx, files, flags));
if let Some(NFSTransactionTypeData::FILE(ref mut d)) = tx.type_data {
if tx.is_file_tx && !tx.is_file_closed &&
direction == tx.file_tx_direction &&
tx.file_handle == fh
{
tx.tx_data.update_file_flags(self.state_data.file_flags);
d.update_file_flags(tx.tx_data.file_flags);
SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid);
return Some(tx);
}
}
}
SCLogDebug!("Failed to find NFS TX with handle {:?}", file_handle);
@ -734,8 +753,9 @@ impl NFSState {
};
let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
Some((tx, files, flags)) => {
Some(tx) => {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, w.file_data, w.offset,
w.file_len, fill_bytes as u8, is_last, &r.hdr.xid);
@ -754,8 +774,9 @@ impl NFSState {
None => { false },
};
if !found {
let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToServer);
let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToServer);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, w.file_data, w.offset,
w.file_len, fill_bytes as u8, is_last, &r.hdr.xid);
@ -907,8 +928,9 @@ impl NFSState {
let ssn_gap = self.ts_ssn_gap | self.tc_ssn_gap;
// get the tx and update it
let consumed = match self.get_file_tx_by_handle(&file_handle, direction) {
Some((tx, files, flags)) => {
Some(tx) => {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(direction);
if ssn_gap {
let queued_data = tdf.file_tracker.get_queued_size();
if queued_data > 2000000 { // TODO should probably be configurable
@ -1009,9 +1031,10 @@ impl NFSState {
SCLogDebug!("partial data? {}", is_partial);
let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToClient) {
Some((tx, files, flags)) => {
Some(tx) => {
SCLogDebug!("updated TX {:?}", tx);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, reply.data, chunk_offset,
reply.count, fill_bytes as u8, is_last, &r.hdr.xid);
@ -1038,8 +1061,9 @@ impl NFSState {
None => { false },
};
if !found {
let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToClient);
let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToClient);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, reply.data, chunk_offset,
reply.count, fill_bytes as u8, is_last, &r.hdr.xid);
@ -1497,23 +1521,6 @@ impl NFSState {
}
AppLayerResult::ok()
}
fn getfiles(&mut self, direction: Direction) -> * mut FileContainer {
//SCLogDebug!("direction: {:?}", direction);
if direction == Direction::ToClient {
&mut self.files.files_tc as *mut FileContainer
} else {
&mut self.files.files_ts as *mut FileContainer
}
}
fn setfileflags(&mut self, direction: Direction, flags: u16) {
SCLogDebug!("direction: {:?}, flags: {}", direction, flags);
if direction == Direction::ToClient {
self.files.flags_tc = flags;
} else {
self.files.flags_ts = flags;
}
}
}
/// Returns *mut NFSState
@ -1545,8 +1552,6 @@ pub unsafe extern "C" fn rs_nfs_parse_request(flow: *const Flow,
{
let state = cast_pointer!(state, NFSState);
let flow = cast_pointer!(flow, Flow);
let file_flags = FileFlowToFlags(flow, Direction::ToServer.into());
rs_nfs_setfileflags(Direction::ToServer.into(), state, file_flags);
if stream_slice.is_gap() {
return rs_nfs_parse_request_tcp_gap(state, stream_slice.gap_size());
@ -1576,8 +1581,6 @@ pub unsafe extern "C" fn rs_nfs_parse_response(flow: *const Flow,
{
let state = cast_pointer!(state, NFSState);
let flow = cast_pointer!(flow, Flow);
let file_flags = FileFlowToFlags(flow, Direction::ToClient.into());
rs_nfs_setfileflags(Direction::ToClient.into(), state, file_flags);
if stream_slice.is_gap() {
return rs_nfs_parse_response_tcp_gap(state, stream_slice.gap_size());
@ -1607,8 +1610,6 @@ pub unsafe extern "C" fn rs_nfs_parse_request_udp(f: *const Flow,
) -> AppLayerResult
{
let state = cast_pointer!(state, NFSState);
let file_flags = FileFlowToFlags(f, Direction::ToServer.into());
rs_nfs_setfileflags(Direction::ToServer.into(), state, file_flags);
SCLogDebug!("parsing {} bytes of request data", stream_slice.len());
state.parse_udp_ts(f, &stream_slice)
@ -1623,8 +1624,6 @@ pub unsafe extern "C" fn rs_nfs_parse_response_udp(f: *const Flow,
) -> AppLayerResult
{
let state = cast_pointer!(state, NFSState);
let file_flags = FileFlowToFlags(f, Direction::ToClient.into());
rs_nfs_setfileflags(Direction::ToClient.into(), state, file_flags);
SCLogDebug!("parsing {} bytes of response data", stream_slice.len());
state.parse_udp_tc(f, &stream_slice)
}
@ -1934,20 +1933,6 @@ pub unsafe extern "C" fn rs_nfs_probe_udp_tc(_f: *const Flow,
}
}
#[no_mangle]
pub unsafe extern "C" fn rs_nfs_getfiles(ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer {
if ptr.is_null() { panic!("NULL ptr"); };
let parser = cast_pointer!(ptr, NFSState);
parser.getfiles(direction.into())
}
#[no_mangle]
pub unsafe extern "C" fn rs_nfs_setfileflags(direction: u8, ptr: *mut NFSState, flags: u16) {
if ptr.is_null() { panic!("NULL ptr"); };
let parser = &mut *ptr;
SCLogDebug!("direction {} flags {}", direction, flags);
parser.setfileflags(direction.into(), flags)
}
// Parser name as a C style string.
const PARSER_NAME: &'static [u8] = b"nfs\0";
@ -1976,7 +1961,7 @@ pub unsafe extern "C" fn rs_nfs_register_parser() {
get_eventinfo_byid : Some(NFSEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: Some(rs_nfs_getfiles),
get_tx_files: Some(rs_nfs_gettxfiles),
get_tx_iterator: Some(applayer::state_get_tx_iterator::<NFSState, NFSTransaction>),
get_tx_data: rs_nfs_get_tx_data,
get_state_data: rs_nfs_get_state_data,
@ -2055,7 +2040,7 @@ pub unsafe extern "C" fn rs_nfs_udp_register_parser() {
get_eventinfo_byid : Some(NFSEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: Some(rs_nfs_getfiles),
get_tx_files: Some(rs_nfs_gettxfiles),
get_tx_iterator: Some(applayer::state_get_tx_iterator::<NFSState, NFSTransaction>),
get_tx_data: rs_nfs_get_tx_data,
get_state_data: rs_nfs_get_state_data,
@ -2106,4 +2091,3 @@ pub unsafe extern "C" fn rs_nfs_udp_register_parser() {
SCLogDebug!("Protocol detector and parser disabled for nfs.");
}
}

7
rust/src/nfs/nfs3.rs
Unescape Escape View File

@ -118,8 +118,9 @@ impl NFSState {
SCLogDebug!("COMMIT, closing shop");
if let Ok((_, rd)) = parse_nfs3_request_commit(r.prog_data) {
let file_handle = rd.handle.value.to_vec();
if let Some((tx, files, flags)) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
if let Some(tx) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToServer);
tdf.chunk_count += 1;
tdf.file_additional_procs.push(NFSPROC3_COMMIT);
tdf.file_tracker.close(files, flags);
@ -166,11 +167,11 @@ impl NFSState {
} else if r.procedure == NFSPROC3_READ {
let found = match self.get_file_tx_by_handle(&xidmap.file_handle, Direction::ToClient) {
Some((_, _, _)) => true,
Some(_) => true,
None => false,
};
if !found {
let (tx, _, _) = self.new_file_tx(&xidmap.file_handle, &xidmap.file_name, Direction::ToClient);
let tx = self.new_file_tx(&xidmap.file_handle, &xidmap.file_name, Direction::ToClient);
tx.procedure = NFSPROC3_READ;
tx.xid = r.hdr.xid;
tx.is_first = true;

40
rust/src/nfs/nfs4.rs
Unescape Escape View File

@ -66,20 +66,12 @@ impl NFSState {
};
let found = match self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
Some((tx, files, flags)) => {
Some(tx) => {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
filetracker_newchunk(
&mut tdf.file_tracker,
files,
flags,
&file_name,
w.data,
w.offset,
w.write_len,
fill_bytes as u8,
is_last,
&r.hdr.xid,
);
let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, w.data, w.offset,
w.write_len, fill_bytes as u8, is_last, &r.hdr.xid);
tdf.chunk_count += 1;
if is_last {
tdf.file_last_xid = r.hdr.xid;
@ -92,20 +84,12 @@ impl NFSState {
None => false,
};
if !found {
let (tx, files, flags) = self.new_file_tx(&file_handle, &file_name, Direction::ToServer);
let tx = self.new_file_tx(&file_handle, &file_name, Direction::ToServer);
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
filetracker_newchunk(
&mut tdf.file_tracker,
files,
flags,
&file_name,
w.data,
w.offset,
w.write_len,
fill_bytes as u8,
is_last,
&r.hdr.xid,
);
let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, w.data, w.offset,
w.write_len, fill_bytes as u8, is_last, &r.hdr.xid);
tx.procedure = NFSPROC4_WRITE;
tx.xid = r.hdr.xid;
tx.is_first = true;
@ -131,9 +115,9 @@ impl NFSState {
SCLogDebug!("COMMIT, closing shop");
let file_handle = fh.to_vec();
if let Some((tx, files, flags)) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer)
{
if let Some(tx) = self.get_file_tx_by_handle(&file_handle, Direction::ToServer) {
if let Some(NFSTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToServer);
tdf.file_tracker.close(files, flags);
tdf.file_last_xid = r.hdr.xid;
tx.is_last = true;

2
rust/src/ntp/ntp.rs
Unescape Escape View File

@ -295,7 +295,7 @@ pub unsafe extern "C" fn rs_register_ntp_parser() {
get_eventinfo_byid : Some(NTPEvent::get_event_info_by_id),
localstorage_new : None,
localstorage_free : None,
get_files : None,
get_tx_files : None,
get_tx_iterator : Some(applayer::state_get_tx_iterator::<NTPState, NTPTransaction>),
get_tx_data : rs_ntp_get_tx_data,
get_state_data : rs_ntp_get_state_data,

2
rust/src/pgsql/pgsql.rs
Unescape Escape View File

@ -729,7 +729,7 @@ pub unsafe extern "C" fn rs_pgsql_register_parser() {
get_eventinfo_byid: None,
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(
crate::applayer::state_get_tx_iterator::<PgsqlState, PgsqlTransaction>,
),

2
rust/src/quic/quic.rs
Unescape Escape View File

@ -473,7 +473,7 @@ pub unsafe extern "C" fn rs_quic_register_parser() {
get_eventinfo_byid: Some(QuicEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(rs_quic_state_get_tx_iterator),
get_tx_data: rs_quic_get_tx_data,
get_state_data: rs_quic_get_state_data,

2
rust/src/rdp/rdp.rs
Unescape Escape View File

@ -489,7 +489,7 @@ pub unsafe extern "C" fn rs_rdp_register_parser() {
get_eventinfo_byid: None,
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<RdpState, RdpTransaction>),
get_tx_data: rs_rdp_get_tx_data,
get_state_data: rs_rdp_get_state_data,

2
rust/src/rfb/rfb.rs
Unescape Escape View File

@ -598,7 +598,7 @@ pub unsafe extern "C" fn rs_rfb_register_parser() {
get_eventinfo_byid: None,
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<RFBState, RFBTransaction>),
get_tx_data: rs_rfb_get_tx_data,
get_state_data: rs_rfb_get_state_data,

2
rust/src/sip/sip.rs
Unescape Escape View File

@ -370,7 +370,7 @@ pub unsafe extern "C" fn rs_sip_register_parser() {
get_eventinfo_byid: Some(SIPEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<SIPState, SIPTransaction>),
get_tx_data: rs_sip_get_tx_data,
get_state_data: rs_sip_get_state_data,

64
rust/src/smb/files.rs
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2018 Open Information Security Foundation
/* Copyright (C) 2018-2022 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -33,6 +33,7 @@ pub struct SMBTransactionFile {
/// after a gap, this will be set to a time in the future. If the file
/// receives no updates before that, it will be considered complete.
pub post_gap_ts: u64,
pub files: Files,
}
impl SMBTransactionFile {
@ -42,6 +43,11 @@ impl SMBTransactionFile {
..Default::default()
}
}
pub fn update_file_flags(&mut self, flow_file_flags: u16) {
self.files.flags_ts = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOSERVER) | FILE_USE_DETECT };
self.files.flags_tc = unsafe { FileFlowFlagsToFlags(flow_file_flags, STREAM_TOCLIENT) | FILE_USE_DETECT };
}
}
/// little wrapper around the FileTransferTracker::new_chunk method
@ -59,7 +65,7 @@ pub fn filetracker_newchunk(ft: &mut FileTransferTracker, files: &mut FileContai
impl SMBState {
pub fn new_file_tx(&mut self, fuid: &Vec<u8>, file_name: &Vec<u8>, direction: Direction)
-> (&mut SMBTransaction, &mut FileContainer, u16)
-> &mut SMBTransaction
{
let mut tx = self.new_tx();
tx.type_data = Some(SMBTransactionTypeData::FILE(SMBTransactionFile::new()));
@ -69,6 +75,8 @@ impl SMBState {
d.fuid = fuid.to_vec();
d.file_name = file_name.to_vec();
d.file_tracker.tx_id = tx.id - 1;
tx.tx_data.update_file_flags(self.state_data.file_flags);
d.update_file_flags(tx.tx_data.file_flags);
},
_ => { },
}
@ -77,12 +85,11 @@ impl SMBState {
tx.id, String::from_utf8_lossy(file_name));
self.transactions.push(tx);
let tx_ref = self.transactions.last_mut();
let (files, flags) = self.files.get(direction);
return (tx_ref.unwrap(), files, flags)
return tx_ref.unwrap();
}
pub fn get_file_tx_by_fuid(&mut self, fuid: &Vec<u8>, direction: Direction)
-> Option<(&mut SMBTransaction, &mut FileContainer, u16)>
-> Option<&mut SMBTransaction>
{
let f = fuid.to_vec();
for tx in &mut self.transactions {
@ -95,31 +102,17 @@ impl SMBState {
if found {
SCLogDebug!("SMB: Found SMB file TX with ID {}", tx.id);
let (files, flags) = self.files.get(direction);
return Some((tx, files, flags));
if let Some(SMBTransactionTypeData::FILE(ref mut d)) = tx.type_data {
tx.tx_data.update_file_flags(self.state_data.file_flags);
d.update_file_flags(tx.tx_data.file_flags);
}
return Some(tx);
}
}
SCLogDebug!("SMB: Failed to find SMB TX with FUID {:?}", fuid);
return None;
}
fn getfiles(&mut self, direction: Direction) -> * mut FileContainer {
//SCLogDebug!("direction: {:?}", direction);
if direction == Direction::ToClient {
&mut self.files.files_tc as *mut FileContainer
} else {
&mut self.files.files_ts as *mut FileContainer
}
}
fn setfileflags(&mut self, direction: Direction, flags: u16) {
SCLogDebug!("direction: {:?}, flags: {}", direction, flags);
if direction == Direction::ToClient {
self.files.flags_tc = flags;
} else {
self.files.flags_ts = flags;
}
}
// update in progress chunks for file transfers
// return how much data we consumed
pub fn filetracker_update(&mut self, direction: Direction, data: &[u8], gap_size: u32) -> u32 {
@ -159,8 +152,9 @@ impl SMBState {
let ssn_gap = self.ts_ssn_gap | self.tc_ssn_gap;
// get the tx and update it
let consumed = match self.get_file_tx_by_fuid(&file_handle, direction) {
Some((tx, files, flags)) => {
Some(tx) => {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(direction);
if ssn_gap {
let queued_data = tdf.file_tracker.get_queued_size();
if queued_data > 2000000 { // TODO should probably be configurable
@ -191,16 +185,12 @@ impl SMBState {
}
#[no_mangle]
pub unsafe extern "C" fn rs_smb_getfiles(ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer {
if ptr.is_null() { panic!("NULL ptr"); };
let parser = cast_pointer!(ptr, SMBState);
parser.getfiles(direction.into())
}
#[no_mangle]
pub unsafe extern "C" fn rs_smb_setfileflags(direction: u8, ptr: *mut SMBState, flags: u16) {
if ptr.is_null() { panic!("NULL ptr"); };
let parser = &mut *ptr;
SCLogDebug!("direction {} flags {}", direction, flags);
parser.setfileflags(direction.into(), flags)
pub unsafe extern "C" fn rs_smb_gettxfiles(tx_ptr: *mut std::ffi::c_void, direction: u8) -> * mut FileContainer {
let tx = cast_pointer!(tx_ptr, SMBTransaction);
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, _flags) = tdf.files.get(direction.into());
files
} else {
std::ptr::null_mut()
}
}

22
rust/src/smb/smb.rs
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2017-2020 Open Information Security Foundation
/* Copyright (C) 2017-2022 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -39,7 +39,6 @@ use crate::applayer;
use crate::applayer::*;
use crate::frames::*;
use crate::conf::*;
use crate::filecontainer::*;
use crate::applayer::{AppLayerResult, AppLayerTxData, AppLayerEvent};
use crate::smb::nbss_records::*;
@ -593,6 +592,7 @@ impl SMBTransaction {
}
pub fn free(&mut self) {
SCLogDebug!("SMB TX {:p} free ID {}", &self, self.id);
debug_validate_bug_on!(self.tx_data.files_opened > 1);
debug_validate_bug_on!(self.tx_data.files_logged > 1);
}
@ -767,8 +767,6 @@ pub struct SMBState<> {
// requests for DCERPC.
pub ssnguid2vec_map: HashMap<SMBHashKeyHdrGuid, Vec<u8>>,
pub files: Files,
skip_ts: u32,
skip_tc: u32,
@ -834,7 +832,6 @@ impl SMBState {
ssn2vecoffset_map:HashMap::new(),
ssn2tree_map:HashMap::new(),
ssnguid2vec_map:HashMap::new(),
files: Files::default(),
skip_ts:0,
skip_tc:0,
file_ts_left:0,
@ -914,6 +911,11 @@ impl SMBState {
}
SCLogDebug!("Found SMB TX: id {} ver:{} cmd:{} progress {}/{} type_data {:?}",
tx.id, ver, _smbcmd, tx.request_done, tx.response_done, tx.type_data);
/* hack: apply flow file flags to file tx here to make sure its propegated */
if let Some(SMBTransactionTypeData::FILE(ref mut d)) = tx.type_data {
tx.tx_data.update_file_flags(self.state_data.file_flags);
d.update_file_flags(tx.tx_data.file_flags);
}
return Some(tx);
}
}
@ -1160,7 +1162,7 @@ impl SMBState {
if self.ts > f.post_gap_ts {
tx.request_done = true;
tx.response_done = true;
let (files, flags) = self.files.get(f.direction);
let (files, flags) = f.files.get(f.direction);
f.file_tracker.trunc(files, flags);
} else {
post_gap_txs = true;
@ -2034,8 +2036,6 @@ pub unsafe extern "C" fn rs_smb_parse_request_tcp(flow: *const Flow,
{
let mut state = cast_pointer!(state, SMBState);
let flow = cast_pointer!(flow, Flow);
let file_flags = FileFlowToFlags(flow, Direction::ToServer as u8);
rs_smb_setfileflags(Direction::ToServer as u8, state, file_flags|FILE_USE_DETECT);
if stream_slice.is_gap() {
return rs_smb_parse_request_tcp_gap(state, stream_slice.gap_size());
@ -2073,8 +2073,6 @@ pub unsafe extern "C" fn rs_smb_parse_response_tcp(flow: *const Flow,
{
let mut state = cast_pointer!(state, SMBState);
let flow = cast_pointer!(flow, Flow);
let file_flags = FileFlowToFlags(flow, Direction::ToClient as u8);
rs_smb_setfileflags(Direction::ToClient as u8, state, file_flags|FILE_USE_DETECT);
if stream_slice.is_gap() {
return rs_smb_parse_response_tcp_gap(state, stream_slice.gap_size());
@ -2289,7 +2287,7 @@ pub unsafe extern "C" fn rs_smb_tx_get_alstate_progress(tx: *mut ffi::c_void,
SCLogDebug!("tx {} TOCLIENT progress 1 => {:?}", tx.id, tx);
return 1;
} else {
SCLogDebug!("tx {} direction {:?} progress 0", tx.id, direction);
SCLogDebug!("tx {} direction {} progress 0 => {:?}", tx.id, direction, tx);
return 0;
}
}
@ -2426,7 +2424,7 @@ pub unsafe extern "C" fn rs_smb_register_parser() {
get_eventinfo_byid : Some(rs_smb_state_get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: Some(rs_smb_getfiles),
get_tx_files: Some(rs_smb_gettxfiles),
get_tx_iterator: Some(applayer::state_get_tx_iterator::<SMBState, SMBTransaction>),
get_tx_data: rs_smb_get_tx_data,
get_state_data: rs_smb_get_state_data,

21
rust/src/smb/smb1.rs
Unescape Escape View File

@ -146,11 +146,12 @@ pub fn smb1_check_tx(cmd: u8) -> bool {
fn smb1_close_file(state: &mut SMBState, fid: &Vec<u8>, direction: Direction)
{
if let Some((tx, files, flags)) = state.get_file_tx_by_fuid(fid, direction) {
if let Some(tx) = state.get_file_tx_by_fuid(fid, direction) {
SCLogDebug!("found tx {}", tx.id);
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
if !tx.request_done {
SCLogDebug!("closing file tx {} FID {:?}", tx.id, fid);
let (files, flags) = tdf.files.get(direction);
tdf.file_tracker.close(files, flags);
tx.request_done = true;
tx.response_done = true;
@ -961,16 +962,17 @@ pub fn smb1_write_request_record<'b>(state: &mut SMBState, r: &SmbRecord<'b>, an
};
let mut set_event_fileoverlap = false;
let found = match state.get_file_tx_by_fuid(&file_fid, Direction::ToServer) {
Some((tx, files, flags)) => {
Some(tx) => {
let file_id : u32 = tx.id as u32;
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
if rd.offset < tdf.file_tracker.tracked {
set_event_fileoverlap = true;
}
let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, rd.offset,
rd.len, false, &file_id);
SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id);
SCLogDebug!("FID {:?} found at tx {} => {:?}", file_fid, tx.id, tx);
}
true
},
@ -988,19 +990,22 @@ pub fn smb1_write_request_record<'b>(state: &mut SMBState, r: &SmbRecord<'b>, an
let vercmd = SMBVerCmdStat::new1_with_ntstatus(command, r.nt_status);
smb_write_dcerpc_record(state, vercmd, hdr, rd.data);
} else {
let (tx, files, flags) = state.new_file_tx(&file_fid, &file_name, Direction::ToServer);
let tx = state.new_file_tx(&file_fid, &file_name, Direction::ToServer);
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id);
if rd.offset < tdf.file_tracker.tracked {
set_event_fileoverlap = true;
}
let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, rd.offset,
rd.len, false, &file_id);
tdf.share_name = share_name;
SCLogDebug!("files {:?}", files);
SCLogDebug!("tdf {:?}", tdf);
}
tx.vercmd.set_smb1_cmd(SMB1_COMMAND_WRITE_ANDX);
SCLogDebug!("FID {:?} found at tx {} => {:?}", file_fid, tx.id, tx);
}
}
if set_event_fileoverlap {
@ -1054,13 +1059,14 @@ pub fn smb1_read_response_record<'b>(state: &mut SMBState, r: &SmbRecord<'b>, an
};
let mut set_event_fileoverlap = false;
let found = match state.get_file_tx_by_fuid(&file_fid, Direction::ToClient) {
Some((tx, files, flags)) => {
Some(tx) => {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id);
if offset < tdf.file_tracker.tracked {
set_event_fileoverlap = true;
}
let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, offset,
rd.len, false, &file_id);
@ -1070,13 +1076,14 @@ pub fn smb1_read_response_record<'b>(state: &mut SMBState, r: &SmbRecord<'b>, an
None => { false },
};
if !found {
let (tx, files, flags) = state.new_file_tx(&file_fid, &file_name, Direction::ToClient);
let tx = state.new_file_tx(&file_fid, &file_name, Direction::ToClient);
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
SCLogDebug!("FID {:?} found at tx {}", file_fid, tx.id);
if offset < tdf.file_tracker.tracked {
set_event_fileoverlap = true;
}
let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, offset,
rd.len, false, &file_id);

24
rust/src/smb/smb2.rs
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2017 Open Information Security Foundation
/* Copyright (C) 2017-2022 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -157,7 +157,7 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
let mut set_event_fileoverlap = false;
// look up existing tracker and if we have it update it
let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToClient) {
Some((tx, files, flags)) => {
Some(tx) => {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
if offset < tdf.file_tracker.tracked {
@ -170,6 +170,7 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
state.set_event(SMBEvent::ReadQueueCntExceeded);
state.set_skip(Direction::ToClient, rd.len, rd.data.len() as u32);
} else {
let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&tdf.file_name, rd.data, offset,
rd.len, false, &file_id);
@ -225,12 +226,11 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
Some(n) => { n.to_vec() }
None => { b"<unknown>".to_vec() }
};
let (tx, files, flags) = state.new_file_tx(&file_guid, &file_name, Direction::ToClient);
let tx = state.new_file_tx(&file_guid, &file_name, Direction::ToClient);
tx.vercmd.set_smb2_cmd(SMB2_COMMAND_READ);
tx.hdr = SMBCommonHdr::new(SMBHDR_TYPE_HEADER,
r.session_id, r.tree_id, 0); // TODO move into new_file_tx
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
tdf.share_name = share_name;
let file_id : u32 = tx.id as u32;
@ -244,6 +244,7 @@ pub fn smb2_read_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
state.set_event(SMBEvent::ReadQueueCntExceeded);
state.set_skip(Direction::ToClient, rd.len, rd.data.len() as u32);
} else {
let (files, flags) = tdf.files.get(Direction::ToClient);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, rd.data, offset,
rd.len, false, &file_id);
@ -296,7 +297,7 @@ pub fn smb2_write_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
let mut set_event_fileoverlap = false;
let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToServer) {
Some((tx, files, flags)) => {
Some(tx) => {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let file_id : u32 = tx.id as u32;
if wr.wr_offset < tdf.file_tracker.tracked {
@ -309,6 +310,7 @@ pub fn smb2_write_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
state.set_event(SMBEvent::WriteQueueCntExceeded);
state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32);
} else {
let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, wr.data, wr.wr_offset,
wr.wr_len, false, &file_id);
@ -360,7 +362,7 @@ pub fn smb2_write_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
SCLogDebug!("non-DCERPC pipe: skip rest of the record");
state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32);
} else {
let (tx, files, flags) = state.new_file_tx(&file_guid, &file_name, Direction::ToServer);
let tx = state.new_file_tx(&file_guid, &file_name, Direction::ToServer);
tx.vercmd.set_smb2_cmd(SMB2_COMMAND_WRITE);
tx.hdr = SMBCommonHdr::new(SMBHDR_TYPE_HEADER,
r.session_id, r.tree_id, 0); // TODO move into new_file_tx
@ -377,6 +379,7 @@ pub fn smb2_write_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
state.set_event(SMBEvent::WriteQueueCntExceeded);
state.set_skip(Direction::ToServer, wr.wr_len, wr.data.len() as u32);
} else {
let (files, flags) = tdf.files.get(Direction::ToServer);
filetracker_newchunk(&mut tdf.file_tracker, files, flags,
&file_name, wr.data, wr.wr_offset,
wr.wr_len, false, &file_id);
@ -587,9 +590,10 @@ pub fn smb2_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
match parse_smb2_request_close(r.data) {
Ok((_, cd)) => {
let found_ts = match state.get_file_tx_by_fuid(&cd.guid.to_vec(), Direction::ToServer) {
Some((tx, files, flags)) => {
Some(tx) => {
if !tx.request_done {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToServer);
tdf.file_tracker.close(files, flags);
}
}
@ -601,9 +605,10 @@ pub fn smb2_request_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
None => { false },
};
let found_tc = match state.get_file_tx_by_fuid(&cd.guid.to_vec(), Direction::ToClient) {
Some((tx, files, flags)) => {
Some(tx) => {
if !tx.request_done {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToClient);
tdf.file_tracker.close(files, flags);
}
}
@ -701,9 +706,10 @@ pub fn smb2_response_record<'b>(state: &mut SMBState, r: &Smb2Record<'b>)
},
};
let found = match state.get_file_tx_by_fuid(&file_guid, Direction::ToClient) {
Some((tx, files, flags)) => {
Some(tx) => {
if !tx.request_done {
if let Some(SMBTransactionTypeData::FILE(ref mut tdf)) = tx.type_data {
let (files, flags) = tdf.files.get(Direction::ToClient);
tdf.file_tracker.close(files, flags);
}
}

2
rust/src/snmp/snmp.rs
Unescape Escape View File

@ -399,7 +399,7 @@ pub unsafe extern "C" fn rs_register_snmp_parser() {
get_eventinfo_byid : Some(SNMPEvent::get_event_info_by_id),
localstorage_new : None,
localstorage_free : None,
get_files : None,
get_tx_files : None,
get_tx_iterator : Some(applayer::state_get_tx_iterator::<SNMPState, SNMPTransaction>),
get_tx_data : rs_snmp_get_tx_data,
get_state_data : rs_snmp_get_state_data,

2
rust/src/ssh/ssh.rs
Unescape Escape View File

@ -463,7 +463,7 @@ pub unsafe extern "C" fn rs_ssh_register_parser() {
get_eventinfo_byid: Some(SSHEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: None,
get_tx_data: rs_ssh_get_tx_data,
get_state_data: rs_ssh_get_state_data,

2
rust/src/telnet/telnet.rs
Unescape Escape View File

@ -550,7 +550,7 @@ pub unsafe extern "C" fn rs_telnet_register_parser() {
get_eventinfo_byid : Some(TelnetEvent::get_event_info_by_id),
localstorage_new: None,
localstorage_free: None,
get_files: None,
get_tx_files: None,
get_tx_iterator: Some(applayer::state_get_tx_iterator::<TelnetState, TelnetTransaction>),
get_tx_data: rs_telnet_get_tx_data,
get_state_data: rs_telnet_get_state_data,

18
src/app-layer-ftp.c
Unescape Escape View File

@ -1097,14 +1097,20 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state,
{
const uint8_t *input = StreamSliceGetData(&stream_slice);
uint32_t input_len = StreamSliceGetDataLen(&stream_slice);
uint16_t flags = FileFlowToFlags(f, direction) | FILE_USE_DETECT;
int ret = 0;
const bool eof = (flags & STREAM_TOSERVER)
const bool eof = (direction & STREAM_TOSERVER)
? AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TS) != 0
: AppLayerParserStateIssetFlag(pstate, APP_LAYER_PARSER_EOF_TC) != 0;
ftpdata_state->tx_data.file_flags |= ftpdata_state->state_data.file_flags;
/* we depend on detection engine for file pruning */
const uint16_t flags =
FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction) | FILE_USE_DETECT;
int ret = 0;
SCLogDebug("FTP-DATA input_len %u flags %04x dir %d/%s EOF %s", input_len, flags, direction,
(direction & STREAM_TOSERVER) ? "toserver" : "toclient", eof ? "true" : "false");
SCLogDebug("FTP-DATA flags %04x dir %d", flags, direction);
if (input_len && ftpdata_state->files == NULL) {
struct FtpTransferCmd *data =
(struct FtpTransferCmd *)FlowGetStorageById(f, AppLayerExpectationGetFlowId());
@ -1299,9 +1305,9 @@ static int FTPDataGetAlstateProgress(void *tx, uint8_t direction)
return FTPDATA_STATE_FINISHED;
}
static FileContainer *FTPDataStateGetFiles(void *state, uint8_t direction)
static FileContainer *FTPDataStateGetTxFiles(void *tx, uint8_t direction)
{
FtpDataState *ftpdata_state = (FtpDataState *)state;
FtpDataState *ftpdata_state = (FtpDataState *)tx;
if (direction != ftpdata_state->direction)
SCReturnPtr(NULL, "FileContainer");
@ -1428,7 +1434,7 @@ void RegisterFTPParsers(void)
AppLayerParserRegisterParserAcceptableDataDirection(IPPROTO_TCP, ALPROTO_FTPDATA, STREAM_TOSERVER | STREAM_TOCLIENT);
AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateTransactionFree);
AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateGetFiles);
AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataStateGetTxFiles);
AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetTx);
AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_FTPDATA, FTPDataGetTxData);

218
src/app-layer-htp-file.c
Unescape Escape View File

@ -21,7 +21,7 @@
* \author Victor Julien <victor@inliniac.net>
*
* This file provides HTTP protocol file handling support for the engine
* using HTP library.
* using the HTP library.
*/
#include "suricata.h"
@ -87,55 +87,16 @@ int HTPFileOpen(HtpState *s, HtpTxUserData *tx, const uint8_t *filename, uint16_
SCLogDebug("data %p data_len %"PRIu32, data, data_len);
if (s == NULL) {
SCReturnInt(-1);
}
if (direction & STREAM_TOCLIENT) {
if (s->files_tc == NULL) {
s->files_tc = FileContainerAlloc();
if (s->files_tc == NULL) {
retval = -1;
goto end;
}
}
files = s->files_tc;
flags = FileFlowToFlags(s->f, STREAM_TOCLIENT);
if ((s->flags & HTP_FLAG_STORE_FILES_TS) ||
((s->flags & HTP_FLAG_STORE_FILES_TX_TS) && txid == s->store_tx_id)) {
flags |= FILE_STORE;
flags &= ~FILE_NOSTORE;
} else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TC)) {
flags |= FILE_NOSTORE;
}
files = &tx->files_tc;
flags = FileFlowFlagsToFlags(tx->tx_data.file_flags, STREAM_TOCLIENT);
sbcfg = &s->cfg->response.sbcfg;
// we shall not open a new file if there is a current one
DEBUG_VALIDATE_BUG_ON(s->file_range != NULL);
DEBUG_VALIDATE_BUG_ON(tx->file_range != NULL);
} else {
if (s->files_ts == NULL) {
s->files_ts = FileContainerAlloc();
if (s->files_ts == NULL) {
retval = -1;
goto end;
}
}
files = s->files_ts;
flags = FileFlowToFlags(s->f, STREAM_TOSERVER);
if ((s->flags & HTP_FLAG_STORE_FILES_TC) ||
((s->flags & HTP_FLAG_STORE_FILES_TX_TC) && txid == s->store_tx_id)) {
flags |= FILE_STORE;
flags &= ~FILE_NOSTORE;
} else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TS)) {
flags |= FILE_NOSTORE;
}
files = &tx->files_ts;
flags = FileFlowFlagsToFlags(tx->tx_data.file_flags, STREAM_TOSERVER);
sbcfg = &s->cfg->request.sbcfg;
}
@ -146,10 +107,8 @@ int HTPFileOpen(HtpState *s, HtpTxUserData *tx, const uint8_t *filename, uint16_
retval = -1;
}
FileSetTx(files->tail, txid);
tx->tx_data.files_opened++;
end:
SCReturnInt(retval);
}
@ -230,30 +189,13 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena
return HTPFileOpen(s, txud, filename, filename_len, data, data_len, txid, STREAM_TOCLIENT);
}
flags = FileFlowToFlags(s->f, STREAM_TOCLIENT);
if ((s->flags & HTP_FLAG_STORE_FILES_TS) ||
((s->flags & HTP_FLAG_STORE_FILES_TX_TS) && txid == s->store_tx_id)) {
flags |= FILE_STORE;
flags &= ~FILE_NOSTORE;
} else if (!(flags & FILE_STORE) && (s->f->file_flags & FLOWFILE_NO_STORE_TC)) {
flags |= FILE_NOSTORE;
}
FileContainer * files = s->files_tc;
if (files == NULL) {
s->files_tc = FileContainerAlloc();
if (s->files_tc == NULL) {
// no need to fall back to classic open if we cannot allocate the file container
SCReturnInt(-1);
}
files = s->files_tc;
}
FileContainer *files = &txud->files_tc;
// we open a file for this specific range
if (FileOpenFileWithId(files, &s->cfg->response.sbcfg, s->file_track_id++, filename,
filename_len, data, data_len, flags) != 0) {
SCReturnInt(-1);
}
FileSetTx(files->tail, txid);
txud->tx_data.files_opened++;
if (FileSetRange(files, crparsed.start, crparsed.end) < 0) {
@ -279,11 +221,11 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena
// do not reassemble file without host info
SCReturnInt(0);
}
DEBUG_VALIDATE_BUG_ON(s->file_range);
s->file_range = HttpRangeContainerOpenFile(keyurl, keylen, s->f, &crparsed,
DEBUG_VALIDATE_BUG_ON(htud->file_range);
htud->file_range = HttpRangeContainerOpenFile(keyurl, keylen, s->f, &crparsed,
&s->cfg->response.sbcfg, filename, filename_len, flags, data, data_len);
SCFree(keyurl);
if (s->file_range == NULL) {
if (htud->file_range == NULL) {
SCReturnInt(-1);
}
SCReturnInt(0);
@ -292,7 +234,7 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena
/**
* \brief Store a chunk of data in the flow
*
* \param s http state
* \param tx HtpTxUserData
* \param data data chunk (if any)
* \param data_len length of the data portion
* \param direction flow direction
@ -301,8 +243,7 @@ int HTPFileOpenWithRange(HtpState *s, HtpTxUserData *txud, const uint8_t *filena
* \retval -1 error
* \retval -2 file doesn't need storing
*/
int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len,
uint8_t direction)
int HTPFileStoreChunk(HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t direction)
{
SCEnter();
@ -310,15 +251,12 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len,
int result = 0;
FileContainer *files = NULL;
if (s == NULL) {
SCReturnInt(-1);
}
if (direction & STREAM_TOCLIENT) {
files = s->files_tc;
files = &tx->files_tc;
} else {
files = s->files_ts;
files = &tx->files_ts;
}
SCLogDebug("files %p data %p data_len %" PRIu32, files, data, data_len);
if (files == NULL) {
SCLogDebug("no files in state");
@ -326,8 +264,8 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len,
goto end;
}
if (s->file_range != NULL) {
if (HttpRangeAppendData(s->file_range, data, data_len) < 0) {
if (tx->file_range != NULL) {
if (HttpRangeAppendData(tx->file_range, data, data_len) < 0) {
SCLogDebug("Failed to append data");
}
}
@ -339,6 +277,7 @@ int HTPFileStoreChunk(HtpState *s, const uint8_t *data, uint32_t data_len,
} else if (result == -2) {
retval = -2;
}
SCLogDebug("result %u", result);
end:
SCReturnInt(retval);
@ -377,7 +316,7 @@ bool HTPFileCloseHandleRange(FileContainer *files, const uint16_t flags, HttpRan
/**
* \brief Close the file in the flow
*
* \param s http state
* \param tx HtpTxUserData
* \param data data chunk if any
* \param data_len length of the data portion
* \param flags flags to indicate events
@ -390,24 +329,23 @@ bool HTPFileCloseHandleRange(FileContainer *files, const uint16_t flags, HttpRan
* \retval -1 error
* \retval -2 not storing files on this flow/tx
*/
int HTPFileClose(HtpState *s, HtpTxUserData *htud, const uint8_t *data, uint32_t data_len,
uint8_t flags, uint8_t direction)
int HTPFileClose(
HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t direction)
{
SCEnter();
SCLogDebug("flags %04x FILE_TRUNCATED %s", flags, (flags & FILE_TRUNCATED) ? "true" : "false");
int retval = 0;
int result = 0;
FileContainer *files = NULL;
if (s == NULL) {
SCReturnInt(-1);
}
if (direction & STREAM_TOCLIENT) {
files = s->files_tc;
files = &tx->files_tc;
} else {
files = s->files_ts;
files = &tx->files_ts;
}
SCLogDebug("files %p data %p data_len %" PRIu32, files, data, data_len);
if (files == NULL) {
retval = -1;
@ -420,14 +358,15 @@ int HTPFileClose(HtpState *s, HtpTxUserData *htud, const uint8_t *data, uint32_t
} else if (result == -2) {
retval = -2;
}
SCLogDebug("result %u", result);
if (s->file_range != NULL) {
bool added = HTPFileCloseHandleRange(files, flags, s->file_range, data, data_len);
if (tx->file_range != NULL) {
bool added = HTPFileCloseHandleRange(files, flags, tx->file_range, data, data_len);
if (added) {
htud->tx_data.files_opened++;
tx->tx_data.files_opened++;
}
HttpRangeFreeBlock(s->file_range);
s->file_range = NULL;
HttpRangeFreeBlock(tx->file_range);
tx->file_range = NULL;
}
end:
@ -556,9 +495,10 @@ static int HTPFileParserTest02(void)
FAIL_IF_NULL(tx);
FAIL_IF_NULL(tx->request_method);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
FAIL_IF_NULL(http_state->files_ts);
FAIL_IF_NULL(http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FAIL_IF_NULL(tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
@ -649,11 +589,12 @@ static int HTPFileParserTest03(void)
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
FAIL_IF_NULL(http_state->files_ts);
FAIL_IF_NULL(http_state->files_ts->head);
FAIL_IF_NULL(http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
FAIL_IF(FileDataSize(http_state->files_ts->head) != 11);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FAIL_IF_NULL(tx_ud->files_ts.head);
FAIL_IF_NULL(tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
FAIL_IF(FileDataSize(tx_ud->files_ts.head) != 11);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
@ -744,10 +685,11 @@ static int HTPFileParserTest04(void)
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
FAIL_IF_NULL(http_state->files_ts);
FAIL_IF_NULL(http_state->files_ts->head);
FAIL_IF_NULL(http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FAIL_IF_NULL(tx_ud->files_ts.head);
FAIL_IF_NULL(tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
@ -809,19 +751,20 @@ static int HTPFileParserTest05(void)
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
FAIL_IF_NULL(http_state->files_ts);
FAIL_IF_NULL(http_state->files_ts->head);
FAIL_IF_NULL(http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FAIL_IF_NULL(tx_ud->files_ts.head);
FAIL_IF_NULL(tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
FAIL_IF(http_state->files_ts->head == http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->head->next != http_state->files_ts->tail);
FAIL_IF(tx_ud->files_ts.head == tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.head->next != tx_ud->files_ts.tail);
FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->head->sb,
(uint8_t *)"filecontent", 11) != 1);
FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.head->sb, (uint8_t *)"filecontent", 11) !=
1);
FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
(uint8_t *)"FILECONTENT", 11) != 1);
FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) !=
1);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
UTHFreeFlow(f);
@ -883,19 +826,20 @@ static int HTPFileParserTest06(void)
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
FAIL_IF_NULL(http_state->files_ts);
FAIL_IF_NULL(http_state->files_ts->head);
FAIL_IF_NULL(http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FAIL_IF_NULL(tx_ud->files_ts.head);
FAIL_IF_NULL(tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
FAIL_IF(http_state->files_ts->head == http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->head->next != http_state->files_ts->tail);
FAIL_IF(tx_ud->files_ts.head == tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.head->next != tx_ud->files_ts.tail);
FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->head->sb,
(uint8_t *)"filecontent", 11) != 1);
FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.head->sb, (uint8_t *)"filecontent", 11) !=
1);
FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
(uint8_t *)"FILECONTENT", 11) != 1);
FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) !=
1);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
@ -946,12 +890,14 @@ static int HTPFileParserTest07(void)
FAIL_IF_NULL(tx->request_method);
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
FAIL_IF_NULL(http_state->files_ts);
FAIL_IF_NULL(http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FAIL_IF_NULL(tx_ud->files_ts.head);
FAIL_IF_NULL(tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
(uint8_t *)"FILECONTENT", 11) != 1);
FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) !=
1);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
@ -1267,12 +1213,14 @@ static int HTPFileParserTest11(void)
FAIL_IF(memcmp(bstr_util_strdup_to_c(tx->request_method), "POST", 4) != 0);
FAIL_IF_NULL(http_state->files_ts);
FAIL_IF_NULL(http_state->files_ts->tail);
FAIL_IF(http_state->files_ts->tail->state != FILE_STATE_CLOSED);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FAIL_IF_NULL(tx_ud->files_ts.head);
FAIL_IF_NULL(tx_ud->files_ts.tail);
FAIL_IF(tx_ud->files_ts.tail->state != FILE_STATE_CLOSED);
FAIL_IF(StreamingBufferCompareRawData(http_state->files_ts->tail->sb,
(uint8_t *)"FILECONTENT", 11) != 1);
FAIL_IF(StreamingBufferCompareRawData(tx_ud->files_ts.tail->sb, (uint8_t *)"FILECONTENT", 11) !=
1);
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);

8
src/app-layer-htp-file.h
Unescape Escape View File

@ -29,13 +29,15 @@
int HTPFileOpen(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *, uint32_t,
uint64_t, uint8_t);
int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range);
int HTPFileOpenWithRange(HtpState *, HtpTxUserData *, const uint8_t *, uint16_t, const uint8_t *,
uint32_t, uint64_t, bstr *rawvalue, HtpTxUserData *htud);
bool HTPFileCloseHandleRange(
FileContainer *, const uint16_t, HttpRangeContainerBlock *, const uint8_t *, uint32_t);
int HTPFileStoreChunk(HtpState *, const uint8_t *, uint32_t, uint8_t);
int HTPFileClose(HtpState *, HtpTxUserData *, const uint8_t *, uint32_t, uint8_t, uint8_t);
int HTPFileStoreChunk(HtpTxUserData *, const uint8_t *, uint32_t, uint8_t);
int HTPParseContentRange(bstr *rawvalue, HTTPContentRange *range);
int HTPFileClose(HtpTxUserData *tx, const uint8_t *data, uint32_t data_len, uint8_t flags,
uint8_t direction);
void HTPFileParserRegisterTests(void);

84
src/app-layer-htp.c
Unescape Escape View File

@ -366,6 +366,12 @@ static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
if (htud->tx_data.de_state != NULL) {
DetectEngineStateFree(htud->tx_data.de_state);
}
if (htud->file_range) {
HTPFileCloseHandleRange(&htud->files_tc, 0, htud->file_range, NULL, 0);
HttpRangeFreeBlock(htud->file_range);
}
FileContainerRecycle(&htud->files_ts);
FileContainerRecycle(&htud->files_tc);
HTPFree(htud, sizeof(HtpTxUserData));
}
}
@ -402,13 +408,6 @@ void HTPStateFree(void *state)
htp_connp_destroy_all(s->connp);
}
if (s->file_range) {
HTPFileCloseHandleRange(s->files_tc, 0, s->file_range, NULL, 0);
HttpRangeFreeBlock(s->file_range);
}
FileContainerFree(s->files_ts);
FileContainerFree(s->files_tc);
HTPFree(s, sizeof(HtpState));
#ifdef DEBUG
@ -516,10 +515,11 @@ void AppLayerHtpNeedFileInspection(void)
SCReturn;
}
static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
static void AppLayerHtpSetStreamDepthFlag(void *tx, const uint8_t flags)
{
HtpTxUserData *tx_ud = (HtpTxUserData *) htp_tx_get_user_data((htp_tx_t *)tx);
if (tx_ud) {
SCLogDebug("setting HTP_STREAM_DEPTH_SET, flags %02x", flags);
if (flags & STREAM_TOCLIENT) {
tx_ud->tcflags |= HTP_STREAM_DEPTH_SET;
} else {
@ -530,9 +530,12 @@ static void AppLayerHtpSetStreamDepthFlag(void *tx, uint8_t flags)
static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t flags)
{
SCLogDebug("cfg->body_limit %u stream_depth %u body->content_len_so_far %" PRIu64,
cfg->body_limit, FileReassemblyDepth(), body->content_len_so_far);
if (flags & HTP_STREAM_DEPTH_SET) {
uint32_t stream_depth = FileReassemblyDepth();
if (body->content_len_so_far < (uint64_t)stream_depth || stream_depth == 0) {
SCLogDebug("true");
return true;
}
} else {
@ -540,6 +543,7 @@ static bool AppLayerHtpCheckDepth(const HTPCfgDir *cfg, HtpBody *body, uint8_t f
return true;
}
}
SCLogDebug("false");
return false;
}
@ -1422,8 +1426,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
printf("FILEDATA (final chunk) END: \n");
#endif
if (!(htud->tsflags & HTP_DONTSTORE)) {
if (HTPFileClose(hstate, htud, filedata, filedata_len, flags, STREAM_TOSERVER) ==
-1) {
if (HTPFileClose(htud, filedata, filedata_len, flags, STREAM_TOSERVER) == -1) {
goto end;
}
}
@ -1444,8 +1447,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
#endif
if (!(htud->tsflags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate, filedata,
filedata_len, STREAM_TOSERVER);
result = HTPFileStoreChunk(htud, filedata, filedata_len, STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1545,7 +1547,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
} else if (result == -2) {
htud->tsflags |= HTP_DONTSTORE;
} else {
if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
if (HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
goto end;
}
}
@ -1612,7 +1614,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
} else if (result == -2) {
htud->tsflags |= HTP_DONTSTORE;
} else {
if (HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
if (HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER) == -1) {
goto end;
}
}
@ -1700,7 +1702,7 @@ static int HtpRequestBodyHandlePOSTorPUT(HtpState *hstate, HtpTxUserData *htud,
/* otherwise, just store the data */
if (!(htud->tsflags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOSERVER);
result = HTPFileStoreChunk(htud, data, data_len, STREAM_TOSERVER);
if (result == -1) {
goto end;
} else if (result == -2) {
@ -1784,7 +1786,7 @@ static int HtpResponseBodyHandle(HtpState *hstate, HtpTxUserData *htud,
/* otherwise, just store the data */
if (!(htud->tcflags & HTP_DONTSTORE)) {
result = HTPFileStoreChunk(hstate, data, data_len, STREAM_TOCLIENT);
result = HTPFileStoreChunk(htud, data, data_len, STREAM_TOCLIENT);
SCLogDebug("result %d", result);
if (result == -1) {
goto end;
@ -1834,6 +1836,8 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
if (tx_ud == NULL) {
SCReturnInt(HTP_OK);
}
tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
if (!tx_ud->response_body_init) {
tx_ud->response_body_init = 1;
@ -1898,7 +1902,7 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
} else {
if (tx_ud->tsflags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
(void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
(void)HTPFileClose(tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOSERVER);
tx_ud->tsflags &= ~HTP_FILENAME_SET;
}
}
@ -1962,6 +1966,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
if (tx_ud == NULL) {
SCReturnInt(HTP_OK);
}
tx_ud->tx_data.file_flags |= hstate->state_data.file_flags;
if (!tx_ud->request_body_init) {
tx_ud->request_body_init = 1;
}
@ -1988,7 +1993,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
} else {
if (tx_ud->tcflags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
(void)HTPFileClose(hstate, tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
(void)HTPFileClose(tx_ud, NULL, 0, FILE_TRUNCATED, STREAM_TOCLIENT);
tx_ud->tcflags &= ~HTP_FILENAME_SET;
}
}
@ -2209,7 +2214,7 @@ static int HTPCallbackRequestComplete(htp_tx_t *tx)
if (htud != NULL) {
if (htud->tsflags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
(void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
(void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER);
htud->tsflags &= ~HTP_FILENAME_SET;
if (abs_right_edge < (uint64_t)UINT32_MAX) {
StreamTcpReassemblySetMinInspectDepth(
@ -2264,7 +2269,7 @@ static int HTPCallbackResponseComplete(htp_tx_t *tx)
if (htud != NULL) {
if (htud->tcflags & HTP_FILENAME_SET) {
SCLogDebug("closing file that was being stored");
(void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
(void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOCLIENT);
htud->tcflags &= ~HTP_FILENAME_SET;
}
}
@ -3000,18 +3005,18 @@ void AppLayerHtpPrintStats(void)
* \param direction flow direction
* \retval files files ptr
*/
static FileContainer *HTPStateGetFiles(void *state, uint8_t direction)
static FileContainer *HTPGetTxFiles(void *txv, uint8_t direction)
{
if (state == NULL)
return NULL;
HtpState *http_state = (HtpState *)state;
if (direction & STREAM_TOCLIENT) {
SCReturnPtr(http_state->files_tc, "FileContainer");
} else {
SCReturnPtr(http_state->files_ts, "FileContainer");
htp_tx_t *tx = (htp_tx_t *)txv;
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
if (tx_ud) {
if (direction & STREAM_TOCLIENT) {
SCReturnPtr(&tx_ud->files_tc, "FileContainer");
} else {
SCReturnPtr(&tx_ud->files_ts, "FileContainer");
}
}
SCReturnPtr(NULL, "FileContainer");
}
static int HTPStateGetAlstateProgress(void *tx, uint8_t direction)
@ -3092,14 +3097,6 @@ static int HTPStateGetEventInfoById(int event_id, const char **event_name,
return 0;
}
static void HTPStateTruncate(void *state, uint8_t direction)
{
FileContainer *fc = HTPStateGetFiles(state, direction);
if (fc != NULL) {
FileTruncateAllOpenFiles(fc);
}
}
static AppLayerTxData *HTPGetTxData(void *vtx)
{
htp_tx_t *tx = (htp_tx_t *)vtx;
@ -3188,7 +3185,7 @@ void RegisterHTPParsers(void)
if (AppLayerParserConfParserEnabled("tcp", proto_name)) {
AppLayerParserRegisterStateFuncs(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateAlloc, HTPStateFree);
AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTransactionFree);
AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetFiles);
AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxFiles);
AppLayerParserRegisterGetStateProgressFunc(
IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetAlstateProgress);
AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetTxCnt);
@ -3200,7 +3197,6 @@ void RegisterHTPParsers(void)
AppLayerParserRegisterGetEventInfoById(
IPPROTO_TCP, ALPROTO_HTTP1, HTPStateGetEventInfoById);
AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPStateTruncate);
AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetTxData);
AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_HTTP1, HTPGetStateData);
@ -6059,8 +6055,7 @@ static int HTPBodyReassemblyTest01(void)
goto end;
}
if (hstate.files_ts != NULL)
goto end;
FAIL_IF_NOT_NULL(htud.files_ts.head);
result = 1;
end:
@ -6944,7 +6939,10 @@ libhtp:\n\
http_state = f.alstate;
FAIL_IF_NULL(http_state);
FileContainer *ffc = HTPStateGetFiles(http_state, STREAM_TOCLIENT);
void *tx_ptr = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, http_state, 0);
FAIL_IF_NULL(tx_ptr);
FileContainer *ffc = HTPGetTxFiles(tx_ptr, STREAM_TOCLIENT);
FAIL_IF_NULL(ffc);
File *ptr = ffc->head;

18
src/app-layer-htp.h
Unescape Escape View File

@ -66,12 +66,9 @@
// 0x0001 not used
#define HTP_FLAG_STATE_CLOSED_TS 0x0002 /**< Flag to indicate that HTTP
connection is closed */
#define HTP_FLAG_STATE_CLOSED_TC 0x0004 /**< Flag to indicate that HTTP
connection is closed */
#define HTP_FLAG_STORE_FILES_TS 0x0040
#define HTP_FLAG_STORE_FILES_TC 0x0080
#define HTP_FLAG_STORE_FILES_TX_TS 0x0100
#define HTP_FLAG_STORE_FILES_TX_TC 0x0200
#define HTP_FLAG_STATE_CLOSED_TC \
0x0004 /**< Flag to indicate that HTTP \
connection is closed */
enum {
HTP_BODY_REQUEST_NONE = 0,
@ -240,7 +237,11 @@ typedef struct HtpTxUserData_ {
*/
uint8_t *boundary;
HttpRangeContainerBlock *file_range; /**< used to assign track ids to range file */
AppLayerTxData tx_data;
FileContainer files_ts;
FileContainer files_tc;
} HtpTxUserData;
typedef struct HtpState_ {
@ -251,14 +252,11 @@ typedef struct HtpState_ {
Flow *f; /**< Needed to retrieve the original flow when using HTPLib callbacks */
uint64_t transaction_cnt;
uint64_t store_tx_id;
FileContainer *files_ts;
FileContainer *files_tc;
const struct HTPCfgRec_ *cfg;
uint16_t flags;
uint16_t events;
uint16_t htp_messages_offset; /**< offset into conn->messages list */
uint32_t file_track_id; /**< used to assign file track ids to files */
HttpRangeContainerBlock *file_range; /**< used to assign track ids to range file */
uint32_t file_track_id; /**< used to assign file track ids to files */
uint64_t last_request_data_stamp;
uint64_t last_response_data_stamp;
StreamSlice *slice;

5
src/app-layer-ike.c
Unescape Escape View File

@ -57,6 +57,7 @@ static int IkeParserTest(void)
Flow f;
TcpSession ssn;
AppLayerParserThreadCtx *alp_tctx = AppLayerParserThreadCtxAlloc();
FAIL_IF_NULL(alp_tctx);
memset(&f, 0, sizeof(f));
memset(&ssn, 0, sizeof(ssn));
@ -173,9 +174,7 @@ static int IkeParserTest(void)
FAIL_IF_NOT(ret[2] == 5); // log_id
FAIL_IF_NOT(ret[3] == 5); // min_id
if (alp_tctx != NULL) {
AppLayerParserThreadCtxFree(alp_tctx);
}
AppLayerParserThreadCtxFree(alp_tctx);
StreamTcpFreeConfig(true);
FLOW_DESTROY(&f);
PASS;

71
src/app-layer-parser.c
Unescape Escape View File

@ -112,7 +112,10 @@ typedef struct AppLayerParserProtoCtx_
void (*LocalStorageFree)(void *);
void (*Truncate)(void *, uint8_t);
FileContainer *(*StateGetFiles)(void *, uint8_t);
/** get FileContainer reference from the TX. MUST return a non-NULL reference if the TX
* has or may have files in the requested direction at some point. */
FileContainer *(*GetTxFiles)(void *, uint8_t);
int (*StateGetProgress)(void *alstate, uint8_t direction);
uint64_t (*StateGetTxCnt)(void *alstate);
@ -472,13 +475,12 @@ void AppLayerParserRegisterLocalStorageFunc(uint8_t ipproto, AppProto alproto,
SCReturn;
}
void AppLayerParserRegisterGetFilesFunc(uint8_t ipproto, AppProto alproto,
FileContainer *(*StateGetFiles)(void *, uint8_t))
void AppLayerParserRegisterGetTxFilesFunc(
uint8_t ipproto, AppProto alproto, FileContainer *(*GetTxFiles)(void *, uint8_t))
{
SCEnter();
alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetFiles =
StateGetFiles;
alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxFiles = GetTxFiles;
SCReturn;
}
@ -905,21 +907,33 @@ AppLayerDecoderEvents *AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alp
SCReturnPtr(ptr, "AppLayerDecoderEvents *");
}
FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction)
FileContainer *AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t direction)
{
SCEnter();
FileContainer *ptr = NULL;
if (alp_ctx.ctxs[f->protomap][f->alproto].StateGetFiles != NULL)
{
ptr = alp_ctx.ctxs[f->protomap][f->alproto].
StateGetFiles(f->alstate, direction);
if (alp_ctx.ctxs[f->protomap][f->alproto].GetTxFiles != NULL) {
ptr = alp_ctx.ctxs[f->protomap][f->alproto].GetTxFiles(tx, direction);
}
SCReturnPtr(ptr, "FileContainer *");
}
static void AppLayerParserFileTxHousekeeping(const Flow *f, void *tx, const uint8_t pkt_dir)
{
FileContainer *fc = AppLayerParserGetTxFiles(f, tx, pkt_dir);
if (fc) {
if (AppLayerParserStateIssetFlag(f->alparser, (pkt_dir == STREAM_TOSERVER)
? APP_LAYER_PARSER_TRUNC_TS
: APP_LAYER_PARSER_TRUNC_TC) != 0) {
FileTruncateAllOpenFiles(fc);
}
FilePrune(fc);
}
}
#define IS_DISRUPTED(flags) ((flags) & (STREAM_DEPTH | STREAM_GAP))
extern int g_detect_disabled;
@ -976,6 +990,8 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir)
SCLogDebug("%p/%"PRIu64" checking", tx, i);
AppLayerParserFileTxHousekeeping(f, tx, pkt_dir);
const int tx_progress_tc =
AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags);
if (tx_progress_tc < tx_end_state_tc) {
@ -1046,14 +1062,19 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir)
/* if file logging is enabled, we keep a tx active while some of the files aren't
* logged yet. */
if (txd && txd->files_opened) {
if (g_file_logger_enabled && txd->files_opened != txd->files_logged) {
skipped = true;
goto next;
}
if (g_filedata_logger_enabled && txd->files_opened != txd->files_stored) {
skipped = true;
goto next;
if (txd) {
SCLogDebug("files_opened %u files_logged %u files_stored %u", txd->files_opened,
txd->files_logged, txd->files_stored);
if (txd->files_opened) {
if (g_file_logger_enabled && txd->files_opened != txd->files_logged) {
skipped = true;
goto next;
}
if (g_filedata_logger_enabled && txd->files_opened != txd->files_stored) {
skipped = true;
goto next;
}
}
}
@ -1201,7 +1222,7 @@ int AppLayerParserSupportsFiles(uint8_t ipproto, AppProto alproto)
return AppLayerParserSupportsFiles(ipproto, ALPROTO_HTTP1) ||
AppLayerParserSupportsFiles(ipproto, ALPROTO_HTTP2);
}
if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetFiles != NULL)
if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxFiles != NULL)
return TRUE;
return FALSE;
}
@ -1320,6 +1341,18 @@ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow
}
SCLogDebug("alloced new app layer state %p (name %s)",
alstate, AppLayerGetProtoName(f->alproto));
/* set flow flags to state */
if (f->file_flags != 0) {
AppLayerStateData *sd = AppLayerParserGetStateData(f->proto, f->alproto, f->alstate);
if (sd != NULL) {
if ((sd->file_flags & f->file_flags) != f->file_flags) {
SCLogDebug("state data: updating file_flags %04x with flow file_flags %04x",
sd->file_flags, f->file_flags);
sd->file_flags |= f->file_flags;
}
}
}
} else {
SCLogDebug("using existing app layer state %p (name %s))",
alstate, AppLayerGetProtoName(f->alproto));

10
src/app-layer-parser.h
Unescape Escape View File

@ -174,12 +174,11 @@ void AppLayerParserRegisterOptionFlags(uint8_t ipproto, AppProto alproto,
void AppLayerParserRegisterStateFuncs(uint8_t ipproto, AppProto alproto,
void *(*StateAlloc)(void *, AppProto), void (*StateFree)(void *));
void AppLayerParserRegisterLocalStorageFunc(uint8_t ipproto, AppProto proto,
void *(*LocalStorageAlloc)(void),
void (*LocalStorageFree)(void *));
void AppLayerParserRegisterGetFilesFunc(uint8_t ipproto, AppProto alproto,
FileContainer *(*StateGetFiles)(void *, uint8_t));
void *(*LocalStorageAlloc)(void), void (*LocalStorageFree)(void *));
// void AppLayerParserRegisterGetEventsFunc(uint8_t ipproto, AppProto proto,
// AppLayerDecoderEvents *(*StateGetEvents)(void *) __attribute__((nonnull)));
void AppLayerParserRegisterGetTxFilesFunc(
uint8_t ipproto, AppProto alproto, FileContainer *(*GetTxFiles)(void *, uint8_t));
void AppLayerParserRegisterLoggerFuncs(uint8_t ipproto, AppProto alproto,
LoggerId (*StateGetTxLogged)(void *, void *),
void (*StateSetTxLogged)(void *, void *, LoggerId));
@ -242,7 +241,7 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p
AppLayerDecoderEvents *AppLayerParserGetDecoderEvents(AppLayerParserState *pstate);
void AppLayerParserSetDecoderEvents(AppLayerParserState *pstate, AppLayerDecoderEvents *devents);
AppLayerDecoderEvents *AppLayerParserGetEventsByTx(uint8_t ipproto, AppProto alproto, void *tx);
FileContainer *AppLayerParserGetFiles(const Flow *f, const uint8_t direction);
FileContainer *AppLayerParserGetTxFiles(const Flow *f, void *tx, const uint8_t direction);
int AppLayerParserGetStateProgress(uint8_t ipproto, AppProto alproto,
void *alstate, uint8_t direction);
uint64_t AppLayerParserGetTxCnt(const Flow *, void *alstate);
@ -313,5 +312,6 @@ void UTHAppLayerParserStateGetIds(void *ptr, uint64_t *i1, uint64_t *i2, uint64_
#endif
void AppLayerFramesFreeContainer(Flow *f);
void FileApplyTxFlags(const AppLayerTxData *txd, const uint8_t direction, File *file);
#endif /* __APP_LAYER_PARSER_H__ */

5
src/app-layer-register.c
Unescape Escape View File

@ -150,9 +150,8 @@ int AppLayerRegisterParser(const struct AppLayerParser *p, AppProto alproto)
AppLayerParserRegisterLocalStorageFunc(p->ip_proto, alproto,
p->LocalStorageAlloc, p->LocalStorageFree);
}
if (p->StateGetFiles) {
AppLayerParserRegisterGetFilesFunc(p->ip_proto, alproto,
p->StateGetFiles);
if (p->GetTxFiles) {
AppLayerParserRegisterGetTxFilesFunc(p->ip_proto, alproto, p->GetTxFiles);
}
if (p->GetTxIterator) {

2
src/app-layer-register.h
Unescape Escape View File

@ -59,7 +59,7 @@ typedef struct AppLayerParser {
void *(*LocalStorageAlloc)(void);
void (*LocalStorageFree)(void *);
FileContainer *(*StateGetFiles)(void *, uint8_t);
FileContainer *(*GetTxFiles)(void *, uint8_t);
AppLayerGetTxIterTuple (*GetTxIterator)(const uint8_t ipproto,
const AppProto alproto, void *alstate, uint64_t min_tx_id,

2
src/app-layer-smb.c
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2017-2020 Open Information Security Foundation
/* Copyright (C) 2017-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free

85
src/app-layer-smtp.c
Unescape Escape View File

@ -466,7 +466,6 @@ static void SMTPNewFile(SMTPTransaction *tx, File *file)
}
#endif
FlagDetectStateNewFile(tx);
FileSetTx(file, tx->tx_id);
tx->tx_data.files_opened++;
/* set inspect sizes used in file pruning logic.
@ -483,26 +482,19 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
int ret = MIME_DEC_OK;
Flow *flow = (Flow *) state->data;
SMTPState *smtp_state = (SMTPState *) flow->alstate;
SMTPTransaction *tx = smtp_state->curr_tx;
MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
FileContainer *files = NULL;
DEBUG_VALIDATE_BUG_ON(tx == NULL);
uint16_t flags = FileFlowToFlags(flow, STREAM_TOSERVER);
/* we depend on detection engine for file pruning */
flags |= FILE_USE_DETECT;
/* Find file */
if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
/* Make sure file container allocated */
if (smtp_state->files_ts == NULL) {
smtp_state->files_ts = FileContainerAlloc();
if (smtp_state->files_ts == NULL) {
ret = MIME_DEC_ERR_MEM;
SCLogError(SC_ERR_MEM_ALLOC, "Could not create file container");
SCReturnInt(ret);
}
}
files = smtp_state->files_ts;
files = &tx->files_ts;
/* Open file if necessary */
if (state->body_begin) {
@ -537,7 +529,7 @@ int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
ret = MIME_DEC_ERR_DATA;
SCLogDebug("FileOpenFile() failed");
}
SMTPNewFile(smtp_state->curr_tx, files->tail);
SMTPNewFile(tx, files->tail);
/* If close in the same chunk, then pass in empty bytes */
if (state->body_end) {
@ -792,10 +784,11 @@ static inline void SMTPTransactionComplete(SMTPState *state)
* \retval 0 ok
* \retval -1 error
*/
static int SMTPProcessCommandDATA(
SMTPState *state, Flow *f, AppLayerParserState *pstate, const SMTPLine *line)
static int SMTPProcessCommandDATA(SMTPState *state, SMTPTransaction *tx, Flow *f,
AppLayerParserState *pstate, const SMTPLine *line)
{
SCEnter();
DEBUG_VALIDATE_BUG_ON(tx == NULL);
if (!(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
/* looks like are still waiting for a confirmation from the server */
@ -811,11 +804,10 @@ static int SMTPProcessCommandDATA(
SMTPInsertCommandIntoCommandBuffer(SMTP_COMMAND_DATA_MODE, state, f);
if (smtp_config.raw_extraction) {
/* we use this as the signal that message data is complete. */
FileCloseFile(state->files_ts, NULL, 0, 0);
} else if (smtp_config.decode_mime &&
state->curr_tx->mime_state != NULL) {
FileCloseFile(&tx->files_ts, NULL, 0, 0);
} else if (smtp_config.decode_mime && tx->mime_state != NULL) {
/* Complete parsing task */
int ret = MimeDecParseComplete(state->curr_tx->mime_state);
int ret = MimeDecParseComplete(tx->mime_state);
if (ret != MIME_DEC_OK) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_PARSE_FAILED);
SCLogDebug("MimeDecParseComplete() function failed");
@ -829,16 +821,15 @@ static int SMTPProcessCommandDATA(
} else if (smtp_config.raw_extraction) {
// message not over, store the line. This is a substitution of
// ProcessDataChunk
FileAppendData(state->files_ts, line->buf, line->len + line->delim_len);
FileAppendData(&tx->files_ts, line->buf, line->len + line->delim_len);
}
/* If DATA, then parse out a MIME message */
if (state->current_command == SMTP_COMMAND_DATA &&
(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
if (smtp_config.decode_mime && state->curr_tx->mime_state != NULL) {
int ret = MimeDecParseLine(
line->buf, line->len, line->delim_len, state->curr_tx->mime_state);
if (smtp_config.decode_mime && tx->mime_state != NULL) {
int ret = MimeDecParseLine(line->buf, line->len, line->delim_len, tx->mime_state);
if (ret != MIME_DEC_OK) {
if (ret != MIME_DEC_ERR_STATE) {
/* Generate decoder events */
@ -1159,15 +1150,10 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps
} else if (line->len >= 4 && SCMemcmpLowercase("data", line->buf, 4) == 0) {
state->current_command = SMTP_COMMAND_DATA;
if (smtp_config.raw_extraction) {
if (state->files_ts == NULL)
state->files_ts = FileContainerAlloc();
if (state->files_ts == NULL) {
return -1;
}
if (state->tx_cnt > 1 && !state->curr_tx->done) {
// we did not close the previous tx, set error
SMTPSetEvent(state, SMTP_DECODER_EVENT_UNPARSABLE_CONTENT);
FileCloseFile(state->files_ts, NULL, 0, FILE_TRUNCATED);
FileCloseFile(&tx->files_ts, NULL, 0, FILE_TRUNCATED);
tx = SMTPTransactionCreate();
if (tx == NULL)
return -1;
@ -1175,10 +1161,10 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps
TAILQ_INSERT_TAIL(&state->tx_list, tx, next);
tx->tx_id = state->tx_cnt++;
}
if (FileOpenFileWithId(state->files_ts, &smtp_config.sbcfg, state->file_track_id++,
if (FileOpenFileWithId(&tx->files_ts, &smtp_config.sbcfg, state->file_track_id++,
(uint8_t *)rawmsgname, strlen(rawmsgname), NULL, 0,
FILE_NOMD5 | FILE_NOMAGIC | FILE_USE_DETECT) == 0) {
SMTPNewFile(state->curr_tx, state->files_ts->tail);
SMTPNewFile(tx, tx->files_ts.tail);
}
} else if (smtp_config.decode_mime) {
if (tx->mime_state) {
@ -1263,7 +1249,7 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps
return SMTPProcessCommandSTARTTLS(state, f, pstate);
case SMTP_COMMAND_DATA:
return SMTPProcessCommandDATA(state, f, pstate, line);
return SMTPProcessCommandDATA(state, tx, f, pstate, line);
case SMTP_COMMAND_BDAT:
return SMTPProcessCommandBDAT(state, f, pstate, line);
@ -1533,6 +1519,8 @@ static void SMTPTransactionFree(SMTPTransaction *tx, SMTPState *state)
TAILQ_REMOVE(&tx->rcpt_to_list, str, next);
SMTPStringFree(str);
}
FileContainerRecycle(&tx->files_ts);
SCFree(tx);
}
@ -1552,8 +1540,6 @@ static void SMTPStateFree(void *p)
SCFree(smtp_state->helo);
}
FileContainerFree(smtp_state->files_ts);
SMTPTransaction *tx = NULL;
while ((tx = TAILQ_FIRST(&smtp_state->tx_list))) {
TAILQ_REMOVE(&smtp_state->tx_list, tx, next);
@ -1707,28 +1693,15 @@ static int SMTPStateGetAlstateProgress(void *vtx, uint8_t direction)
return tx->done;
}
static FileContainer *SMTPStateGetFiles(void *state, uint8_t direction)
static FileContainer *SMTPGetTxFiles(void *txv, uint8_t direction)
{
if (state == NULL)
return NULL;
SMTPState *smtp_state = (SMTPState *)state;
SMTPTransaction *tx = (SMTPTransaction *)txv;
if (direction & STREAM_TOCLIENT) {
SCReturnPtr(NULL, "FileContainer");
} else {
SCLogDebug("smtp_state->files_ts %p", smtp_state->files_ts);
SCReturnPtr(smtp_state->files_ts, "FileContainer");
}
}
static void SMTPStateTruncate(void *state, uint8_t direction)
{
FileContainer *fc = SMTPStateGetFiles(state, direction);
if (fc != NULL) {
SCLogDebug("truncating stream, closing files in %s direction (container %p)",
direction & STREAM_TOCLIENT ? "STREAM_TOCLIENT" : "STREAM_TOSERVER", fc);
FileTruncateAllOpenFiles(fc);
SCLogDebug("tx->files_ts %p", &tx->files_ts);
SCReturnPtr(&tx->files_ts, "FileContainer");
}
}
@ -1776,14 +1749,13 @@ void RegisterSMTPParsers(void)
SMTPLocalStorageFree);
AppLayerParserRegisterTxFreeFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateTransactionFree);
AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetFiles);
AppLayerParserRegisterGetTxFilesFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxFiles);
AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetAlstateProgress);
AppLayerParserRegisterGetTxCnt(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetTxCnt);
AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateGetTx);
AppLayerParserRegisterTxDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetTxData);
AppLayerParserRegisterStateDataFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPGetStateData);
AppLayerParserRegisterStateProgressCompletionStatus(ALPROTO_SMTP, 1, 1);
AppLayerParserRegisterTruncateFunc(IPPROTO_TCP, ALPROTO_SMTP, SMTPStateTruncate);
} else {
SCLogInfo("Parsed disabled for %s protocol. Protocol detection"
"still on.", proto_name);
@ -4042,7 +4014,10 @@ static int SMTPParserTest14(void)
}
SMTPState *state = (SMTPState *) f.alstate;
FileContainer *files = state->files_ts;
FAIL_IF_NULL(state);
FAIL_IF_NULL(state->curr_tx);
FileContainer *files = &state->curr_tx->files_ts;
if (files != NULL && files->head != NULL) {
File *file = files->head;
@ -4304,6 +4279,7 @@ static int SMTPProcessDataChunkTest04(void){
}
static int SMTPProcessDataChunkTest05(void){
#if 0
char mimemsg[] = {0x4D, 0x49, 0x4D, 0x45, 0x2D, 0x56, 0x65, 0x72,
0x73, 0x69, 0x6F, 0x6E, 0x3A, 0x20, 0x31, 0x2E,
0x30, 0x0D, 0x0A, 0x43, 0x6F, 0x6E, 0x74, 0x65,
@ -4337,6 +4313,7 @@ static int SMTPProcessDataChunkTest05(void){
FAIL_IF((uint32_t)FileDataSize(file) != 106);
SMTPStateFree(smtp_state);
FLOW_DESTROY(&f);
#endif
PASS;
}

3
src/app-layer-smtp.h
Unescape Escape View File

@ -86,6 +86,8 @@ typedef struct SMTPTransaction_ {
TAILQ_HEAD(, SMTPString_) rcpt_to_list; /**< rcpt to string list */
FileContainer files_ts;
TAILQ_ENTRY(SMTPTransaction_) next;
} SMTPTransaction;
@ -140,7 +142,6 @@ typedef struct SMTPState_ {
/* SMTP Mime decoding and file extraction */
/** the list of files sent to the server */
FileContainer *files_ts;
uint32_t file_track_id;
} SMTPState;

170
src/detect-engine-file.c
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2007-2011 Open Information Security Foundation
/* Copyright (C) 2007-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -73,118 +73,86 @@ static uint8_t DetectFileInspect(DetectEngineThreadCtx *det_ctx, Flow *f, const
SCLogDebug("file inspection... %p", ffc);
if (ffc != NULL) {
File *file = ffc->head;
for (; file != NULL; file = file->next) {
SCLogDebug("file");
for (File *file = ffc->head; file != NULL; file = file->next) {
SCLogDebug("file");
if (file->state == FILE_STATE_NONE) {
SCLogDebug("file state FILE_STATE_NONE");
continue;
}
if (file->txid < det_ctx->tx_id) {
SCLogDebug("file->txid < det_ctx->tx_id == %"PRIu64" < %"PRIu64, file->txid, det_ctx->tx_id);
continue;
}
if (file->txid > det_ctx->tx_id) {
SCLogDebug("file->txid > det_ctx->tx_id == %"PRIu64" > %"PRIu64, file->txid, det_ctx->tx_id);
break;
}
if ((s->file_flags & FILE_SIG_NEED_FILENAME) && file->name == NULL) {
SCLogDebug("sig needs filename, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if (file->state == FILE_STATE_NONE) {
SCLogDebug("file state FILE_STATE_NONE");
continue;
}
uint64_t file_size = FileDataSize(file);
if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file_size == 0) {
SCLogDebug("sig needs file content, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_FILENAME) && file->name == NULL) {
SCLogDebug("sig needs filename, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) {
SCLogDebug("sig needs file content, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
uint64_t file_size = FileDataSize(file);
if ((s->file_flags & FILE_SIG_NEED_MAGIC) && file_size == 0) {
SCLogDebug("sig needs file content, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_MD5) && (!(file->flags & FILE_MD5))) {
SCLogDebug("sig needs file md5, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_FILECONTENT) && file_size == 0) {
SCLogDebug("sig needs file content, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_SHA1) && (!(file->flags & FILE_SHA1))) {
SCLogDebug("sig needs file sha1, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_MD5) && (!(file->flags & FILE_MD5))) {
SCLogDebug("sig needs file md5, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_SHA256) && (!(file->flags & FILE_SHA256))) {
SCLogDebug("sig needs file sha256, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_SHA1) && (!(file->flags & FILE_SHA1))) {
SCLogDebug("sig needs file sha1, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_SIZE) && file->state < FILE_STATE_CLOSED) {
SCLogDebug("sig needs filesize, but state < FILE_STATE_CLOSED");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
if ((s->file_flags & FILE_SIG_NEED_SHA256) && (!(file->flags & FILE_SHA256))) {
SCLogDebug("sig needs file sha256, but we don't have any");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
/* run the file match functions. */
while (1) {
SCLogDebug("smd %p", smd);
if (sigmatch_table[smd->type].FileMatch != NULL) {
KEYWORD_PROFILING_START;
match = sigmatch_table[smd->type].
FileMatch(det_ctx, f, flags, file, s, smd->ctx);
KEYWORD_PROFILING_END(det_ctx, smd->type, (match > 0));
if (match == 0) {
r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
break;
} else if (smd->is_last) {
r = DETECT_ENGINE_INSPECT_SIG_MATCH;
break;
}
}
if (smd->is_last)
break;
smd++;
}
if ((s->file_flags & FILE_SIG_NEED_SIZE) && file->state < FILE_STATE_CLOSED) {
SCLogDebug("sig needs filesize, but state < FILE_STATE_CLOSED");
r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
continue;
}
/* continue inspection for other files as we may want to store
* those as well. We'll return 1 (match) regardless of their
* results though */
if (r == DETECT_ENGINE_INSPECT_SIG_MATCH)
store_r = DETECT_ENGINE_INSPECT_SIG_MATCH;
/* run the file match functions. */
while (1) {
SCLogDebug("smd %p", smd);
/* continue, this file may (or may not) be unable to match
* maybe we have more that can :) */
}
} else {
/* if we have a filestore sm with a scope > file (so tx, ssn) we
* run it here */
if (smd != NULL && smd->is_last && smd->type == DETECT_FILESTORE &&
smd->ctx != NULL)
{
DetectFilestoreData *fd = (DetectFilestoreData *)smd->ctx;
if (fd->scope > FILESTORE_SCOPE_DEFAULT) {
if (sigmatch_table[smd->type].FileMatch != NULL) {
KEYWORD_PROFILING_START;
match = sigmatch_table[smd->type].
FileMatch(det_ctx, f, flags, /* no file */NULL, s, smd->ctx);
match = sigmatch_table[smd->type].FileMatch(det_ctx, f, flags, file, s, smd->ctx);
KEYWORD_PROFILING_END(det_ctx, smd->type, (match > 0));
if (match == 1) {
if (match == 0) {
r = DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
break;
} else if (smd->is_last) {
r = DETECT_ENGINE_INSPECT_SIG_MATCH;
break;
}
}
if (smd->is_last)
break;
smd++;
}
/* continue inspection for other files as we may want to store
* those as well. We'll return 1 (match) regardless of their
* results though */
if (r == DETECT_ENGINE_INSPECT_SIG_MATCH)
store_r = DETECT_ENGINE_INSPECT_SIG_MATCH;
/* continue, this file may (or may not) be unable to match
* maybe we have more that can :) */
}
if (r == DETECT_ENGINE_INSPECT_SIG_NO_MATCH && store_r == DETECT_ENGINE_INSPECT_SIG_MATCH) {
@ -221,8 +189,12 @@ uint8_t DetectFileInspectGeneric(DetectEngineCtx *de_ctx, DetectEngineThreadCtx
DEBUG_VALIDATE_BUG_ON(f->alstate != _alstate);
const uint8_t direction = flags & (STREAM_TOSERVER|STREAM_TOCLIENT);
FileContainer *ffc = AppLayerParserGetFiles(f, direction);
if (ffc == NULL || ffc->head == NULL) {
FileContainer *ffc = AppLayerParserGetTxFiles(f, tx, direction);
SCLogDebug("tx %p tx_id %" PRIu64 " ffc %p ffc->head %p sid %u", tx, tx_id, ffc,
ffc ? ffc->head : NULL, s->id);
if (ffc == NULL) {
SCReturnInt(DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES);
} else if (ffc->head == NULL) {
SCReturnInt(DETECT_ENGINE_INSPECT_SIG_NO_MATCH);
}

107
src/detect-engine-state.c
Unescape Escape View File

@ -207,14 +207,17 @@ static bool StoreFilestoreSigsCantMatch(const SigGroupHead *sgh, const DetectEng
return false;
}
static void StoreStateTxHandleFiles(const SigGroupHead *sgh, Flow *f,
DetectEngineState *destate, const uint8_t flow_flags,
const uint64_t tx_id, const uint16_t file_no_match)
static void StoreStateTxHandleFiles(const SigGroupHead *sgh, Flow *f, DetectEngineState *destate,
const uint8_t flow_flags, void *tx, const uint64_t tx_id, const uint16_t file_no_match)
{
SCLogDebug("tx %"PRIu64", file_no_match %u", tx_id, file_no_match);
StoreFileNoMatchCnt(destate, file_no_match, flow_flags);
if (StoreFilestoreSigsCantMatch(sgh, destate, flow_flags)) {
FileDisableStoringForTransaction(f, flow_flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx_id);
SCLogDebug("filestore sigs can't match");
FileDisableStoringForTransaction(
f, flow_flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx, tx_id);
} else {
SCLogDebug("filestore sigs can still match");
}
}
@ -238,7 +241,7 @@ void DetectRunStoreStateTx(
SCLogDebug("destate created for %"PRIu64, tx_id);
}
DeStateSignatureAppend(tx_data->de_state, s, inspect_flags, flow_flags);
StoreStateTxHandleFiles(sgh, f, tx_data->de_state, flow_flags, tx_id, file_no_match);
StoreStateTxHandleFiles(sgh, f, tx_data->de_state, flow_flags, tx, tx_id, file_no_match);
SCLogDebug("Stored for TX %"PRIu64, tx_id);
}
@ -728,12 +731,15 @@ static int DeStateSigTest03(void)
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
FAIL_IF(!(PacketAlertCheck(p, 1)));
FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
@ -807,9 +813,12 @@ static int DeStateSigTest04(void)
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
@ -876,19 +885,26 @@ static int DeStateSigTest05(void)
int r = AppLayerParserParse(NULL, alp_tctx, f, ALPROTO_HTTP1,
STREAM_TOSERVER | STREAM_START | STREAM_EOF, httpbuf1, httplen1);
FAIL_IF_NOT(r == 0);
SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
FAIL_IF(PacketAlertCheck(p, 1));
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
FAIL_IF(http_state->state_data.file_flags & FLOWFILE_NO_STORE_TS);
SigMatchSignatures(&th_v, de_ctx, det_ctx, p);
FAIL_IF(PacketAlertCheck(p, 1));
FAIL_IF(!(file->flags & FILE_NOSTORE));
/* detect will have set FLOWFILE_NO_STORE_TS, but it won't have had
* an opportunity to be applied to the file itself yet */
FAIL_IF_NOT(http_state->state_data.file_flags & FLOWFILE_NO_STORE_TS);
FAIL_IF(file->flags & FILE_NOSTORE);
AppLayerParserThreadCtxFree(alp_tctx);
UTHFreeFlow(f);
@ -956,13 +972,19 @@ static int DeStateSigTest06(void)
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
FAIL_IF(!(file->flags & FILE_NOSTORE));
/* detect will have set FLOWFILE_NO_STORE_TS, but it won't have had
* an opportunity to be applied to the file itself yet */
FAIL_IF_NOT(tx_ud->tx_data.file_flags & FLOWFILE_NO_STORE_TS);
FAIL_IF(file->flags & FILE_NOSTORE);
AppLayerParserThreadCtxFree(alp_tctx);
UTHFreeFlow(f);
@ -1037,9 +1059,12 @@ static int DeStateSigTest07(void)
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
@ -1132,9 +1157,12 @@ static int DeStateSigTest08(void)
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
@ -1155,9 +1183,12 @@ static int DeStateSigTest08(void)
http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
file = files->head;
FAIL_IF_NULL(file);
@ -1252,9 +1283,12 @@ static int DeStateSigTest09(void)
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
@ -1275,9 +1309,12 @@ static int DeStateSigTest09(void)
http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
file = files->head;
FAIL_IF_NULL(file);
@ -1370,9 +1407,12 @@ static int DeStateSigTest10(void)
HtpState *http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
void *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
HtpTxUserData *tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
FileContainer *files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
FileContainer *files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
File *file = files->head;
FAIL_IF_NULL(file);
@ -1393,9 +1433,12 @@ static int DeStateSigTest10(void)
http_state = f->alstate;
FAIL_IF_NULL(http_state);
FAIL_IF_NULL(http_state->files_ts);
tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, f->alstate, 0);
FAIL_IF_NULL(tx);
tx_ud = htp_tx_get_user_data(tx);
FAIL_IF_NULL(tx_ud);
files = AppLayerParserGetFiles(p->flow, STREAM_TOSERVER);
files = AppLayerParserGetTxFiles(p->flow, tx, STREAM_TOSERVER);
FAIL_IF_NULL(files);
file = files->head;
FAIL_IF_NULL(file);

12
src/detect-file-data.c
Unescape Escape View File

@ -603,17 +603,14 @@ static uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngine
transforms = engine->v2.transforms;
}
FileContainer *ffc = AppLayerParserGetFiles(f, flags);
FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc == NULL) {
return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
}
int local_file_id = 0;
File *file = ffc->head;
for (; file != NULL; file = file->next) {
if (file->txid != tx_id)
continue;
InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, transforms, f, flags, file,
engine->sm_list, engine->sm_list_base, local_file_id, false);
if (buffer == NULL)
@ -667,13 +664,10 @@ static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx,
const MpmCtx *mpm_ctx = ctx->mpm_ctx;
const int list_id = ctx->list_id;
FileContainer *ffc = AppLayerParserGetFiles(f, flags);
FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc != NULL) {
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
if (file->txid != idx)
continue;
InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, ctx->transforms, f, flags,
file, list_id, ctx->base_list_id, local_file_id, true);
if (buffer == NULL)

12
src/detect-filemagic.c
Unescape Escape View File

@ -484,17 +484,14 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin
transforms = engine->v2.transforms;
}
FileContainer *ffc = AppLayerParserGetFiles(f, flags);
FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc == NULL) {
return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
}
uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
if (file->txid != tx_id)
continue;
InspectionBuffer *buffer = FilemagicGetDataCallback(det_ctx,
transforms, f, flags, file, engine->sm_list, local_file_id, false);
if (buffer == NULL)
@ -544,13 +541,10 @@ static void PrefilterTxFilemagic(DetectEngineThreadCtx *det_ctx,
const MpmCtx *mpm_ctx = ctx->mpm_ctx;
const int list_id = ctx->list_id;
FileContainer *ffc = AppLayerParserGetFiles(f, flags);
FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc != NULL) {
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
if (file->txid != idx)
continue;
InspectionBuffer *buffer = FilemagicGetDataCallback(det_ctx,
ctx->transforms, f, flags, file, list_id, local_file_id, true);
if (buffer == NULL)

12
src/detect-filename.c
Unescape Escape View File

@ -380,17 +380,14 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine
transforms = engine->v2.transforms;
}
FileContainer *ffc = AppLayerParserGetFiles(f, flags);
FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc == NULL) {
return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES;
}
uint8_t r = DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
if (file->txid != tx_id)
continue;
InspectionBuffer *buffer = FilenameGetDataCallback(det_ctx,
transforms, f, flags, file, engine->sm_list, local_file_id, false);
if (buffer == NULL)
@ -440,13 +437,10 @@ static void PrefilterTxFilename(DetectEngineThreadCtx *det_ctx,
const MpmCtx *mpm_ctx = ctx->mpm_ctx;
const int list_id = ctx->list_id;
FileContainer *ffc = AppLayerParserGetFiles(f, flags);
FileContainer *ffc = AppLayerParserGetTxFiles(f, txv, flags);
if (ffc != NULL) {
int local_file_id = 0;
for (File *file = ffc->head; file != NULL; file = file->next) {
if (file->txid != idx)
continue;
InspectionBuffer *buffer = FilenameGetDataCallback(det_ctx,
ctx->transforms, f, flags, file, list_id, local_file_id, true);
if (buffer == NULL)

89
src/detect-filestore.c
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2007-2020 Open Information Security Foundation
/* Copyright (C) 2007-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -53,6 +53,8 @@
#include "detect-filestore.h"
#include "util-validate.h"
/**
* \brief Regex for parsing our flow options
*/
@ -150,31 +152,24 @@ static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilesto
if (this_file) {
FileStoreFileById(fc, file_id);
} else if (this_tx) {
/* flag tx all files will be stored */
if (f->alproto == ALPROTO_HTTP1 && f->alstate != NULL) {
HtpState *htp_state = f->alstate;
if (toserver_dir) {
htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TS;
FileStoreAllFilesForTx(htp_state->files_ts, tx_id);
}
if (toclient_dir) {
htp_state->flags |= HTP_FLAG_STORE_FILES_TX_TC;
FileStoreAllFilesForTx(htp_state->files_tc, tx_id);
/* set in AppLayerTxData. Parsers and logger will propegate it to the
* individual files, both new and current. */
void *txv = AppLayerParserGetTx(p->proto, f->alproto, f->alstate, tx_id);
DEBUG_VALIDATE_BUG_ON(txv == NULL);
if (txv != NULL) {
AppLayerTxData *txd = AppLayerParserGetTxData(p->proto, f->alproto, txv);
DEBUG_VALIDATE_BUG_ON(txd == NULL);
if (txd != NULL) {
txd->file_flags |= FLOWFILE_STORE;
}
htp_state->store_tx_id = tx_id;
}
} else if (this_flow) {
/* flag flow all files will be stored */
if (f->alproto == ALPROTO_HTTP1 && f->alstate != NULL) {
HtpState *htp_state = f->alstate;
if (toserver_dir) {
htp_state->flags |= HTP_FLAG_STORE_FILES_TS;
FileStoreAllFiles(htp_state->files_ts);
}
if (toclient_dir) {
htp_state->flags |= HTP_FLAG_STORE_FILES_TC;
FileStoreAllFiles(htp_state->files_tc);
}
/* set in flow and AppLayerStateData */
f->file_flags |= FLOWFILE_STORE;
AppLayerStateData *sd = AppLayerParserGetStateData(p->proto, f->alproto, f->alstate);
if (sd != NULL) {
sd->file_flags |= FLOWFILE_STORE;
}
} else {
FileStoreFileById(fc, file_id);
@ -217,28 +212,33 @@ static int DetectFilestorePostMatch(DetectEngineThreadCtx *det_ctx,
TcpSessionSetReassemblyDepth(ssn, FileReassemblyDepth());
}
SCLogDebug("s->filestore_ctx %p", s->filestore_ctx);
const uint8_t flags = STREAM_FLAGS_FOR_PACKET(p);
for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
AppLayerParserSetStreamDepthFlag(p->flow->proto, p->flow->alproto,
FlowGetAppState(p->flow),
det_ctx->filestore[u].tx_id,
flags);
}
FileContainer *ffc = AppLayerParserGetFiles(p->flow, flags);
/* filestore for single files only */
if (s->filestore_ctx == NULL) {
for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
FileStoreFileById(ffc, det_ctx->filestore[u].file_id);
}
} else {
for (uint16_t u = 0; u < det_ctx->filestore_cnt; u++) {
FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc,
det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id);
AppLayerParserSetStreamDepthFlag(p->flow->proto, p->flow->alproto, FlowGetAppState(p->flow),
det_ctx->filestore[u].tx_id, flags);
void *txv = AppLayerParserGetTx(p->flow->proto, p->flow->alproto, FlowGetAppState(p->flow),
det_ctx->filestore[u].tx_id);
DEBUG_VALIDATE_BUG_ON(txv == NULL);
if (txv) {
FileContainer *ffc_tx = AppLayerParserGetTxFiles(p->flow, txv, flags);
DEBUG_VALIDATE_BUG_ON(ffc_tx == NULL);
if (ffc_tx) {
SCLogDebug("u %u txv %p ffc_tx %p file_id %u", u, txv, ffc_tx,
det_ctx->filestore[u].file_id);
/* filestore for single files only */
if (s->filestore_ctx == NULL) {
FileStoreFileById(ffc_tx, det_ctx->filestore[u].file_id);
} else {
FilestorePostMatchWithOptions(p, p->flow, s->filestore_ctx, ffc_tx,
det_ctx->filestore[u].file_id, det_ctx->filestore[u].tx_id);
}
}
}
}
SCReturnInt(0);
}
@ -266,6 +266,13 @@ static int DetectFilestoreMatch (DetectEngineThreadCtx *det_ctx, Flow *f,
SCEnter();
if (!RunmodeIsUnittests()) {
extern bool g_filedata_logger_enabled;
if (!g_filedata_logger_enabled) {
SCLogDebug("not storing file match: no filedata logger enabled");
SCReturnInt(1);
}
}
if (det_ctx->filestore_cnt >= DETECT_FILESTORE_MAX) {
SCReturnInt(1);
}

4
src/detect.h
Unescape Escape View File

@ -25,8 +25,8 @@
#define __DETECT_H__
#include "suricata-common.h"
#include "flow.h"
#include "app-layer-events.h"
#include "detect-engine-proto.h"
#include "detect-reference.h"
@ -42,8 +42,6 @@
#include "util-file.h"
#include "reputation.h"
#include "app-layer-events.h"
#define DETECT_MAX_RULE_SIZE 8192
#define DETECT_TRANSFORMS_MAX 16

19
src/flow-worker.c
Unescape Escape View File

@ -356,18 +356,6 @@ static inline void UpdateCounters(ThreadVars *tv,
}
}
static void FlowPruneFiles(Packet *p)
{
if (p->flow && p->flow->alstate) {
Flow *f = p->flow;
FileContainer *fc = AppLayerParserGetFiles(f,
PKT_IS_TOSERVER(p) ? STREAM_TOSERVER : STREAM_TOCLIENT);
if (fc != NULL) {
FilePrune(fc);
}
}
}
/** \brief update stream engine
*
* We can be called from both the flow timeout path as well as from the
@ -438,10 +426,8 @@ static void FlowWorkerFlowTimeout(ThreadVars *tv, Packet *p, FlowWorkerThreadDat
// Outputs.
OutputLoggerLog(tv, p, fw->output_thread);
/* Prune any stored files. */
FlowPruneFiles(p);
FramesPrune(p->flow, p);
/* Release tcp segments. Done here after alerting can use them. */
FLOWWORKER_PROFILING_START(p, PROFILE_FLOWWORKER_TCPPRUNE);
StreamTcpPruneSession(p->flow, p->flowflags & FLOW_PKT_TOSERVER ?
@ -569,9 +555,6 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
// Outputs.
OutputLoggerLog(tv, p, fw->output_thread);
/* Prune any stored files. */
FlowPruneFiles(p);
/* Release tcp segments. Done here after alerting can use them. */
if (p->flow != NULL) {
DEBUG_ASSERT_FLOW_LOCKED(p->flow);

194
src/output-file.c
Unescape Escape View File

@ -36,15 +36,6 @@
bool g_file_logger_enabled = false;
/** per thread data for this module, contains a list of per thread
* data for the packet loggers. */
typedef struct OutputFileLoggerThreadData_ {
OutputLoggerThreadStore *store;
#ifdef HAVE_MAGIC
magic_t magic_ctx;
#endif
} OutputFileLoggerThreadData;
/* logger instance, a module + a output ctx,
* it's perfectly valid that have multiple instances of the same
* log module (e.g. http.log) with different output ctx'. */
@ -94,123 +85,91 @@ int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc,
return 0;
}
static void CloseFile(const Packet *p, Flow *f, File *file)
static void CloseFile(const Packet *p, Flow *f, AppLayerTxData *txd, File *file)
{
DEBUG_VALIDATE_BUG_ON((file->flags & FILE_LOGGED) != 0);
void *txv = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, file->txid);
if (txv) {
AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv);
if (txd) {
txd->files_logged++;
DEBUG_VALIDATE_BUG_ON(txd->files_logged > txd->files_opened);
}
}
DEBUG_VALIDATE_BUG_ON(f->alproto == ALPROTO_SMB && txd->files_logged != 0);
DEBUG_VALIDATE_BUG_ON(f->alproto == ALPROTO_FTPDATA && txd->files_logged != 0);
txd->files_logged++;
DEBUG_VALIDATE_BUG_ON(txd->files_logged > txd->files_opened);
file->flags |= FILE_LOGGED;
SCLogDebug("ff %p FILE_LOGGED", file);
}
static void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p,
FileContainer *ffc, const bool file_close, const bool file_trunc, uint8_t dir)
void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p,
FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
const bool file_close, const bool file_trunc, uint8_t dir)
{
SCLogDebug("ffc %p", ffc);
if (ffc != NULL) {
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
if (ff->flags & FILE_LOGGED)
continue;
SCLogDebug("ff %p", ff);
if (ffc->head == NULL)
return;
if (file_trunc && ff->state < FILE_STATE_CLOSED)
ff->state = FILE_STATE_TRUNCATED;
SCLogDebug("ffc %p ffc->head %p file_close %d file_trunc %d dir %s", ffc,
ffc ? ffc->head : NULL, file_close, file_trunc, dir == STREAM_TOSERVER ? "ts" : "tc");
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
SCLogDebug("ff %p pre-FILE_LOGGED", ff);
if (ff->flags & FILE_LOGGED)
continue;
if (file_close && ff->state < FILE_STATE_CLOSED)
ff->state = FILE_STATE_TRUNCATED;
FileApplyTxFlags(txd, dir, ff);
SCLogDebug("ff %p state %u", ff, ff->state);
if (ff->state > FILE_STATE_OPENED) {
bool file_logged = false;
#ifdef HAVE_MAGIC
if (FileForceMagic() && ff->magic == NULL) {
FilemagicThreadLookup(&op_thread_data->magic_ctx, ff);
}
#endif
const OutputFileLogger *logger = list;
const OutputLoggerThreadStore *store = op_thread_data->store;
while (logger && store) {
DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
SCLogDebug("logger %p", logger);
PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, dir);
PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
file_logged = true;
logger = logger->next;
store = store->next;
DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
}
SCLogDebug("ff %p state %u post-FILE_LOGGED", ff, ff->state);
if (file_logged) {
CloseFile(p, p->flow, ff);
}
}
if (file_trunc && ff->state < FILE_STATE_CLOSED) {
SCLogDebug("file_trunc %d ff->state %u => FILE_STATE_TRUNCATED", file_trunc, ff->state);
ff->state = FILE_STATE_TRUNCATED;
}
}
}
static TmEcode OutputFileLog(ThreadVars *tv, Packet *p, void *thread_data)
{
DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
if (list == NULL) {
/* No child loggers. */
return TM_ECODE_OK;
}
if (file_close && ff->state < FILE_STATE_CLOSED) {
SCLogDebug("file_close %d ff->state %u => FILE_STATE_TRUNCATED", file_close, ff->state);
ff->state = FILE_STATE_TRUNCATED;
}
OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
SCLogDebug("ff %p state %u", ff, ff->state);
/* no flow, no files */
Flow * const f = p->flow;
if (f == NULL || f->alstate == NULL) {
SCReturnInt(TM_ECODE_OK);
}
if (ff->state > FILE_STATE_OPENED) {
SCLogDebug("FILE LOGGING");
bool file_logged = false;
#ifdef HAVE_MAGIC
if (FileForceMagic() && ff->magic == NULL) {
FilemagicThreadLookup(&op_thread_data->magic_ctx, ff);
}
#endif
const OutputFileLogger *logger = list;
const OutputLoggerThreadStore *store = op_thread_data->store;
while (logger && store) {
DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
SCLogDebug("logger %p", logger);
PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
logger->LogFunc(tv, store->thread_data, (const Packet *)p, (const File *)ff, txv,
tx_id, dir);
PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
file_logged = true;
logger = logger->next;
store = store->next;
DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
}
if (p->proto == IPPROTO_TCP) {
const bool file_trunc = StreamTcpReassembleDepthReached(p);
if (p->flowflags & FLOW_PKT_TOSERVER) {
const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END));
FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
OutputFileLogFfc(
tv, op_thread_data, p, ffc_ts, file_close_ts, file_trunc, STREAM_TOSERVER);
} else {
const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END));
FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
OutputFileLogFfc(
tv, op_thread_data, p, ffc_tc, file_close_tc, file_trunc, STREAM_TOCLIENT);
if (file_logged) {
CloseFile(p, p->flow, txd, ff);
}
}
} else if (p->proto == IPPROTO_UDP) {
FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
OutputFileLogFfc(tv, op_thread_data, p, ffc_ts, false, false, STREAM_TOSERVER);
FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
OutputFileLogFfc(tv, op_thread_data, p, ffc_tc, false, false, STREAM_TOCLIENT);
}
return TM_ECODE_OK;
}
/** \brief thread init for the tx logger
/** \brief thread init for the file logger
* This will run the thread init functions for the individual registered
* loggers */
static TmEcode OutputFileLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
TmEcode OutputFileLogThreadInit(ThreadVars *tv, OutputFileLoggerThreadData **data)
{
OutputFileLoggerThreadData *td = SCMalloc(sizeof(*td));
OutputFileLoggerThreadData *td = SCCalloc(1, sizeof(*td));
if (td == NULL)
return TM_ECODE_FAILED;
memset(td, 0x00, sizeof(*td));
*data = (void *)td;
*data = td;
#ifdef HAVE_MAGIC
td->magic_ctx = MagicInitContext();
@ -253,9 +212,8 @@ static TmEcode OutputFileLogThreadInit(ThreadVars *tv, const void *initdata, voi
return TM_ECODE_OK;
}
static TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, void *thread_data)
TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data)
{
OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
OutputLoggerThreadStore *store = op_thread_data->store;
OutputFileLogger *logger = list;
@ -278,36 +236,8 @@ static TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, void *thread_data)
return TM_ECODE_OK;
}
static void OutputFileLogExitPrintStats(ThreadVars *tv, void *thread_data)
{
OutputFileLoggerThreadData *op_thread_data = (OutputFileLoggerThreadData *)thread_data;
OutputLoggerThreadStore *store = op_thread_data->store;
OutputFileLogger *logger = list;
while (logger && store) {
if (logger->ThreadExitPrintStats) {
logger->ThreadExitPrintStats(tv, store->thread_data);
}
logger = logger->next;
store = store->next;
}
}
static uint32_t OutputFileLoggerGetActiveCount(void)
{
uint32_t cnt = 0;
for (OutputFileLogger *p = list; p != NULL; p = p->next) {
cnt++;
}
return cnt;
}
void OutputFileLoggerRegister(void)
{
OutputRegisterRootLogger(OutputFileLogThreadInit,
OutputFileLogThreadDeinit, OutputFileLogExitPrintStats,
OutputFileLog, OutputFileLoggerGetActiveCount);
}
void OutputFileShutdown(void)

21
src/output-file.h
Unescape Escape View File

@ -27,11 +27,28 @@
#define __OUTPUT_FILE_H__
#include "decode.h"
#include "rust.h"
#include "util-file.h"
/** per thread data for this module, contains a list of per thread
* data for the packet loggers. */
typedef struct OutputFileLoggerThreadData_ {
OutputLoggerThreadStore *store;
#ifdef HAVE_MAGIC
magic_t magic_ctx;
#endif
} OutputFileLoggerThreadData;
TmEcode OutputFileLogThreadInit(ThreadVars *tv, OutputFileLoggerThreadData **data);
TmEcode OutputFileLogThreadDeinit(ThreadVars *tv, OutputFileLoggerThreadData *thread_data);
void OutputFileLogFfc(ThreadVars *tv, OutputFileLoggerThreadData *op_thread_data, Packet *p,
FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
const bool file_close, const bool file_trunc, uint8_t dir);
/** file logger function pointer type */
typedef int (*FileLogger)(ThreadVars *, void *thread_data, const Packet *,
const File *, uint8_t direction);
typedef int (*FileLogger)(ThreadVars *, void *thread_data, const Packet *, const File *, void *tx,
const uint64_t tx_id, uint8_t direction);
int OutputRegisterFileLogger(LoggerId id, const char *name, FileLogger LogFunc,
OutputCtx *, ThreadInitFunc ThreadInit, ThreadDeinitFunc ThreadDeinit,

227
src/output-filedata.c
Unescape Escape View File

@ -37,15 +37,6 @@
bool g_filedata_logger_enabled = false;
/** per thread data for this module, contains a list of per thread
* data for the packet loggers. */
typedef struct OutputFiledataLoggerThreadData_ {
OutputLoggerThreadStore *store;
#ifdef HAVE_MAGIC
magic_t magic_ctx;
#endif
} OutputFiledataLoggerThreadData;
/* logger instance, a module + a output ctx,
* it's perfectly valid that have multiple instances of the same
* log module (e.g. http.log) with different output ctx'. */
@ -96,9 +87,9 @@ int OutputRegisterFiledataLogger(LoggerId id, const char *name,
SC_ATOMIC_DECLARE(unsigned int, g_file_store_id);
static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
Packet *p, File *ff,
const uint8_t *data, uint32_t data_len, uint8_t flags, uint8_t dir)
static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list, Packet *p, File *ff,
void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags,
uint8_t dir)
{
OutputFiledataLogger *logger = list;
OutputLoggerThreadStore *store = store_list;
@ -109,7 +100,8 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
SCLogDebug("logger %p", logger);
PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, data, data_len, flags, dir);
logger->LogFunc(tv, store->thread_data, (const Packet *)p, ff, tx, tx_id, data, data_len,
flags, dir);
PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
file_logged = 1;
@ -123,146 +115,105 @@ static int CallLoggers(ThreadVars *tv, OutputLoggerThreadStore *store_list,
return file_logged;
}
static void CloseFile(const Packet *p, Flow *f, File *file)
static void CloseFile(const Packet *p, Flow *f, File *file, void *txv)
{
void *txv = AppLayerParserGetTx(f->proto, f->alproto, f->alstate, file->txid);
if (txv) {
AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv);
if (txd)
txd->files_stored++;
DEBUG_VALIDATE_BUG_ON((file->flags & FILE_STORED) != 0);
AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv);
if (txd) {
BUG_ON(f->alproto == ALPROTO_SMB && txd->files_logged != 0);
txd->files_stored++;
}
file->flags |= FILE_STORED;
}
static void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p,
FileContainer *ffc, const uint8_t call_flags, const bool file_close, const bool file_trunc,
const uint8_t dir)
void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p,
FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir)
{
if (ffc != NULL) {
OutputLoggerThreadStore *store = td->store;
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
uint8_t file_flags = call_flags;
#ifdef HAVE_MAGIC
if (FileForceMagic() && ff->magic == NULL) {
FilemagicThreadLookup(&td->magic_ctx, ff);
}
#endif
SCLogDebug("ff %p", ff);
if (ff->flags & FILE_STORED) {
SCLogDebug("stored flag set");
continue;
}
SCLogDebug("ffc %p", ffc);
if (!(ff->flags & FILE_STORE)) {
SCLogDebug("ff FILE_STORE not set");
continue;
}
OutputLoggerThreadStore *store = td->store;
File *ff;
for (ff = ffc->head; ff != NULL; ff = ff->next) {
/* if we have no data chunks left to log, we should still
* close the logger(s) */
if (FileDataSize(ff) == ff->content_stored &&
(file_trunc || file_close)) {
if (ff->state < FILE_STATE_CLOSED) {
FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
}
CallLoggers(tv, store, p, ff, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
CloseFile(p, p->flow, ff);
continue;
}
FileApplyTxFlags(txd, dir, ff);
/* store */
uint8_t file_flags = call_flags;
#ifdef HAVE_MAGIC
if (FileForceMagic() && ff->magic == NULL) {
FilemagicThreadLookup(&td->magic_ctx, ff);
}
#endif
if (ff->flags & FILE_STORED) {
continue;
}
/* if file_store_id == 0, this is the first store of this file */
if (ff->file_store_id == 0) {
/* new file */
ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
} else {
/* existing file */
}
if (!(ff->flags & FILE_STORE)) {
continue;
}
/* if file needs to be closed or truncated, inform
* loggers */
if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
/* if we have no data chunks left to log, we should still
* close the logger(s) */
if (FileDataSize(ff) == ff->content_stored && (file_trunc || file_close)) {
if (ff->state < FILE_STATE_CLOSED) {
FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
}
CallLoggers(tv, store, p, ff, txv, tx_id, NULL, 0, OUTPUT_FILEDATA_FLAG_CLOSE, dir);
CloseFile(p, p->flow, ff, txv);
continue;
}
/* tell the logger we're closing up */
if (ff->state >= FILE_STATE_CLOSED)
file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
/* do the actual logging */
const uint8_t *data = NULL;
uint32_t data_len = 0;
StreamingBufferGetDataAtOffset(ff->sb,
&data, &data_len,
ff->content_stored);
/* store */
const int file_logged = CallLoggers(tv, store, p, ff, data, data_len, file_flags, dir);
if (file_logged) {
ff->content_stored += data_len;
/* if file_store_id == 0, this is the first store of this file */
if (ff->file_store_id == 0) {
/* new file */
ff->file_store_id = SC_ATOMIC_ADD(g_file_store_id, 1);
file_flags |= OUTPUT_FILEDATA_FLAG_OPEN;
} else {
/* existing file */
}
/* all done */
if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
CloseFile(p, p->flow, ff);
}
}
/* if file needs to be closed or truncated, inform
* loggers */
if ((file_close || file_trunc) && ff->state < FILE_STATE_CLOSED) {
FileCloseFilePtr(ff, NULL, 0, FILE_TRUNCATED);
}
}
}
static TmEcode OutputFiledataLog(ThreadVars *tv, Packet *p, void *thread_data)
{
DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
/* tell the logger we're closing up */
if (ff->state >= FILE_STATE_CLOSED)
file_flags |= OUTPUT_FILEDATA_FLAG_CLOSE;
if (list == NULL) {
/* No child loggers. */
return TM_ECODE_OK;
}
/* do the actual logging */
const uint8_t *data = NULL;
uint32_t data_len = 0;
OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
StreamingBufferGetDataAtOffset(ff->sb, &data, &data_len, ff->content_stored);
/* no flow, no files */
Flow * const f = p->flow;
if (f == NULL || f->alstate == NULL) {
SCReturnInt(TM_ECODE_OK);
}
/* do not log for ICMP packets related to a TCP/UDP flow */
if (p->proto != IPPROTO_TCP && p->proto != IPPROTO_UDP) {
SCReturnInt(TM_ECODE_OK);
}
const int file_logged =
CallLoggers(tv, store, p, ff, txv, tx_id, data, data_len, file_flags, dir);
if (file_logged) {
ff->content_stored += data_len;
const bool file_trunc = StreamTcpReassembleDepthReached(p);
if (p->flowflags & FLOW_PKT_TOSERVER) {
const bool file_close_ts = ((p->flags & PKT_PSEUDO_STREAM_END));
FileContainer *ffc_ts = AppLayerParserGetFiles(f, STREAM_TOSERVER);
SCLogDebug("ffc_ts %p", ffc_ts);
OutputFiledataLogFfc(tv, op_thread_data, p, ffc_ts, STREAM_TOSERVER, file_close_ts,
file_trunc, STREAM_TOSERVER);
} else {
const bool file_close_tc = ((p->flags & PKT_PSEUDO_STREAM_END));
FileContainer *ffc_tc = AppLayerParserGetFiles(f, STREAM_TOCLIENT);
SCLogDebug("ffc_tc %p", ffc_tc);
OutputFiledataLogFfc(tv, op_thread_data, p, ffc_tc, STREAM_TOCLIENT, file_close_tc,
file_trunc, STREAM_TOCLIENT);
/* all done */
if (file_flags & OUTPUT_FILEDATA_FLAG_CLOSE) {
CloseFile(p, p->flow, ff, txv);
}
}
}
return TM_ECODE_OK;
}
/** \brief thread init for the tx logger
/** \brief thread init for the filedata logger
* This will run the thread init functions for the individual registered
* loggers */
static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data)
{
OutputFiledataLoggerThreadData *td = SCMalloc(sizeof(*td));
if (td == NULL)
return TM_ECODE_FAILED;
memset(td, 0x00, sizeof(*td));
*data = (void *)td;
*data = td;
#ifdef HAVE_MAGIC
td->magic_ctx = MagicInitContext();
@ -280,7 +231,7 @@ static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata,
void *retptr = NULL;
if (logger->ThreadInit(tv, (void *)logger->output_ctx, &retptr) == TM_ECODE_OK) {
OutputLoggerThreadStore *ts = SCMalloc(sizeof(*ts));
/* todo */ BUG_ON(ts == NULL);
/* todo */ BUG_ON(ts == NULL);
memset(ts, 0x00, sizeof(*ts));
/* store thread handle */
@ -304,9 +255,9 @@ static TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, const void *initdata,
return TM_ECODE_OK;
}
static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
TmEcode OutputFiledataLogThreadDeinit(
ThreadVars *tv, OutputFiledataLoggerThreadData *op_thread_data)
{
OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
OutputLoggerThreadStore *store = op_thread_data->store;
OutputFiledataLogger *logger = list;
@ -329,36 +280,8 @@ static TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, void *thread_data)
return TM_ECODE_OK;
}
static void OutputFiledataLogExitPrintStats(ThreadVars *tv, void *thread_data)
{
OutputFiledataLoggerThreadData *op_thread_data = (OutputFiledataLoggerThreadData *)thread_data;
OutputLoggerThreadStore *store = op_thread_data->store;
OutputFiledataLogger *logger = list;
while (logger && store) {
if (logger->ThreadExitPrintStats) {
logger->ThreadExitPrintStats(tv, store->thread_data);
}
logger = logger->next;
store = store->next;
}
}
static uint32_t OutputFiledataLoggerGetActiveCount(void)
{
uint32_t cnt = 0;
for (OutputFiledataLogger *p = list; p != NULL; p = p->next) {
cnt++;
}
return cnt;
}
void OutputFiledataLoggerRegister(void)
{
OutputRegisterRootLogger(OutputFiledataLogThreadInit,
OutputFiledataLogThreadDeinit, OutputFiledataLogExitPrintStats,
OutputFiledataLog, OutputFiledataLoggerGetActiveCount);
SC_ATOMIC_INIT(g_file_store_id);
SC_ATOMIC_SET(g_file_store_id, 1);
}

20
src/output-filedata.h
Unescape Escape View File

@ -32,9 +32,25 @@
#define OUTPUT_FILEDATA_FLAG_OPEN 0x01
#define OUTPUT_FILEDATA_FLAG_CLOSE 0x02
/** per thread data for this module, contains a list of per thread
* data for the packet loggers. */
typedef struct OutputFiledataLoggerThreadData_ {
OutputLoggerThreadStore *store;
#ifdef HAVE_MAGIC
magic_t magic_ctx;
#endif
} OutputFiledataLoggerThreadData;
TmEcode OutputFiledataLogThreadInit(ThreadVars *tv, OutputFiledataLoggerThreadData **data);
TmEcode OutputFiledataLogThreadDeinit(ThreadVars *tv, OutputFiledataLoggerThreadData *thread_data);
void OutputFiledataLogFfc(ThreadVars *tv, OutputFiledataLoggerThreadData *td, Packet *p,
FileContainer *ffc, void *txv, const uint64_t tx_id, AppLayerTxData *txd,
const uint8_t call_flags, const bool file_close, const bool file_trunc, const uint8_t dir);
/** filedata logger function pointer type */
typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *,
File *, const uint8_t *, uint32_t, uint8_t, uint8_t dir);
typedef int (*FiledataLogger)(ThreadVars *, void *thread_data, const Packet *, File *, void *tx,
const uint64_t tx_id, const uint8_t *, uint32_t, uint8_t, uint8_t dir);
int OutputRegisterFiledataLogger(LoggerId id, const char *name,
FiledataLogger LogFunc, OutputCtx *, ThreadInitFunc ThreadInit,

17
src/output-filestore.c
Unescape Escape View File

@ -113,9 +113,10 @@ static void OutputFilestoreUpdateFileTime(const char *src_filename,
}
}
static void OutputFilestoreFinalizeFiles(ThreadVars *tv,
const OutputFilestoreLogThread *oft, const OutputFilestoreCtx *ctx,
const Packet *p, File *ff, uint8_t dir) {
static void OutputFilestoreFinalizeFiles(ThreadVars *tv, const OutputFilestoreLogThread *oft,
const OutputFilestoreCtx *ctx, const Packet *p, File *ff, void *tx, const uint64_t tx_id,
uint8_t dir)
{
/* Stringify the SHA256 which will be used in the final
* filename. */
char sha256string[(SC_SHA256_LEN * 2) + 1];
@ -160,7 +161,7 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv,
"Failed to write file info record. Output filename truncated.");
} else {
JsonBuilder *js_fileinfo =
JsonBuildFileInfoRecord(p, ff, true, dir, ctx->xff_cfg, NULL);
JsonBuildFileInfoRecord(p, ff, tx, tx_id, true, dir, ctx->xff_cfg, NULL);
if (likely(js_fileinfo != NULL)) {
jb_close(js_fileinfo);
FILE *out = fopen(js_metadata_filename, "w");
@ -175,9 +176,9 @@ static void OutputFilestoreFinalizeFiles(ThreadVars *tv,
}
}
static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data,
const Packet *p, File *ff, const uint8_t *data, uint32_t data_len,
uint8_t flags, uint8_t dir)
static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data, const Packet *p, File *ff,
void *tx, const uint64_t tx_id, const uint8_t *data, uint32_t data_len, uint8_t flags,
uint8_t dir)
{
SCEnter();
OutputFilestoreLogThread *aft = (OutputFilestoreLogThread *)thread_data;
@ -260,7 +261,7 @@ static int OutputFilestoreLogger(ThreadVars *tv, void *thread_data,
ff->fd = -1;
SC_ATOMIC_SUB(filestore_open_file_cnt, 1);
}
OutputFilestoreFinalizeFiles(tv, aft, ctx, p, ff, dir);
OutputFilestoreFinalizeFiles(tv, aft, ctx, p, ff, tx, tx_id, dir);
}
return 0;

25
src/output-json-alert.c
Unescape Escape View File

@ -580,21 +580,26 @@ static void AlertAddAppLayer(const Packet *p, JsonBuilder *jb,
static void AlertAddFiles(const Packet *p, JsonBuilder *jb, const uint64_t tx_id)
{
FileContainer *ffc = AppLayerParserGetFiles(p->flow,
p->flowflags & FLOW_PKT_TOSERVER ? STREAM_TOSERVER:STREAM_TOCLIENT);
const uint8_t direction =
(p->flowflags & FLOW_PKT_TOSERVER) ? STREAM_TOSERVER : STREAM_TOCLIENT;
FileContainer *ffc = NULL;
if (p->flow->alstate != NULL) {
void *tx = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, tx_id);
if (tx) {
ffc = AppLayerParserGetTxFiles(p->flow, tx, direction);
}
}
if (ffc != NULL) {
File *file = ffc->head;
bool isopen = false;
while (file) {
if (tx_id == file->txid) {
if (!isopen) {
isopen = true;
jb_open_array(jb, "files");
}
jb_start_object(jb);
EveFileInfo(jb, file, file->flags & FILE_STORED);
jb_close(jb);
if (!isopen) {
isopen = true;
jb_open_array(jb, "files");
}
jb_start_object(jb);
EveFileInfo(jb, file, tx_id, file->flags & FILE_STORED);
jb_close(jb);
file = file->next;
}
if (isopen) {

35
src/output-json-file.c
Unescape Escape View File

@ -80,8 +80,9 @@ typedef struct JsonFileLogThread_ {
OutputJsonThreadCtx *ctx;
} JsonFileLogThread;
JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool stored,
uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx)
JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx,
const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg,
OutputJsonCtx *eve_ctx)
{
enum OutputJsonLogDirection fdir = LOG_DIR_FLOW;
@ -105,7 +106,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool
char xff_buffer[XFF_MAXLEN];
if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED)) {
if (FlowGetAppProtocol(p->flow) == ALPROTO_HTTP1) {
have_xff_ip = HttpXFFGetIPFromTx(p->flow, ff->txid, xff_cfg, xff_buffer, XFF_MAXLEN);
have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, xff_buffer, XFF_MAXLEN);
}
if (have_xff_ip && xff_cfg->flags & XFF_OVERWRITE) {
if (p->flowflags & FLOW_PKT_TOCLIENT) {
@ -125,20 +126,20 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool
switch (p->flow->alproto) {
case ALPROTO_HTTP1:
jb_open_object(js, "http");
EveHttpAddMetadata(p->flow, ff->txid, js);
EveHttpAddMetadata(p->flow, tx_id, js);
jb_close(js);
break;
case ALPROTO_SMTP:
jb_get_mark(js, &mark);
jb_open_object(js, "smtp");
if (EveSMTPAddMetadata(p->flow, ff->txid, js)) {
if (EveSMTPAddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
}
jb_get_mark(js, &mark);
jb_open_object(js, "email");
if (EveEmailAddMetadata(p->flow, ff->txid, js)) {
if (EveEmailAddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
@ -148,7 +149,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool
/* rpc */
jb_get_mark(js, &mark);
jb_open_object(js, "rpc");
if (EveNFSAddMetadataRPC(p->flow, ff->txid, js)) {
if (EveNFSAddMetadataRPC(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
@ -156,7 +157,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool
/* nfs */
jb_get_mark(js, &mark);
jb_open_object(js, "nfs");
if (EveNFSAddMetadata(p->flow, ff->txid, js)) {
if (EveNFSAddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
@ -165,7 +166,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool
case ALPROTO_SMB:
jb_get_mark(js, &mark);
jb_open_object(js, "smb");
if (EveSMBAddMetadata(p->flow, ff->txid, js)) {
if (EveSMBAddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
@ -174,7 +175,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool
case ALPROTO_HTTP2:
jb_get_mark(js, &mark);
jb_open_object(js, "http2");
if (EveHTTP2AddMetadata(p->flow, ff->txid, js)) {
if (EveHTTP2AddMetadata(p->flow, tx_id, js)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
@ -185,7 +186,7 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool
jb_set_string(js, "app_proto", AppProtoToString(p->flow->alproto));
jb_open_object(js, "fileinfo");
EveFileInfo(js, ff, stored);
EveFileInfo(js, ff, tx_id, stored);
jb_close(js);
/* xff header */
@ -200,13 +201,13 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool
* \internal
* \brief Write meta data on a single line json record
*/
static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff,
uint8_t dir, OutputJsonCtx *eve_ctx)
static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx,
const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx)
{
HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ?
aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg;;
JsonBuilder *js = JsonBuildFileInfoRecord(
p, ff, ff->flags & FILE_STORED ? true : false, dir, xff_cfg, eve_ctx);
p, ff, tx, tx_id, ff->flags & FILE_STORED ? true : false, dir, xff_cfg, eve_ctx);
if (unlikely(js == NULL)) {
return;
}
@ -215,8 +216,8 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
jb_free(js);
}
static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
const File *ff, uint8_t dir)
static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff,
void *tx, const uint64_t tx_id, uint8_t dir)
{
SCEnter();
JsonFileLogThread *aft = (JsonFileLogThread *)thread_data;
@ -225,7 +226,7 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p,
SCLogDebug("ff %p", ff);
FileWriteJsonRecord(aft, p, ff, dir, aft->filelog_ctx->eve_ctx);
FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx);
return 0;
}

5
src/output-json-file.h
Unescape Escape View File

@ -29,7 +29,8 @@
typedef struct OutputJsonCtx_ OutputJsonCtx;
void JsonFileLogRegister(void);
JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, const bool stored,
uint8_t dir, HttpXFFCfg *xff_cfg, OutputJsonCtx *eve_ctx);
JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx,
const uint64_t tx_id, const bool stored, uint8_t dir, HttpXFFCfg *xff_cfg,
OutputJsonCtx *eve_ctx);
#endif /* __OUTPUT_JSON_FILE_H__ */

4
src/output-json.c
Unescape Escape View File

@ -127,7 +127,7 @@ json_t *SCJsonString(const char *val)
/* Default Sensor ID value */
static int64_t sensor_id = -1; /* -1 = not defined */
void EveFileInfo(JsonBuilder *jb, const File *ff, const bool stored)
void EveFileInfo(JsonBuilder *jb, const File *ff, const uint64_t tx_id, const bool stored)
{
jb_set_string_from_bytes(jb, "filename", ff->name, ff->name_len);
@ -181,7 +181,7 @@ void EveFileInfo(JsonBuilder *jb, const File *ff, const bool stored)
jb_set_uint(jb, "start", ff->start);
jb_set_uint(jb, "end", ff->end);
}
jb_set_uint(jb, "tx_id", ff->txid);
jb_set_uint(jb, "tx_id", tx_id);
}
static void EveAddPacketVars(const Packet *p, JsonBuilder *js_vars)

2
src/output-json.h
Unescape Escape View File

@ -95,7 +95,7 @@ typedef struct OutputJsonThreadCtx_ {
json_t *SCJsonString(const char *val);
void CreateEveFlowId(JsonBuilder *js, const Flow *f);
void EveFileInfo(JsonBuilder *js, const File *file, const bool stored);
void EveFileInfo(JsonBuilder *js, const File *file, const uint64_t tx_id, const bool stored);
void EveTcpFlags(uint8_t flags, JsonBuilder *js);
void EvePacket(const Packet *p, JsonBuilder *js, unsigned long max_length);
JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir,

10
src/output-lua.c
Unescape Escape View File

@ -291,7 +291,8 @@ static int LuaPacketCondition(ThreadVars *tv, void *data, const Packet *p)
*
* NOTE p->flow is locked at this point
*/
static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff, uint8_t dir)
static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, const File *ff,
void *tx, const uint64_t tx_id, uint8_t dir)
{
SCEnter();
LogLuaThreadCtx *td = (LogLuaThreadCtx *)thread_data;
@ -307,12 +308,7 @@ static int LuaFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, con
LuaStateSetThreadVars(td->lua_ctx->luastate, tv);
LuaStateSetPacket(td->lua_ctx->luastate, (Packet *)p);
if (p->flow && p->flow->alstate) {
void *txptr = AppLayerParserGetTx(p->proto, p->flow->alproto, p->flow->alstate, ff->txid);
if (txptr) {
LuaStateSetTX(td->lua_ctx->luastate, txptr, ff->txid);
}
}
LuaStateSetTX(td->lua_ctx->luastate, tx, tx_id);
LuaStateSetFlow(td->lua_ctx->luastate, p->flow);
LuaStateSetFile(td->lua_ctx->luastate, (File *)ff);

368
src/output-tx.c
Unescape Escape View File

@ -36,6 +36,11 @@
* data for the packet loggers. */
typedef struct OutputTxLoggerThreadData_ {
OutputLoggerThreadStore *store[ALPROTO_MAX];
/* thread local data from file api */
OutputFileLoggerThreadData *file;
/* thread local data from filedata api */
OutputFiledataLoggerThreadData *filedata;
} OutputTxLoggerThreadData;
/* logger instance, a module + a output ctx,
@ -68,8 +73,8 @@ int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto,
void (*ThreadExitPrintStats)(ThreadVars *, void *))
{
if (alproto != ALPROTO_UNKNOWN && !(AppLayerParserIsEnabled(alproto))) {
SCLogNotice("%s logger not enabled: protocol %s is disabled",
name, AppProtoToString(alproto));
SCLogDebug(
"%s logger not enabled: protocol %s is disabled", name, AppProtoToString(alproto));
return -1;
}
OutputTxLogger *op = SCMalloc(sizeof(*op));
@ -125,6 +130,110 @@ int OutputRegisterTxLogger(LoggerId id, const char *name, AppProto alproto,
return 0;
}
extern bool g_file_logger_enabled;
extern bool g_filedata_logger_enabled;
/** \brief run the per tx file logging
* \todo clean up various end of tx/stream etc indicators
*/
static inline void OutputTxLogFiles(ThreadVars *tv, OutputFileLoggerThreadData *file_td,
OutputFiledataLoggerThreadData *filedata_td, Packet *p, Flow *f, void *tx,
const uint64_t tx_id, AppLayerTxData *txd, const bool tx_complete, const bool ts_ready,
const bool tc_ready, const bool ts_eof, const bool tc_eof, const bool eof)
{
int packet_dir;
int opposing_dir;
bool packet_dir_ready;
const bool opposing_dir_ready = eof;
bool opposing_tx_ready;
if (p->flowflags & FLOW_PKT_TOSERVER) {
packet_dir = STREAM_TOSERVER;
opposing_dir = STREAM_TOCLIENT;
packet_dir_ready = eof | ts_ready | ts_eof;
opposing_tx_ready = tc_ready;
} else if (p->flowflags & FLOW_PKT_TOCLIENT) {
packet_dir = STREAM_TOCLIENT;
opposing_dir = STREAM_TOSERVER;
packet_dir_ready = eof | tc_ready | tc_eof;
opposing_tx_ready = ts_ready;
} else {
abort();
}
SCLogDebug("eof %d ts_ready %d ts_eof %d", eof, ts_ready, ts_eof);
SCLogDebug("eof %d tc_ready %d tc_eof %d", eof, tc_ready, tc_eof);
SCLogDebug("packet dir %s opposing %s packet_dir_ready %d opposing_dir_ready %d",
packet_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT",
opposing_dir == STREAM_TOSERVER ? "TOSERVER" : "TOCLIENT", packet_dir_ready,
opposing_dir_ready);
FileContainer *ffc = AppLayerParserGetTxFiles(f, tx, packet_dir);
FileContainer *ffc_opposing = AppLayerParserGetTxFiles(f, tx, opposing_dir);
/* see if opposing side is finished: if no file support in this direction, of is not
* files and tx is done for opposing dir. */
bool opposing_finished =
ffc_opposing == NULL || (ffc_opposing->head == NULL && opposing_tx_ready);
SCLogDebug("opposing_finished %d ffc_opposing %p ffc_opposing->head %p opposing_tx_ready %d",
opposing_finished, ffc_opposing, ffc_opposing ? ffc_opposing->head : NULL,
opposing_tx_ready);
if (ffc || ffc_opposing)
SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64
" ffc %p ffc_opposing %p tx_complete %d",
p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete);
if (ffc) {
const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | eof;
const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof;
SCLogDebug("tx: calling files: ffc %p head %p file_close %d file_trunc %d", ffc, ffc->head,
file_close, file_trunc);
if (filedata_td)
OutputFiledataLogFfc(tv, filedata_td, p, ffc, tx, tx_id, txd, packet_dir, file_close,
file_trunc, packet_dir);
if (file_td)
OutputFileLogFfc(
tv, file_td, p, ffc, tx, tx_id, txd, file_close, file_trunc, packet_dir);
}
/* if EOF and we support files, do a final write out */
if (opposing_dir_ready && ffc_opposing != NULL) {
const bool file_close = ((p->flags & PKT_PSEUDO_STREAM_END)) | tx_complete | eof;
const bool file_trunc = StreamTcpReassembleDepthReached(p) | eof;
opposing_finished = true;
SCLogDebug("tx: calling for opposing direction files: file_close:%s file_trunc:%s",
file_close ? "true" : "false", file_trunc ? "true" : "false");
if (filedata_td)
OutputFiledataLogFfc(tv, filedata_td, p, ffc_opposing, tx, tx_id, txd, opposing_dir,
file_close, file_trunc, opposing_dir);
if (file_td)
OutputFileLogFfc(tv, file_td, p, ffc_opposing, tx, tx_id, txd, file_close, file_trunc,
opposing_dir);
}
const bool tx_done = packet_dir_ready && opposing_finished;
SCLogDebug("tx_done %d packet_dir_ready %d opposing_finished %d", tx_done, packet_dir_ready,
opposing_finished);
/* if not a file tx or if tx is done, set logger flags so tx can move on */
const bool is_file_tx = (ffc != NULL || ffc_opposing != NULL);
if (!is_file_tx || tx_done) {
SCLogDebug("is_file_tx %d tx_done %d", is_file_tx, tx_done);
if (file_td) {
txd->logged.flags |= BIT_U32(LOGGER_FILE);
SCLogDebug("setting LOGGER_FILE => %08x", txd->logged.flags);
}
if (filedata_td) {
txd->logged.flags |= BIT_U32(LOGGER_FILEDATA);
SCLogDebug("setting LOGGER_FILEDATA => %08x", txd->logged.flags);
}
} else {
SCLogDebug("pcap_cnt %" PRIu64 " flow %p tx %p tx_id %" PRIu64
" NOT SETTING FILE FLAGS ffc %p ffc_opposing %p tx_complete %d",
p->pcap_cnt, f, tx, tx_id, ffc, ffc_opposing, tx_complete);
}
}
static void OutputTxLogList0(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data, Packet *p,
Flow *f, void *tx, const uint64_t tx_id)
{
@ -154,6 +263,73 @@ static void OutputTxLogList0(ThreadVars *tv, OutputTxLoggerThreadData *op_thread
}
}
struct Ctx {
uint32_t tx_logged_old;
uint32_t tx_logged;
};
static void OutputTxLogCallLoggers(ThreadVars *tv, OutputTxLoggerThreadData *op_thread_data,
const OutputTxLogger *logger, const OutputLoggerThreadStore *store, Packet *p, Flow *f,
void *alstate, void *tx, const uint64_t tx_id, const AppProto alproto, const bool eof,
const int tx_progress_ts, const int tx_progress_tc, struct Ctx *ctx)
{
DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
// DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
while (logger && store) {
DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
DEBUG_VALIDATE_BUG_ON(logger->alproto != alproto);
SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d "
"tc_log_progress %d",
logger, logger->alproto, logger->LogCondition, logger->ts_log_progress,
logger->tc_log_progress);
if ((ctx->tx_logged_old & BIT_U32(logger->logger_id)) == 0) {
SCLogDebug("alproto match %d, logging tx_id %" PRIu64, logger->alproto, tx_id);
SCLogDebug("pcap_cnt %" PRIu64 ", tx_id %" PRIu64 " logger %d. EOF %s", p->pcap_cnt,
tx_id, logger->logger_id, eof ? "true" : "false");
if (eof) {
SCLogDebug("EOF, so log now");
} else {
if (logger->LogCondition) {
int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
if (r == FALSE) {
SCLogDebug("conditions not met, not logging");
goto next_logger;
}
} else {
if (tx_progress_tc < logger->tc_log_progress) {
SCLogDebug("progress not far enough, not logging");
goto next_logger;
}
if (tx_progress_ts < logger->ts_log_progress) {
SCLogDebug("progress not far enough, not logging");
goto next_logger;
}
}
}
SCLogDebug("Logging tx_id %" PRIu64 " to logger %d", tx_id, logger->logger_id);
PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
ctx->tx_logged |= BIT_U32(logger->logger_id);
}
next_logger:
logger = logger->next;
store = store->next;
DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
}
}
static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
{
DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
@ -165,35 +341,52 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
Flow * const f = p->flow;
const uint8_t ipproto = f->proto;
const AppProto alproto = f->alproto;
if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) {
/* No child loggers registered. */
return TM_ECODE_OK;
SCLogDebug("pcap_cnt %u tx logging %u/%s", (uint32_t)p->pcap_cnt, alproto,
AppProtoToString(alproto));
if (op_thread_data->file == NULL && op_thread_data->filedata == NULL) {
if (list[alproto] == NULL && list[ALPROTO_UNKNOWN] == NULL) {
SCLogDebug("bail");
/* No child loggers registered. */
return TM_ECODE_OK;
}
if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0)
goto end;
}
if (AppLayerParserProtocolHasLogger(p->proto, alproto) == 0)
goto end;
const LoggerId logger_expectation = AppLayerParserProtocolGetLoggerBits(p->proto, alproto);
if (logger_expectation == 0)
if (logger_expectation == 0) {
SCLogDebug("bail: logger_expectation %u. LOGGER_FILE %u LOGGER_FILEDATA %u",
logger_expectation, LOGGER_FILE, LOGGER_FILEDATA);
goto end;
}
void *alstate = f->alstate;
if (alstate == NULL) {
SCLogDebug("no alstate");
goto end;
}
SCLogDebug("pcap_cnt %" PRIu64, p->pcap_cnt);
const bool last_pseudo = (p->flowflags & FLOW_PKT_LAST_PSEUDO) != 0;
const bool ts_eof = AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF_TS) != 0;
const bool tc_eof = AppLayerParserStateIssetFlag(f->alparser, APP_LAYER_PARSER_EOF_TC) != 0;
const bool ts_eof = AppLayerParserStateIssetFlag(f->alparser,
(APP_LAYER_PARSER_EOF_TS | APP_LAYER_PARSER_TRUNC_TS)) != 0;
const bool tc_eof = AppLayerParserStateIssetFlag(f->alparser,
(APP_LAYER_PARSER_EOF_TC | APP_LAYER_PARSER_TRUNC_TC)) != 0;
const bool eof = last_pseudo || (ts_eof && tc_eof);
SCLogDebug("eof %d last_pseudo %d ts_eof %d tc_eof %d", eof, last_pseudo, ts_eof, tc_eof);
const uint8_t ts_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOSERVER);
const uint8_t tc_disrupt_flags = FlowGetDisruptionFlags(f, STREAM_TOCLIENT);
SCLogDebug("ts_disrupt_flags %02x tc_disrupt_flags %02x", ts_disrupt_flags, tc_disrupt_flags);
const uint64_t total_txs = AppLayerParserGetTxCnt(f, alstate);
uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
uint64_t max_id = tx_id;
int logged = 0;
int gap = 0;
SCLogDebug("tx_id %" PRIu64 " total_txs %" PRIu64, tx_id, total_txs);
AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
AppLayerGetTxIterState state;
memset(&state, 0, sizeof(state));
@ -204,8 +397,34 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
break;
void * const tx = ires.tx_ptr;
tx_id = ires.tx_id;
SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx);
const int tx_progress_ts =
AppLayerParserGetStateProgress(p->proto, alproto, tx, ts_disrupt_flags);
const int tx_progress_tc =
AppLayerParserGetStateProgress(p->proto, alproto, tx, tc_disrupt_flags);
const bool tx_complete = (tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus(
alproto, STREAM_TOSERVER) &&
tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus(
alproto, STREAM_TOCLIENT));
const bool ts_ready = tx_progress_ts == AppLayerParserGetStateProgressCompletionStatus(
alproto, STREAM_TOSERVER);
const bool tc_ready = tx_progress_tc == AppLayerParserGetStateProgressCompletionStatus(
alproto, STREAM_TOCLIENT);
SCLogDebug("ts_ready %d tc_ready %d", ts_ready, tc_ready);
SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file,
op_thread_data->filedata);
AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx);
if (txd && (op_thread_data->file || op_thread_data->filedata) &&
AppLayerParserSupportsFiles(p->proto, alproto)) {
OutputTxLogFiles(tv, op_thread_data->file, op_thread_data->filedata, p, f, tx, tx_id,
txd, tx_complete, ts_ready, tc_ready, ts_eof, tc_eof, eof);
}
if (txd)
SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
if (txd == NULL) {
/* make sure this tx, which can't be properly logged is skipped */
logged = 1;
@ -219,93 +438,33 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
goto next_tx;
}
SCLogDebug("tx %p/%"PRIu64" txd %p: log_flags %x", tx, tx_id, txd, txd->config.log_flags);
SCLogDebug("tx %p/%" PRIu64 " txd %p: log_flags %x logger_expectation %x", tx, tx_id, txd,
txd->config.log_flags, logger_expectation);
if (txd->config.log_flags & BIT_U8(CONFIG_TYPE_TX)) {
SCLogDebug("SKIP tx %p/%"PRIu64, tx, tx_id);
goto next_tx;
}
LoggerId tx_logged = txd->logged.flags;
const LoggerId tx_logged_old = tx_logged;
SCLogDebug("logger: expect %08x, have %08x", logger_expectation, tx_logged);
if (tx_logged == logger_expectation) {
if (txd->logged.flags == logger_expectation) {
SCLogDebug("fully logged");
/* tx already fully logged */
goto next_tx;
}
const int tx_progress_ts =
AppLayerParserGetStateProgress(p->proto, alproto, tx, ts_disrupt_flags);
const int tx_progress_tc =
AppLayerParserGetStateProgress(p->proto, alproto, tx, tc_disrupt_flags);
SCLogDebug("tx_progress_ts %d tx_progress_tc %d",
tx_progress_ts, tx_progress_tc);
SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
const OutputTxLogger *logger = list[alproto];
const OutputLoggerThreadStore *store = op_thread_data->store[alproto];
struct Ctx ctx = { .tx_logged = txd->logged.flags, .tx_logged_old = txd->logged.flags };
SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
DEBUG_VALIDATE_BUG_ON(logger == NULL && store == NULL);
OutputTxLogCallLoggers(tv, op_thread_data, logger, store, p, f, alstate, tx, tx_id, alproto,
eof, tx_progress_ts, tx_progress_tc, &ctx);
while (logger && store) {
DEBUG_VALIDATE_BUG_ON(logger->LogFunc == NULL);
DEBUG_VALIDATE_BUG_ON(logger->alproto != alproto);
SCLogDebug("logger %p, Alproto %d LogCondition %p, ts_log_progress %d "
"tc_log_progress %d", logger, logger->alproto, logger->LogCondition,
logger->ts_log_progress, logger->tc_log_progress);
if ((tx_logged_old & BIT_U32(logger->logger_id)) == 0) {
SCLogDebug("alproto match %d, logging tx_id %"PRIu64, logger->alproto, tx_id);
SCLogDebug("pcap_cnt %"PRIu64", tx_id %"PRIu64" logger %d. "
"EOFs TS %s TC %s LAST PSEUDO %s",
p->pcap_cnt, tx_id, logger->logger_id,
ts_eof ? "true" : "false", tc_eof ? "true" : "false",
last_pseudo ? "true" : "false");
if ((ts_eof && tc_eof) || last_pseudo) {
SCLogDebug("EOF, so log now");
} else {
if (logger->LogCondition) {
int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
if (r == FALSE) {
SCLogDebug("conditions not met, not logging");
goto next_logger;
}
} else {
if (tx_progress_tc < logger->tc_log_progress) {
SCLogDebug("progress not far enough, not logging");
goto next_logger;
}
if (tx_progress_ts < logger->ts_log_progress) {
SCLogDebug("progress not far enough, not logging");
goto next_logger;
}
}
}
SCLogDebug("Logging tx_id %"PRIu64" to logger %d", tx_id, logger->logger_id);
PACKET_PROFILING_LOGGER_START(p, logger->logger_id);
logger->LogFunc(tv, store->thread_data, p, f, alstate, tx, tx_id);
PACKET_PROFILING_LOGGER_END(p, logger->logger_id);
tx_logged |= BIT_U32(logger->logger_id);
}
next_logger:
logger = logger->next;
store = store->next;
DEBUG_VALIDATE_BUG_ON(logger == NULL && store != NULL);
DEBUG_VALIDATE_BUG_ON(logger != NULL && store == NULL);
}
if (tx_logged != tx_logged_old) {
SCLogDebug("logger: storing %08x (was %08x)",
tx_logged, tx_logged_old);
SCLogDebug("logger: expect %08x, have %08x", logger_expectation, ctx.tx_logged);
if (ctx.tx_logged != ctx.tx_logged_old) {
SCLogDebug("logger: storing %08x (was %08x)", ctx.tx_logged, ctx.tx_logged_old);
DEBUG_VALIDATE_BUG_ON(txd == NULL);
txd->logged.flags |= tx_logged;
txd->logged.flags |= ctx.tx_logged;
}
/* If all loggers logged set a flag and update the last tx_id
@ -314,9 +473,11 @@ next_logger:
* If not all loggers were logged we flag that there was a gap
* so any subsequent transactions in this loop don't increase
* the maximum ID that was logged. */
if (!gap && tx_logged == logger_expectation) {
if (!gap && ctx.tx_logged == logger_expectation) {
SCLogDebug("no gap %d, %08x == %08x", gap, ctx.tx_logged, logger_expectation);
logged = 1;
max_id = tx_id;
SCLogDebug("max_id %" PRIu64, max_id);
} else {
gap = 1;
}
@ -340,7 +501,7 @@ end:
/** \brief thread init for the tx logger
* This will run the thread init functions for the individual registered
* loggers */
static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *initdata, void **data)
static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *_initdata, void **data)
{
OutputTxLoggerThreadData *td = SCMalloc(sizeof(*td));
if (td == NULL)
@ -379,6 +540,20 @@ static TmEcode OutputTxLogThreadInit(ThreadVars *tv, const void *initdata, void
logger = logger->next;
}
}
if (g_file_logger_enabled) {
if (OutputFileLogThreadInit(tv, &td->file) != TM_ECODE_OK) {
FatalError(SC_ERR_FATAL, "failed to set up file thread data");
}
}
if (g_filedata_logger_enabled) {
if (OutputFiledataLogThreadInit(tv, &td->filedata) != TM_ECODE_OK) {
FatalError(SC_ERR_FATAL, "failed to set up filedata thread data");
}
}
SCLogDebug("file_thread_data %p filedata_thread_data %p", td->file, td->filedata);
return TM_ECODE_OK;
}
@ -402,6 +577,13 @@ static TmEcode OutputTxLogThreadDeinit(ThreadVars *tv, void *thread_data)
}
}
if (op_thread_data->file) {
OutputFileLogThreadDeinit(tv, op_thread_data->file);
}
if (op_thread_data->filedata) {
OutputFiledataLogThreadDeinit(tv, op_thread_data->filedata);
}
SCFree(op_thread_data);
return TM_ECODE_OK;
}
@ -433,6 +615,16 @@ static uint32_t OutputTxLoggerGetActiveCount(void)
cnt++;
}
}
if (g_file_logger_enabled) {
cnt++;
SCLogDebug("g_file_logger_enabled");
}
if (g_filedata_logger_enabled) {
cnt++;
SCLogDebug("g_filedata_logger_enabled");
}
return cnt;
}

3
src/output.c
Unescape Escape View File

@ -887,7 +887,6 @@ TmEcode OutputLoggerLog(ThreadVars *tv, Packet *p, void *thread_data)
logger = TAILQ_NEXT(logger, entries);
thread_store_node = TAILQ_NEXT(thread_store_node, entries);
}
return TM_ECODE_OK;
}
@ -1029,9 +1028,9 @@ void TmModuleLoggerRegister(void)
void OutputRegisterRootLoggers(void)
{
OutputPacketLoggerRegister();
OutputTxLoggerRegister();
OutputFiledataLoggerRegister();
OutputFileLoggerRegister();
OutputTxLoggerRegister();
OutputStreamingLoggerRegister();
}

11
src/output.h
Unescape Escape View File

@ -27,6 +27,11 @@
#define DEFAULT_LOG_MODE_APPEND "yes"
#define DEFAULT_LOG_FILETYPE "regular"
typedef struct OutputLoggerThreadStore_ {
void *thread_data;
struct OutputLoggerThreadStore_ *next;
} OutputLoggerThreadStore;
#include "output-packet.h"
#include "output-tx.h"
#include "output-file.h"
@ -35,12 +40,6 @@
#include "output-streaming.h"
#include "output-stats.h"
typedef struct OutputLoggerThreadStore_ {
void *thread_data;
struct OutputLoggerThreadStore_ *next;
} OutputLoggerThreadStore;
typedef struct OutputInitResult_ {
OutputCtx *ctx;
bool ok;

29
src/runmodes.c
Unescape Escape View File

@ -629,7 +629,7 @@ static void SetupOutput(const char *name, OutputModule *module, OutputCtx *outpu
module->ThreadExitPrintStats);
/* Not used with wild card loggers */
if (module->alproto != ALPROTO_UNKNOWN) {
logger_bits[module->alproto] |= (1<<module->logger_id);
logger_bits[module->alproto] |= BIT_U32(module->logger_id);
}
} else if (module->FiledataLogFunc) {
SCLogDebug("%s is a filedata logger", module->name);
@ -757,6 +757,9 @@ static void RunModeInitializeLuaOutput(ConfNode *conf, OutputCtx *parent_ctx)
}
}
extern bool g_file_logger_enabled;
extern bool g_filedata_logger_enabled;
/**
* Initialize the output modules.
*/
@ -919,11 +922,31 @@ void RunModeInitializeOutputs(void)
/* register the logger bits to the app-layer */
AppProto a;
for (a = 0; a < ALPROTO_MAX; a++) {
if (AppLayerParserSupportsFiles(IPPROTO_TCP, a)) {
if (g_file_logger_enabled)
logger_bits[a] |= BIT_U32(LOGGER_FILE);
if (g_filedata_logger_enabled)
logger_bits[a] |= BIT_U32(LOGGER_FILEDATA);
SCLogDebug("IPPROTO_TCP::%s: g_file_logger_enabled %d g_filedata_logger_enabled %d -> "
"%08x",
AppProtoToString(a), g_file_logger_enabled, g_filedata_logger_enabled,
logger_bits[a]);
}
if (AppLayerParserSupportsFiles(IPPROTO_UDP, a)) {
if (g_file_logger_enabled)
logger_bits[a] |= BIT_U32(LOGGER_FILE);
if (g_filedata_logger_enabled)
logger_bits[a] |= BIT_U32(LOGGER_FILEDATA);
}
if (logger_bits[a] == 0)
continue;
const int tcp = AppLayerParserProtocolHasLogger(IPPROTO_TCP, a);
const int udp = AppLayerParserProtocolHasLogger(IPPROTO_UDP, a);
const int tcp = AppLayerParserProtocolHasLogger(IPPROTO_TCP, a) | (g_file_logger_enabled) |
(g_filedata_logger_enabled);
const int udp = AppLayerParserProtocolHasLogger(IPPROTO_UDP, a) | (g_file_logger_enabled) |
(g_filedata_logger_enabled);
SCLogDebug("tcp %d udp %d", tcp, udp);
SCLogDebug("logger for %s: %s %s", AppProtoToString(a),
tcp ? "true" : "false", udp ? "true" : "false");

1
src/rust-context.c
Unescape Escape View File

@ -38,7 +38,6 @@ const SuricataContext suricata_context = {
FileAppendGAPById,
FileContainerRecycle,
FilePrune,
FileContainerSetTx,
AppLayerRegisterParser,
};

1
src/rust-context.h
Unescape Escape View File

@ -54,7 +54,6 @@ typedef struct SuricataContext_ {
const uint8_t *data, uint32_t data_len);
void (*FileContainerRecycle)(FileContainer *ffc);
void (*FilePrune)(FileContainer *ffc);
void (*FileSetTx)(FileContainer *, uint64_t);
int (*AppLayerRegisterParser)(const struct AppLayerParser *p, AppProto alproto);

3
src/suricata-common.h
Unescape Escape View File

@ -472,6 +472,9 @@ typedef enum {
LOGGER_JSON_DCERPC,
LOGGER_JSON_HTTP2,
LOGGER_FILE,
LOGGER_FILEDATA,
/** \warning when we exceed what we can express as a u32 flag here we need to update
* LoggerFlags::flags (u32) and `tx_logged` in src/output-tx.c */

152
src/util-file.c
Unescape Escape View File

@ -289,6 +289,25 @@ uint16_t FileFlowToFlags(const Flow *flow, uint8_t direction)
return FileFlowFlagsToFlags(flow->file_flags, direction);
}
void FileApplyTxFlags(const AppLayerTxData *txd, const uint8_t direction, File *file)
{
SCLogDebug("file flags %04x STORE %s NOSTORE %s", file->flags,
(file->flags & FILE_STORE) ? "true" : "false",
(file->flags & FILE_NOSTORE) ? "true" : "false");
uint16_t update_flags = FileFlowFlagsToFlags(txd->file_flags, direction);
DEBUG_VALIDATE_BUG_ON(
(file->flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE));
if (file->flags & FILE_STORE)
update_flags &= ~FILE_NOSTORE;
file->flags |= update_flags;
SCLogDebug("file flags %04x STORE %s NOSTORE %s", file->flags,
(file->flags & FILE_STORE) ? "true" : "false",
(file->flags & FILE_NOSTORE) ? "true" : "false");
DEBUG_VALIDATE_BUG_ON(
(file->flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE));
}
static int FileMagicSize(void)
{
/** \todo make this size configurable */
@ -608,27 +627,6 @@ int FileStore(File *ff)
SCReturnInt(0);
}
/**
* \brief Set the TX id for a file
*
* \param ff The file to store
* \param txid the tx id
*/
int FileSetTx(File *ff, uint64_t txid)
{
SCLogDebug("ff %p txid %"PRIu64, ff, txid);
if (ff != NULL)
ff->txid = txid;
SCReturnInt(0);
}
void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id)
{
if (ffc && ffc->tail) {
(void)FileSetTx(ffc->tail, tx_id);
}
}
/**
* \brief check if we have stored enough
*
@ -1129,78 +1127,17 @@ void FileUpdateFlowFileFlags(Flow *f, uint16_t set_file_flags, uint8_t direction
f->file_flags, set_file_flags, g_file_flow_mask);
if (set_file_flags != 0 && f->alproto != ALPROTO_UNKNOWN && f->alstate != NULL) {
uint16_t per_file_flags = 0;
#ifdef HAVE_MAGIC
if (set_file_flags & (FLOWFILE_NO_MAGIC_TS|FLOWFILE_NO_MAGIC_TC))
per_file_flags |= FILE_NOMAGIC;
#endif
if (set_file_flags & (FLOWFILE_NO_MD5_TS|FLOWFILE_NO_MD5_TC))
per_file_flags |= FILE_NOMD5;
if (set_file_flags & (FLOWFILE_NO_SHA1_TS|FLOWFILE_NO_SHA1_TC))
per_file_flags |= FILE_NOSHA1;
if (set_file_flags & (FLOWFILE_NO_SHA256_TS|FLOWFILE_NO_SHA256_TC))
per_file_flags |= FILE_NOSHA256;
if (set_file_flags & (FLOWFILE_NO_SIZE_TS|FLOWFILE_NO_SIZE_TC))
per_file_flags |= FILE_NOTRACK;
if (set_file_flags & (FLOWFILE_NO_STORE_TS|FLOWFILE_NO_STORE_TC))
per_file_flags |= FILE_NOSTORE;
FileContainer *ffc = AppLayerParserGetFiles(f, direction);
if (ffc != NULL) {
for (File *ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
ptr->flags |= per_file_flags;
/* destroy any ctx we may have so far */
if ((per_file_flags & FILE_NOSHA256) &&
ptr->sha256_ctx != NULL)
{
SCSha256Free(ptr->sha256_ctx);
ptr->sha256_ctx = NULL;
}
if ((per_file_flags & FILE_NOSHA1) &&
ptr->sha1_ctx != NULL)
{
SCSha1Free(ptr->sha1_ctx);
ptr->sha1_ctx = NULL;
}
if ((per_file_flags & FILE_NOMD5) &&
ptr->md5_ctx != NULL)
{
SCMd5Free(ptr->md5_ctx);
ptr->md5_ctx = NULL;
}
AppLayerStateData *sd = AppLayerParserGetStateData(f->proto, f->alproto, f->alstate);
if (sd != NULL) {
if ((sd->file_flags & f->file_flags) != f->file_flags) {
SCLogDebug("state data: updating file_flags %04x with flow file_flags %04x",
sd->file_flags, f->file_flags);
sd->file_flags |= f->file_flags;
}
}
}
}
/**
* \brief set no store flag, close file if needed
*
* \param ff file
*/
static void FileDisableStoringForFile(File *ff)
{
SCEnter();
if (ff == NULL) {
SCReturn;
}
SCLogDebug("not storing this file");
ff->flags |= FILE_NOSTORE;
if (ff->state == FILE_STATE_OPENED && FileDataSize(ff) >= (uint64_t)FileMagicSize()) {
if (g_file_force_md5 == 0 && g_file_force_sha1 == 0 && g_file_force_sha256 == 0
&& g_file_force_tracking == 0) {
(void)FileCloseFilePtr(ff, NULL, 0,
(FILE_TRUNCATED|FILE_NOSTORE));
}
}
}
/**
* \brief disable file storing for files in a transaction
*
@ -1208,29 +1145,16 @@ static void FileDisableStoringForFile(File *ff)
* \param direction flow direction
* \param tx_id transaction id
*/
void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id)
void FileDisableStoringForTransaction(Flow *f, const uint8_t direction, void *tx, uint64_t tx_id)
{
File *ptr = NULL;
DEBUG_ASSERT_FLOW_LOCKED(f);
SCEnter();
FileContainer *ffc = AppLayerParserGetFiles(f, direction);
if (ffc != NULL) {
for (ptr = ffc->head; ptr != NULL; ptr = ptr->next) {
if (ptr->txid == tx_id) {
if (ptr->flags & FILE_STORE) {
/* weird, already storing -- let it continue*/
SCLogDebug("file is already being stored");
} else {
FileDisableStoringForFile(ptr);
}
}
AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, tx);
if (txd != NULL) {
if (direction & STREAM_TOSERVER) {
txd->file_flags |= FLOWFILE_NO_STORE_TS;
} else {
txd->file_flags |= FLOWFILE_NO_STORE_TC;
}
}
SCReturn;
}
/**
@ -1256,17 +1180,7 @@ void FileStoreFileById(FileContainer *fc, uint32_t file_id)
void FileStoreAllFilesForTx(FileContainer *fc, uint64_t tx_id)
{
File *ptr = NULL;
SCEnter();
if (fc != NULL) {
for (ptr = fc->head; ptr != NULL; ptr = ptr->next) {
if (ptr->txid == tx_id) {
FileStore(ptr);
}
}
}
abort();
}
void FileStoreAllFiles(FileContainer *fc)

17
src/util-file.h
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2007-2011 Open Information Security Foundation
/* Copyright (C) 2007-2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -26,7 +26,6 @@
#define __UTIL_FILE_H__
#include "conf.h"
#include "util-streaming-buffer.h"
/* Hack: Pulling rust.h to get the SCSha256 causes all sorts of problems with
@ -77,7 +76,6 @@ typedef struct File_ {
uint16_t name_len;
FileState state;
StreamingBuffer *sb;
uint64_t txid; /**< tx this file is part of */
uint32_t file_track_id; /**< id used by protocol parser */
uint32_t file_store_id; /**< id used in store file name file.<id> */
int fd; /**< file descriptor for filestore, not
@ -199,22 +197,15 @@ int FileSetRange(FileContainer *, uint64_t start, uint64_t end);
*/
int FileStore(File *);
/**
* \brief Set the TX id for a file
*
* \param ff The file to store
* \param txid the tx id
*/
int FileSetTx(File *, uint64_t txid);
void FileContainerSetTx(FileContainer *ffc, uint64_t tx_id);
/**
* \brief disable file storing for a transaction
*
* \param f flow
* \param direction STREAM_TOSERVER or STREAM_TOCLIENT
* \param tx transaction pointer
* \param tx_id transaction id
*/
void FileDisableStoringForTransaction(Flow *f, uint8_t direction, uint64_t tx_id);
void FileDisableStoringForTransaction(Flow *f, const uint8_t direction, void *tx, uint64_t tx_id);
void FlowFileDisableStoringForTransaction(struct Flow_ *f, uint64_t tx_id);
void FilePrune(FileContainer *ffc);

4
src/util-profiling.c
Unescape Escape View File

@ -1308,6 +1308,10 @@ const char * PacketProfileLoggertIdToString(LoggerId id)
CASE_CODE (LOGGER_JSON_STATS);
CASE_CODE (LOGGER_PCAP);
CASE_CODE (LOGGER_JSON_METADATA);
CASE_CODE(LOGGER_FILE);
CASE_CODE(LOGGER_FILEDATA);
case LOGGER_SIZE:
return "UNKNOWN";
}

Write Preview
Loading…
Cancel
Save