aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

rust/nfs: handle GAPs

Browse Source 
In normal records it will try to continue parsing.

GAP 'data' will be passed to file api as '0's. New call is used
so that the file API does know it is dealing with a GAP. Such
files are flagged as truncated at the end of the file and no
checksums are calculated.
pull/2787/head
Victor Julien 9 years ago
parent a116c16019
commit 58af39131f
10 changed files with 222 additions and 32 deletions
Show Stats Download Patch File Download Diff File

5
rust/src/core.rs
Unescape Escape View File

@ -80,6 +80,10 @@ pub type SCFileAppendDataById = extern "C" fn (
file_container: &FileContainer,
track_id: u32,
data: *const u8, data_len: u32) -> i32;
pub type SCFileAppendGAPById = extern "C" fn (
file_container: &FileContainer,
track_id: u32,
data: *const u8, data_len: u32) -> i32;
// void FilePrune(FileContainer *ffc)
pub type SCFilePrune = extern "C" fn (
file_container: &FileContainer);
@ -109,6 +113,7 @@ pub struct SuricataContext {
pub FileOpenFile: SCFileOpenFileWithId,
pub FileCloseFile: SCFileCloseFileById,
pub FileAppendData: SCFileAppendDataById,
pub FileAppendGAP: SCFileAppendGAPById,
pub FileContainerRecycle: SCFileContainerRecycle,
pub FilePrune: SCFilePrune,
pub FileSetTx: SCFileSetTx,

16
rust/src/filecontainer.rs
Unescape Escape View File

@ -63,7 +63,7 @@ impl FileContainer {
}
}
pub fn file_append(&mut self, track_id: &u32, data: &[u8]) -> i32 {
pub fn file_append(&mut self, track_id: &u32, data: &[u8], is_gap: bool) -> i32 {
SCLogDebug!("FILECONTAINER: append {}", data.len());
if data.len() == 0 {
return 0
@ -71,8 +71,20 @@ impl FileContainer {
match unsafe {SC} {
None => panic!("BUG no suricata_config"),
Some(c) => {
let res = (c.FileAppendData)(&self, *track_id,
let res = match is_gap {
false => {
SCLogDebug!("appending file data");
let r = (c.FileAppendData)(&self, *track_id,
data.as_ptr(), data.len() as u32);
r
},
true => {
SCLogDebug!("appending GAP");
let r = (c.FileAppendGAP)(&self, *track_id,
data.as_ptr(), data.len() as u32);
r
},
};
if res != 0 {
panic!("c.fn_fileappenddata failed");
}

98
rust/src/filetracker.rs
Unescape Escape View File

@ -15,7 +15,19 @@
* 02110-1301, USA.
*/
// written by Victor Julien
/**
* \file
* \author Victor Julien <victor@inliniac.net>
*
* Tracks chunk based file transfers. Chunks may be transfered out
* of order, but cannot be transfered in parallel. So only one
* chunk at a time.
*
* GAP handling. If a data gap is encountered, the file is truncated
* and new data is no longer pushed down to the lower level APIs.
* The tracker does continue to follow the file.
*/
extern crate libc;
use log::*;
use core::*;
@ -23,10 +35,26 @@ use std::collections::HashMap;
use std::collections::hash_map::Entry::{Occupied, Vacant};
use filecontainer::*;
#[derive(Debug)]
pub struct FileChunk {
contains_gap: bool,
chunk: Vec<u8>,
}
impl FileChunk {
pub fn new(size: u32) -> FileChunk {
FileChunk {
contains_gap: false,
chunk: Vec::with_capacity(size as usize),
}
}
}
#[derive(Debug)]
pub struct FileTransferTracker {
file_size: u64,
tracked: u64,
cur_ooo: u64, // how many bytes do we have queued from ooo chunks
track_id: u32,
chunk_left: u32,
@ -36,8 +64,9 @@ pub struct FileTransferTracker {
pub file_open: bool,
chunk_is_last: bool,
chunk_is_ooo: bool,
file_is_truncated: bool,
chunks: HashMap<u64, Vec<u8>>,
chunks: HashMap<u64, FileChunk>,
cur_ooo_chunk_offset: u64,
}
@ -46,6 +75,7 @@ impl FileTransferTracker {
FileTransferTracker {
file_size:0,
tracked:0,
cur_ooo:0,
track_id:0,
chunk_left:0,
tx_id:0,
@ -53,6 +83,7 @@ impl FileTransferTracker {
file_open:false,
chunk_is_last:false,
chunk_is_ooo:false,
file_is_truncated:false,
cur_ooo_chunk_offset:0,
chunks:HashMap::new(),
}
@ -70,12 +101,25 @@ impl FileTransferTracker {
}
pub fn close(&mut self, files: &mut FileContainer, flags: u16) {
if !self.file_is_truncated {
files.file_close(&self.track_id, flags);
}
self.file_open = false;
self.tracked = 0;
files.files_prune();
}
pub fn trunc (&mut self, files: &mut FileContainer, flags: u16) {
if self.file_is_truncated {
return;
}
let myflags = flags | 1; // TODO util-file.c::FILE_TRUNCATED
files.file_close(&self.track_id, myflags);
SCLogDebug!("truncated file");
files.files_prune();
self.file_is_truncated = true;
}
pub fn create(&mut self, name: &[u8], file_size: u64) {
if self.file_open == true { panic!("close existing file first"); }
@ -92,8 +136,15 @@ impl FileTransferTracker {
SCLogDebug!("NEW CHUNK: chunk_size {} fill_bytes {}", chunk_size, fill_bytes);
// for now assume that is_last means its really the last chunk
// so no out of order chunks coming after. This means that if
// the last chunk is out or order, we've missed chunks before.
if chunk_offset != self.tracked {
SCLogDebug!("NEW CHUNK IS OOO: expected {}, got {}", self.tracked, chunk_offset);
if is_last {
SCLogDebug!("last chunk is out of order, this means we missed data before");
self.trunc(files, flags);
}
self.chunk_is_ooo = true;
self.cur_ooo_chunk_offset = chunk_offset;
}
@ -108,14 +159,21 @@ impl FileTransferTracker {
self.open(config, files, flags, name);
}
let res = self.update(files, flags, data);
let res = self.update(files, flags, data, 0);
SCLogDebug!("NEW CHUNK: update res {:?}", res);
res
}
/// update the file tracker
/// If gap_size > 0 'data' should not be used.
/// return how much we consumed of data
pub fn update(&mut self, files: &mut FileContainer, flags: u16, data: &[u8]) -> u32 {
pub fn update(&mut self, files: &mut FileContainer, flags: u16, data: &[u8], gap_size: u32) -> u32 {
let mut consumed = 0 as usize;
let is_gap = gap_size > 0;
if is_gap || gap_size > 0 {
SCLogDebug!("is_gap {} size {} ooo? {}", is_gap, gap_size, self.chunk_is_ooo);
}
if self.chunk_left + self.fill_bytes as u32 == 0 {
//SCLogDebug!("UPDATE: nothing to do");
return 0
@ -140,7 +198,7 @@ impl FileTransferTracker {
let d = &data[0..self.chunk_left as usize];
if self.chunk_is_ooo == false {
let res = files.file_append(&self.track_id, d);
let res = files.file_append(&self.track_id, d, is_gap);
if res != 0 { panic!("append failed"); }
self.tracked += self.chunk_left as u64;
@ -149,11 +207,13 @@ impl FileTransferTracker {
d.len(), self.cur_ooo_chunk_offset, self.tracked);
let c = match self.chunks.entry(self.cur_ooo_chunk_offset) {
Vacant(entry) => {
entry.insert(Vec::with_capacity(self.chunk_left as usize))
entry.insert(FileChunk::new(self.chunk_left))
},
Occupied(entry) => entry.into_mut(),
};
c.extend(d);
self.cur_ooo += d.len() as u64;
c.contains_gap |= is_gap;
c.chunk.extend(d);
}
consumed += self.chunk_left as usize;
@ -169,7 +229,6 @@ impl FileTransferTracker {
SCLogDebug!("CHUNK(post) fill bytes now still {}", self.fill_bytes);
}
self.chunk_left = 0;
//return consumed as u32
} else {
self.chunk_left = 0;
@ -177,13 +236,14 @@ impl FileTransferTracker {
loop {
let offset = self.tracked;
match self.chunks.remove(&self.tracked) {
Some(a) => {
let res = files.file_append(&self.track_id, &a);
if res != 0 { panic!("append failed"); }
Some(c) => {
let res = files.file_append(&self.track_id, &c.chunk, c.contains_gap);
if res != 0 { panic!("append failed: files.file_append() returned {}", res); }
self.tracked += a.len() as u64;
self.tracked += c.chunk.len() as u64;
self.cur_ooo -= c.chunk.len() as u64;
SCLogDebug!("STORED OOO CHUNK at offset {}, tracked now {}, stored len {}", offset, self.tracked, a.len());
SCLogDebug!("STORED OOO CHUNK at offset {}, tracked now {}, stored len {}", offset, self.tracked, c.chunk.len());
},
_ => {
SCLogDebug!("NO STORED CHUNK found at offset {}", self.tracked);
@ -208,15 +268,17 @@ impl FileTransferTracker {
} else {
if self.chunk_is_ooo == false {
let res = files.file_append(&self.track_id, data);
let res = files.file_append(&self.track_id, data, is_gap);
if res != 0 { panic!("append failed"); }
self.tracked += data.len() as u64;
} else {
let c = match self.chunks.entry(self.cur_ooo_chunk_offset) {
Vacant(entry) => entry.insert(Vec::with_capacity(32768)),
Vacant(entry) => entry.insert(FileChunk::new(32768)),
Occupied(entry) => entry.into_mut(),
};
c.extend(data);
c.chunk.extend(data);
c.contains_gap |= is_gap;
self.cur_ooo += data.len() as u64;
}
self.chunk_left -= data.len() as u32;
@ -226,4 +288,8 @@ impl FileTransferTracker {
files.files_prune();
consumed as u32
}
pub fn get_queued_size(&self) -> u64 {
self.cur_ooo
}
}

83
rust/src/nfs/nfs3.rs
Unescape Escape View File

@ -283,6 +283,9 @@ pub struct NFS3State {
ts_chunk_left: u32,
tc_chunk_left: u32,
ts_ssn_gap: bool,
tc_ssn_gap: bool,
/// tx counter for assigning incrementing id's to tx's
tx_id: u64,
@ -308,6 +311,8 @@ impl NFS3State {
tc_chunk_xid:0,
ts_chunk_left:0,
tc_chunk_left:0,
ts_ssn_gap:false,
tc_ssn_gap:false,
tx_id:0,
de_state_count:0,
//ts_txs_updated:false,
@ -701,7 +706,14 @@ impl NFS3State {
let xidmap;
match self.requestmap.remove(&r.hdr.xid) {
Some(p) => { xidmap = p; },
_ => { SCLogDebug!("REPLY: xid {} NOT FOUND", r.hdr.xid); return 0; },
_ => {
SCLogDebug!("REPLY: xid {} NOT FOUND. GAPS? TS:{} TC:{}",
r.hdr.xid, self.ts_ssn_gap, self.tc_ssn_gap);
// TODO we might be able to try to infer from the size + data
// that this is a READ reply and pass the data to the file API anyway?
return 0;
},
}
if xidmap.procedure == NFSPROC3_LOOKUP {
@ -809,7 +821,7 @@ impl NFS3State {
// update in progress chunks for file transfers
// return how much data we consumed
fn filetracker_update(&mut self, direction: u8, data: &[u8]) -> u32 {
fn filetracker_update(&mut self, direction: u8, data: &[u8], gap_size: u32) -> u32 {
let mut chunk_left = if direction == STREAM_TOSERVER {
self.ts_chunk_left
} else {
@ -877,6 +889,7 @@ impl NFS3State {
self.tc_chunk_left = chunk_left;
}
let ssn_gap = self.ts_ssn_gap | self.tc_ssn_gap;
// get the tx and update it
let consumed = match self.get_file_tx_by_handle(&file_handle, direction) {
Some((tx, files, flags)) => {
@ -884,7 +897,15 @@ impl NFS3State {
Some(NFS3TransactionTypeData::FILE(ref mut x)) => x,
_ => { panic!("BUG") },
};
let cs = tdf.file_tracker.update(files, flags, data);
if ssn_gap {
let queued_data = tdf.file_tracker.get_queued_size();
if queued_data > 2000000 { // TODO should probably be configurable
SCLogDebug!("QUEUED size {} while we've seen GAPs. Truncating file.", queued_data);
tdf.file_tracker.trunc(files, flags);
}
}
let cs = tdf.file_tracker.update(files, flags, data, gap_size);
cs
},
None => { 0 },
@ -993,6 +1014,32 @@ impl NFS3State {
xidmap.procedure
}
pub fn parse_tcp_data_ts_gap<'b>(&mut self, gap_size: u32) -> u32 {
if self.tcp_buffer_ts.len() > 0 {
self.tcp_buffer_ts.clear();
}
let gap = vec![0; gap_size as usize];
let consumed = self.filetracker_update(STREAM_TOSERVER, &gap, gap_size);
if consumed > gap_size {
panic!("consumed more than GAP size: {} > {}", consumed, gap_size);
}
self.ts_ssn_gap = true;
return 0
}
pub fn parse_tcp_data_tc_gap<'b>(&mut self, gap_size: u32) -> u32 {
if self.tcp_buffer_tc.len() > 0 {
self.tcp_buffer_tc.clear();
}
let gap = vec![0; gap_size as usize];
let consumed = self.filetracker_update(STREAM_TOCLIENT, &gap, gap_size);
if consumed > gap_size {
panic!("consumed more than GAP size: {} > {}", consumed, gap_size);
}
self.tc_ssn_gap = true;
return 0
}
/// Parsing function, handling TCP chunks fragmentation
pub fn parse_tcp_data_ts<'b>(&mut self, i: &'b[u8]) -> u32 {
let mut v : Vec<u8>;
@ -1006,7 +1053,7 @@ impl NFS3State {
v = self.tcp_buffer_ts.split_off(0);
// sanity check vector length to avoid memory exhaustion
if self.tcp_buffer_ts.len() + i.len() > 1000000 {
SCLogNotice!("parse_tcp_data_ts: TS buffer exploded {} {}",
SCLogDebug!("parse_tcp_data_ts: TS buffer exploded {} {}",
self.tcp_buffer_ts.len(), i.len());
return 1;
};
@ -1017,17 +1064,15 @@ impl NFS3State {
//SCLogDebug!("tcp_buffer ({})",tcp_buffer.len());
let mut cur_i = tcp_buffer;
if cur_i.len() > 1000000 {
SCLogNotice!("BUG buffer exploded: {}", cur_i.len());
SCLogDebug!("BUG buffer exploded: {}", cur_i.len());
}
// take care of in progress file chunk transfers
// and skip buffer beyond it
let consumed = self.filetracker_update(STREAM_TOSERVER, cur_i);
let consumed = self.filetracker_update(STREAM_TOSERVER, cur_i, 0);
if consumed > 0 {
if consumed > cur_i.len() as u32 { panic!("BUG consumed more than we gave it"); }
cur_i = &cur_i[consumed as usize..];
}
while cur_i.len() > 0 { // min record size
match parse_rpc_request_partial(cur_i) {
IResult::Done(_, ref rpc_phdr) => {
@ -1122,6 +1167,7 @@ impl NFS3State {
SCLogDebug!("TC buffer exploded");
return 1;
};
v.extend_from_slice(i);
v.as_slice()
},
@ -1130,17 +1176,16 @@ impl NFS3State {
let mut cur_i = tcp_buffer;
if cur_i.len() > 100000 {
SCLogNotice!("parse_tcp_data_tc: BUG buffer exploded {}", cur_i.len());
SCLogDebug!("parse_tcp_data_tc: BUG buffer exploded {}", cur_i.len());
}
// take care of in progress file chunk transfers
// and skip buffer beyond it
let consumed = self.filetracker_update(STREAM_TOCLIENT, cur_i);
let consumed = self.filetracker_update(STREAM_TOCLIENT, cur_i, 0);
if consumed > 0 {
if consumed > cur_i.len() as u32 { panic!("BUG consumed more than we gave it"); }
cur_i = &cur_i[consumed as usize..];
}
while cur_i.len() > 0 {
match parse_rpc_packet_header(cur_i) {
IResult::Done(_, ref rpc_hdr) => {
@ -1260,6 +1305,14 @@ pub extern "C" fn rs_nfs3_parse_request(_flow: *mut Flow,
{
let buf = unsafe{std::slice::from_raw_parts(input, input_len as usize)};
SCLogDebug!("parsing {} bytes of request data", input_len);
if buf.as_ptr().is_null() && input_len > 0 {
if state.parse_tcp_data_ts_gap(input_len as u32) == 0 {
return 1
}
return -1
}
if state.parse_tcp_data_ts(buf) == 0 {
1
} else {
@ -1278,6 +1331,14 @@ pub extern "C" fn rs_nfs3_parse_response(_flow: *mut Flow,
{
SCLogDebug!("parsing {} bytes of response data", input_len);
let buf = unsafe{std::slice::from_raw_parts(input, input_len as usize)};
if buf.as_ptr().is_null() && input_len > 0 {
if state.parse_tcp_data_tc_gap(input_len as u32) == 0 {
return 1
}
return -1
}
if state.parse_tcp_data_tc(buf) == 0 {
1
} else {

4
src/app-layer-nfs3.c
Unescape Escape View File

@ -346,6 +346,10 @@ void RegisterNFS3Parsers(void)
// NFS3StateGetEventInfo);
// AppLayerParserRegisterGetEventsFunc(IPPROTO_TCP, ALPROTO_NFS3,
// NFS3GetEvents);
/* This parser accepts gaps. */
AppLayerParserRegisterOptionFlags(IPPROTO_TCP, ALPROTO_NFS3,
APP_LAYER_PARSER_OPT_ACCEPT_GAPS);
}
else {
SCLogDebug("NFS3 protocol parsing disabled.");

1
src/output-json-file.c
Unescape Escape View File

@ -124,6 +124,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F
if (ff->magic)
json_object_set_new(fjs, "magic", json_string((char *)ff->magic));
#endif
json_object_set_new(fjs, "gaps", json_boolean((ff->flags & FILE_HAS_GAPS)));
switch (ff->state) {
case FILE_STATE_CLOSED:
json_object_set_new(fjs, "state", json_string("CLOSED"));

2
src/rust.h
Unescape Escape View File

@ -33,6 +33,8 @@ typedef struct SuricataContext_ {
const uint8_t *data, uint32_t data_len, uint16_t flags);
int (*FileAppendDataById)(FileContainer *, uint32_t track_id,
const uint8_t *data, uint32_t data_len);
int (*FileAppendGAPById)(FileContainer *, uint32_t track_id,
const uint8_t *data, uint32_t data_len);
void (*FileContainerRecycle)(FileContainer *ffc);
void (*FilePrune)(FileContainer *ffc);
void (*FileSetTx)(FileContainer *, uint64_t);

1
src/suricata.c
Unescape Escape View File

@ -2795,6 +2795,7 @@ int main(int argc, char **argv)
context.FileOpenFileWithId = FileOpenFileWithId;
context.FileCloseFileById = FileCloseFileById;
context.FileAppendDataById = FileAppendDataById;
context.FileAppendGAPById = FileAppendGAPById;
context.FileContainerRecycle = FileContainerRecycle;
context.FilePrune = FilePrune;
context.FileSetTx = FileContainerSetTx;

37
src/util-file.c
Unescape Escape View File

@ -689,6 +689,41 @@ int FileAppendDataById(FileContainer *ffc, uint32_t track_id,
SCReturnInt(-1);
}
/**
* \brief Store/handle a chunk of file data in the File structure
* The file with 'track_id' in the FileContainer will be used.
*
* \param ffc FileContainer used to append to
* \param track_id id to lookup the file
* \param data data chunk
* \param data_len data chunk len
*
* \retval 0 ok
* \retval -1 error
* \retval -2 no store for this file
*/
int FileAppendGAPById(FileContainer *ffc, uint32_t track_id,
const uint8_t *data, uint32_t data_len)
{
SCEnter();
if (ffc == NULL || ffc->tail == NULL || data == NULL || data_len == 0) {
SCReturnInt(-1);
}
File *ff = ffc->head;
for ( ; ff != NULL; ff = ff->next) {
if (track_id == ff->file_track_id) {
ff->flags |= FILE_HAS_GAPS;
ff->flags |= (FILE_NOMD5|FILE_NOSHA1|FILE_NOSHA256);
ff->flags &= ~(FILE_MD5|FILE_SHA1|FILE_SHA256);
SCLogDebug("FILE_HAS_GAPS set");
int r = FileAppendDataDo(ff, data, data_len);
SCReturnInt(r);
}
}
SCReturnInt(-1);
}
/**
* \brief Open a new File
@ -837,7 +872,7 @@ static int FileCloseFilePtr(File *ff, const uint8_t *data,
}
}
if (flags & FILE_TRUNCATED) {
if ((flags & FILE_TRUNCATED) || (ff->flags & FILE_HAS_GAPS)) {
ff->state = FILE_STATE_TRUNCATED;
SCLogDebug("flowfile state transitioned to FILE_STATE_TRUNCATED");

3
src/util-file.h
Unescape Escape View File

@ -48,6 +48,7 @@
#define FILE_NOTRACK BIT_U16(12) /**< track size of file */
#define FILE_USE_DETECT BIT_U16(13) /**< use content_inspected tracker */
#define FILE_USE_TRACKID BIT_U16(14) /**< File::file_track_id field is in use */
#define FILE_HAS_GAPS BIT_U16(15)
typedef enum FileState_ {
FILE_STATE_NONE = 0, /**< no state */
@ -159,6 +160,8 @@ int FileCloseFileById(FileContainer *, uint32_t track_id,
int FileAppendData(FileContainer *, const uint8_t *data, uint32_t data_len);
int FileAppendDataById(FileContainer *, uint32_t track_id,
const uint8_t *data, uint32_t data_len);
int FileAppendGAPById(FileContainer *ffc, uint32_t track_id,
const uint8_t *data, uint32_t data_len);
/**
* \brief Tag a file for storing

Write Preview
Loading…
Cancel
Save