aiden
/
suricata
mirror of https://github.com/OISF/suricata
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

smtp: use rust for mime parsing

Browse Source 
Ticket: #3487
pull/11227/head
Philippe Antoine 3 years ago committed by Victor Julien
parent 5f75b9a6e3
commit a10c1f1dde
16 changed files with 1861 additions and 4941 deletions
Show Stats Download Patch File Download Diff File

569
rust/src/mime/mime.rs
Unescape Escape View File

@ -0,0 +1,569 @@
/* Copyright (C) 2024 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use crate::common::nom7::take_until_and_consume;
use nom7::branch::alt;
use nom7::bytes::complete::{tag, take, take_till, take_until, take_while};
use nom7::character::complete::char;
use nom7::combinator::{complete, opt, rest, value};
use nom7::error::{make_error, ErrorKind};
use nom7::{Err, IResult};
use std;
use std::collections::HashMap;
#[derive(Clone)]
pub struct HeaderTokens<'a> {
pub tokens: HashMap<&'a [u8], &'a [u8]>,
}
fn mime_parse_value_delimited(input: &[u8]) -> IResult<&[u8], &[u8]> {
let (input, _) = char('"')(input)?;
let mut escaping = false;
for i in 0..input.len() {
if input[i] == b'\\' {
escaping = true;
} else {
if input[i] == b'"' && !escaping {
return Ok((&input[i + 1..], &input[..i]));
}
// unescape can be processed later
escaping = false;
}
}
// should fail
let (input, value) = take_until("\"")(input)?;
let (input, _) = char('"')(input)?;
return Ok((input, value));
}
fn mime_parse_value_until_semicolon(input: &[u8]) -> IResult<&[u8], &[u8]> {
let (input, value) = alt((take_till(|ch: u8| ch == b';'), rest))(input)?;
for i in 0..value.len() {
if !is_mime_space(value[value.len() - i - 1]) {
return Ok((input, &value[..value.len() - i]));
}
}
return Ok((input, value));
}
#[inline]
fn is_mime_space(ch: u8) -> bool {
ch == 0x20 || ch == 0x09 || ch == 0x0a || ch == 0x0d
}
pub fn mime_parse_header_token(input: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> {
// from RFC2047 : like ch.is_ascii_whitespace but without 0x0c FORM-FEED
let (input, _) = take_while(is_mime_space)(input)?;
let (input, name) = take_until("=")(input)?;
let (input, _) = char('=')(input)?;
let (input, value) =
alt((mime_parse_value_delimited, mime_parse_value_until_semicolon))(input)?;
let (input, _) = take_while(is_mime_space)(input)?;
let (input, _) = opt(complete(char(';')))(input)?;
return Ok((input, (name, value)));
}
fn mime_parse_header_tokens(input: &[u8]) -> IResult<&[u8], HeaderTokens> {
let (mut input, _) = take_until_and_consume(b";")(input)?;
let mut tokens = HashMap::new();
while !input.is_empty() {
match mime_parse_header_token(input) {
Ok((rem, t)) => {
tokens.insert(t.0, t.1);
// should never happen
debug_validate_bug_on!(input.len() == rem.len());
if input.len() == rem.len() {
//infinite loop
return Err(Err::Error(make_error(input, ErrorKind::Eof)));
}
input = rem;
}
Err(_) => {
// keep first tokens is error in remaining buffer
break;
}
}
}
return Ok((input, HeaderTokens { tokens }));
}
pub fn mime_find_header_token<'a>(
header: &'a [u8], token: &[u8], sections_values: &'a mut Vec<u8>,
) -> Option<&'a [u8]> {
match mime_parse_header_tokens(header) {
Ok((_rem, t)) => {
// in case of multiple sections for the parameter cf RFC2231
let mut current_section_slice = Vec::new();
// look for the specific token
match t.tokens.get(token) {
// easy nominal case
Some(value) => return Some(value),
None => {
// check for initial section of a parameter
current_section_slice.extend_from_slice(token);
current_section_slice.extend_from_slice(b"*0");
match t.tokens.get(&current_section_slice[..]) {
Some(value) => {
sections_values.extend_from_slice(value);
let l = current_section_slice.len();
current_section_slice[l - 1] = b'1';
}
None => return None,
}
}
}
let mut current_section_seen = 1;
// we have at least the initial section
// try looping until we do not find anymore a next section
loop {
match t.tokens.get(&current_section_slice[..]) {
Some(value) => {
sections_values.extend_from_slice(value);
current_section_seen += 1;
let nbdigits = current_section_slice.len() - token.len() - 1;
current_section_slice.truncate(current_section_slice.len() - nbdigits);
current_section_slice
.extend_from_slice(current_section_seen.to_string().as_bytes());
}
None => return Some(sections_values),
}
}
}
Err(_) => {
return None;
}
}
}
pub(crate) const RS_MIME_MAX_TOKEN_LEN: usize = 255;
#[derive(Debug)]
enum MimeParserState {
Start,
Header,
HeaderEnd,
Chunk,
BoundaryWaitingForEol,
}
impl Default for MimeParserState {
fn default() -> Self {
MimeParserState::Start
}
}
#[derive(Debug, Default)]
pub struct MimeStateHTTP {
boundary: Vec<u8>,
filename: Vec<u8>,
state: MimeParserState,
}
#[repr(u8)]
#[derive(Copy, Clone, PartialOrd, PartialEq, Eq)]
pub enum MimeParserResult {
MimeNeedsMore = 0,
MimeFileOpen = 1,
MimeFileChunk = 2,
MimeFileClose = 3,
}
fn mime_parse_skip_line(input: &[u8]) -> IResult<&[u8], MimeParserState> {
let (input, _) = take_till(|ch: u8| ch == b'\n')(input)?;
let (input, _) = char('\n')(input)?;
return Ok((input, MimeParserState::Start));
}
fn mime_parse_boundary_regular<'a>(
boundary: &[u8], input: &'a [u8],
) -> IResult<&'a [u8], MimeParserState> {
let (input, _) = tag(boundary)(input)?;
let (input, _) = take_till(|ch: u8| ch == b'\n')(input)?;
let (input, _) = char('\n')(input)?;
return Ok((input, MimeParserState::Header));
}
// Number of characters after boundary, without end of line, before changing state to streaming
const MIME_BOUNDARY_MAX_BEFORE_EOL: usize = 128;
const MIME_HEADER_MAX_LINE: usize = 4096;
fn mime_parse_boundary_missing_eol<'a>(
boundary: &[u8], input: &'a [u8],
) -> IResult<&'a [u8], MimeParserState> {
let (input, _) = tag(boundary)(input)?;
let (input, _) = take(MIME_BOUNDARY_MAX_BEFORE_EOL)(input)?;
return Ok((input, MimeParserState::BoundaryWaitingForEol));
}
fn mime_parse_boundary<'a>(boundary: &[u8], input: &'a [u8]) -> IResult<&'a [u8], MimeParserState> {
let r = mime_parse_boundary_regular(boundary, input);
if r.is_ok() {
return r;
}
let r2 = mime_parse_skip_line(input);
if r2.is_ok() {
return r2;
}
return mime_parse_boundary_missing_eol(boundary, input);
}
fn mime_consume_until_eol(input: &[u8]) -> IResult<&[u8], bool> {
return alt((value(true, mime_parse_skip_line), value(false, rest)))(input);
}
pub fn mime_parse_header_line(input: &[u8]) -> IResult<&[u8], &[u8]> {
let (input, name) = take_till(|ch: u8| ch == b':')(input)?;
let (input, _) = char(':')(input)?;
let (input, _) = take_while(is_mime_space)(input)?;
return Ok((input, name));
}
// s2 is already lower case
pub fn slice_equals_lowercase(s1: &[u8], s2: &[u8]) -> bool {
if s1.len() == s2.len() {
for i in 0..s1.len() {
if s1[i].to_ascii_lowercase() != s2[i] {
return false;
}
}
return true;
}
return false;
}
fn mime_parse_headers<'a>(
ctx: &mut MimeStateHTTP, i: &'a [u8],
) -> IResult<&'a [u8], (MimeParserState, bool, bool)> {
let mut fileopen = false;
let mut errored = false;
let mut input = i;
while !input.is_empty() {
if let Ok((input2, line)) = take_until::<_, &[u8], nom7::error::Error<&[u8]>>("\r\n")(input)
{
if let Ok((value, name)) = mime_parse_header_line(line) {
if slice_equals_lowercase(name, "content-disposition".as_bytes()) {
let mut sections_values = Vec::new();
if let Some(filename) =
mime_find_header_token(value, "filename".as_bytes(), &mut sections_values)
{
if !filename.is_empty() {
ctx.filename = Vec::with_capacity(filename.len());
fileopen = true;
for c in filename {
// unescape
if *c != b'\\' {
ctx.filename.push(*c);
}
}
}
}
}
if value.is_empty() {
errored = true;
}
} else if !line.is_empty() {
errored = true;
}
let (input3, _) = tag("\r\n")(input2)?;
input = input3;
if line.is_empty() || (line.len() == 1 && line[0] == b'\r') {
return Ok((input, (MimeParserState::HeaderEnd, fileopen, errored)));
}
} else {
// guard against too long header lines
if input.len() > MIME_HEADER_MAX_LINE {
return Ok((
input,
(MimeParserState::BoundaryWaitingForEol, fileopen, errored),
));
}
if input.len() < i.len() {
return Ok((input, (MimeParserState::Header, fileopen, errored)));
} // else only an incomplete line, ask for more
return Err(Err::Error(make_error(input, ErrorKind::Eof)));
}
}
return Ok((input, (MimeParserState::Header, fileopen, errored)));
}
type NomTakeError<'a> = Err<nom7::error::Error<&'a [u8]>>;
fn mime_consume_chunk<'a>(boundary: &[u8], input: &'a [u8]) -> IResult<&'a [u8], bool> {
let r: Result<(&[u8], &[u8]), NomTakeError> = take_until("\r\n")(input);
if let Ok((input, line)) = r {
let (next_line, _) = tag("\r\n")(input)?;
if next_line.len() < boundary.len() {
if next_line == &boundary[..next_line.len()] {
if !line.is_empty() {
// consume as chunk up to eol (not consuming eol)
return Ok((input, false));
}
// new line beignning like boundary, with nothin to consume as chunk : request more
return Err(Err::Error(make_error(input, ErrorKind::Eof)));
}
// not like boundary : consume everything as chunk
return Ok((&input[input.len()..], false));
} // else
if &next_line[..boundary.len()] == boundary {
// end of file with boundary, consume eol but do not consume boundary
return Ok((next_line, true));
}
// not like boundary : consume everything as chunk
return Ok((next_line, false));
} else {
return Ok((&input[input.len()..], false));
}
}
pub const MIME_EVENT_FLAG_INVALID_HEADER: u32 = 0x01;
pub const MIME_EVENT_FLAG_NO_FILEDATA: u32 = 0x02;
fn mime_process(ctx: &mut MimeStateHTTP, i: &[u8]) -> (MimeParserResult, u32, u32) {
let mut input = i;
let mut consumed = 0;
let mut warnings = 0;
while !input.is_empty() {
match ctx.state {
MimeParserState::Start => {
if let Ok((rem, next)) = mime_parse_boundary(&ctx.boundary, input) {
ctx.state = next;
consumed += (input.len() - rem.len()) as u32;
input = rem;
} else {
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
}
MimeParserState::BoundaryWaitingForEol => {
if let Ok((rem, found)) = mime_consume_until_eol(input) {
if found {
ctx.state = MimeParserState::Header;
}
consumed += (input.len() - rem.len()) as u32;
input = rem;
} else {
// should never happen
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
}
MimeParserState::Header => {
if let Ok((rem, (next, fileopen, err))) = mime_parse_headers(ctx, input) {
ctx.state = next;
consumed += (input.len() - rem.len()) as u32;
input = rem;
if err {
warnings |= MIME_EVENT_FLAG_INVALID_HEADER;
}
if fileopen {
return (MimeParserResult::MimeFileOpen, consumed, warnings);
}
} else {
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
}
MimeParserState::HeaderEnd => {
// check if we start with the boundary
// and transition to chunk, or empty file and back to start
if input.len() < ctx.boundary.len() {
if input == &ctx.boundary[..input.len()] {
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
ctx.state = MimeParserState::Chunk;
} else if input[..ctx.boundary.len()] == ctx.boundary {
ctx.state = MimeParserState::Start;
if !ctx.filename.is_empty() {
warnings |= MIME_EVENT_FLAG_NO_FILEDATA;
}
ctx.filename.clear();
return (MimeParserResult::MimeFileClose, consumed, warnings);
} else {
ctx.state = MimeParserState::Chunk;
}
}
MimeParserState::Chunk => {
if let Ok((rem, eof)) = mime_consume_chunk(&ctx.boundary, input) {
consumed += (input.len() - rem.len()) as u32;
if eof {
ctx.state = MimeParserState::Start;
ctx.filename.clear();
return (MimeParserResult::MimeFileClose, consumed, warnings);
} else {
// + 2 for \r\n
if rem.len() < ctx.boundary.len() + 2 {
return (MimeParserResult::MimeFileChunk, consumed, warnings);
}
input = rem;
}
} else {
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
}
}
}
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
pub fn mime_state_init(i: &[u8]) -> Option<MimeStateHTTP> {
let mut sections_values = Vec::new();
if let Some(value) = mime_find_header_token(i, "boundary".as_bytes(), &mut sections_values) {
if value.len() <= RS_MIME_MAX_TOKEN_LEN {
let mut r = MimeStateHTTP {
boundary: Vec::with_capacity(2 + value.len()),
..Default::default()
};
// start wih 2 additional hyphens
r.boundary.push(b'-');
r.boundary.push(b'-');
for c in value {
// unescape
if *c != b'\\' {
r.boundary.push(*c);
}
}
return Some(r);
}
}
return None;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeStateInit(input: *const u8, input_len: u32) -> *mut MimeStateHTTP {
let slice = build_slice!(input, input_len as usize);
if let Some(ctx) = mime_state_init(slice) {
let boxed = Box::new(ctx);
return Box::into_raw(boxed) as *mut _;
}
return std::ptr::null_mut();
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeParse(
ctx: &mut MimeStateHTTP, input: *const u8, input_len: u32, consumed: *mut u32,
warnings: *mut u32,
) -> MimeParserResult {
let slice = build_slice!(input, input_len as usize);
let (r, c, w) = mime_process(ctx, slice);
*consumed = c;
*warnings = w;
return r;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeStateGetFilename(
ctx: &mut MimeStateHTTP, buffer: *mut *const u8, filename_len: *mut u16,
) {
if !ctx.filename.is_empty() {
*buffer = ctx.filename.as_ptr();
if ctx.filename.len() < u16::MAX.into() {
*filename_len = ctx.filename.len() as u16;
} else {
*filename_len = u16::MAX;
}
} else {
*buffer = std::ptr::null_mut();
*filename_len = 0;
}
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeStateFree(ctx: &mut MimeStateHTTP) {
std::mem::drop(Box::from_raw(ctx));
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_mime_find_header_token() {
let mut outvec = Vec::new();
let undelimok = mime_find_header_token(
"attachment; filename=test;".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(undelimok, Some("test".as_bytes()));
let delimok = mime_find_header_token(
"attachment; filename=\"test2\";".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(delimok, Some("test2".as_bytes()));
let escaped = mime_find_header_token(
"attachment; filename=\"test\\\"2\";".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(escaped, Some("test\\\"2".as_bytes()));
let evasion_othertoken = mime_find_header_token(
"attachment; dummy=\"filename=wrong\"; filename=real;".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(evasion_othertoken, Some("real".as_bytes()));
let evasion_suffixtoken = mime_find_header_token(
"attachment; notafilename=wrong; filename=good;".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(evasion_suffixtoken, Some("good".as_bytes()));
let badending = mime_find_header_token(
"attachment; filename=oksofar; badending".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(badending, Some("oksofar".as_bytes()));
let missend = mime_find_header_token(
"attachment; filename=test".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(missend, Some("test".as_bytes()));
let spaces = mime_find_header_token(
"attachment; filename=test me wrong".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(spaces, Some("test me wrong".as_bytes()));
assert_eq!(outvec.len(), 0);
let multi = mime_find_header_token(
"attachment; filename*0=abc; filename*1=\"def\";".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(multi, Some("abcdef".as_bytes()));
outvec.clear();
let multi = mime_find_header_token(
"attachment; filename*1=456; filename*0=\"123\"".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(multi, Some("123456".as_bytes()));
outvec.clear();
}
}

599
rust/src/mime/mod.rs
Unescape Escape View File

@ -1,4 +1,4 @@
/* Copyright (C) 2021 Open Information Security Foundation
/* Copyright (C) 2021-2024 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
@ -17,597 +17,6 @@
//! MIME protocol parser module.
use crate::common::nom7::take_until_and_consume;
use nom7::branch::alt;
use nom7::bytes::complete::{tag, take, take_till, take_until, take_while};
use nom7::character::complete::char;
use nom7::combinator::{complete, opt, rest, value};
use nom7::error::{make_error, ErrorKind};
use nom7::{Err, IResult};
use std;
use std::collections::HashMap;
#[derive(Clone)]
pub struct MIMEHeaderTokens<'a> {
pub tokens: HashMap<&'a [u8], &'a [u8]>,
}
fn mime_parse_value_delimited(input: &[u8]) -> IResult<&[u8], &[u8]> {
let (input, _) = char('"')(input)?;
let mut escaping = false;
for i in 0..input.len() {
if input[i] == b'\\' {
escaping = true;
} else {
if input[i] == b'"' && !escaping {
return Ok((&input[i + 1..], &input[..i]));
}
// unescape can be processed later
escaping = false;
}
}
// should fail
let (input, value) = take_until("\"")(input)?;
let (input, _) = char('"')(input)?;
return Ok((input, value));
}
fn mime_parse_value_until(input: &[u8]) -> IResult<&[u8], &[u8]> {
let (input, value) = alt((take_till(|ch: u8| ch == b';'), rest))(input)?;
for i in 0..value.len() {
if !is_mime_space(value[value.len() - i - 1]) {
return Ok((input, &value[..value.len() - i]));
}
}
return Ok((input, value));
}
#[inline]
fn is_mime_space(ch: u8) -> bool {
ch == 0x20 || ch == 0x09 || ch == 0x0a || ch == 0x0d
}
pub fn mime_parse_header_token(input: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> {
// from RFC2047 : like ch.is_ascii_whitespace but without 0x0c FORM-FEED
let (input, _) = take_while(|ch: u8| is_mime_space(ch))(input)?;
let (input, name) = take_until("=")(input)?;
let (input, _) = char('=')(input)?;
let (input, value) = alt((mime_parse_value_delimited, mime_parse_value_until))(input)?;
let (input, _) = take_while(|ch: u8| is_mime_space(ch))(input)?;
let (input, _) = opt(complete(char(';')))(input)?;
return Ok((input, (name, value)));
}
fn mime_parse_header_tokens(input: &[u8]) -> IResult<&[u8], MIMEHeaderTokens> {
let (mut input, _) = take_until_and_consume(b";")(input)?;
let mut tokens = HashMap::new();
while !input.is_empty() {
match mime_parse_header_token(input) {
Ok((rem, t)) => {
tokens.insert(t.0, t.1);
// should never happen
debug_validate_bug_on!(input.len() == rem.len());
if input.len() == rem.len() {
//infinite loop
return Err(Err::Error(make_error(input, ErrorKind::Eof)));
}
input = rem;
}
Err(_) => {
// keep first tokens is error in remaining buffer
break;
}
}
}
return Ok((input, MIMEHeaderTokens { tokens }));
}
fn mime_find_header_token<'a>(
header: &'a [u8], token: &[u8], sections_values: &'a mut Vec<u8>,
) -> Result<&'a [u8], ()> {
match mime_parse_header_tokens(header) {
Ok((_rem, t)) => {
// in case of multiple sections for the parameter cf RFC2231
let mut current_section_slice = Vec::new();
// look for the specific token
match t.tokens.get(token) {
// easy nominal case
Some(value) => return Ok(value),
None => {
// check for initial section of a parameter
current_section_slice.extend_from_slice(token);
current_section_slice.extend_from_slice(b"*0");
match t.tokens.get(&current_section_slice[..]) {
Some(value) => {
sections_values.extend_from_slice(value);
let l = current_section_slice.len();
current_section_slice[l - 1] = b'1';
}
None => return Err(()),
}
}
}
let mut current_section_seen = 1;
// we have at least the initial section
// try looping until we do not find anymore a next section
loop {
match t.tokens.get(&current_section_slice[..]) {
Some(value) => {
sections_values.extend_from_slice(value);
current_section_seen += 1;
let nbdigits = current_section_slice.len() - token.len() - 1;
current_section_slice.truncate(current_section_slice.len() - nbdigits);
current_section_slice
.extend_from_slice(current_section_seen.to_string().as_bytes());
}
None => return Ok(sections_values),
}
}
}
Err(_) => {
return Err(());
}
}
}
// used on the C side
pub const RS_MIME_MAX_TOKEN_LEN: usize = 255;
#[no_mangle]
pub unsafe extern "C" fn rs_mime_find_header_token(
hinput: *const u8, hlen: u32, tinput: *const u8, tlen: u32, outbuf: &mut [u8; 255],
outlen: *mut u32,
) -> bool {
let hbuf = build_slice!(hinput, hlen as usize);
let tbuf = build_slice!(tinput, tlen as usize);
let mut sections_values = Vec::new();
if let Ok(value) = mime_find_header_token(hbuf, tbuf, &mut sections_values) {
// limit the copy to the supplied buffer size
if value.len() <= RS_MIME_MAX_TOKEN_LEN {
outbuf[..value.len()].clone_from_slice(value);
} else {
outbuf.clone_from_slice(&value[..RS_MIME_MAX_TOKEN_LEN]);
}
*outlen = value.len() as u32;
return true;
}
return false;
}
#[derive(Debug)]
enum MimeParserState {
MimeStart = 0,
MimeHeader = 1,
MimeHeaderEnd = 2,
MimeChunk = 3,
MimeBoundaryWaitingForEol = 4,
}
impl Default for MimeParserState {
fn default() -> Self {
MimeParserState::MimeStart
}
}
#[derive(Debug, Default)]
pub struct MimeStateHTTP {
boundary: Vec<u8>,
filename: Vec<u8>,
state: MimeParserState,
}
#[repr(u8)]
#[derive(Copy, Clone, PartialOrd, PartialEq)]
pub enum MimeParserResult {
MimeNeedsMore = 0,
MimeFileOpen = 1,
MimeFileChunk = 2,
MimeFileClose = 3,
}
fn mime_parse_skip_line(input: &[u8]) -> IResult<&[u8], MimeParserState> {
let (input, _) = take_till(|ch: u8| ch == b'\n')(input)?;
let (input, _) = char('\n')(input)?;
return Ok((input, MimeParserState::MimeStart));
}
fn mime_parse_boundary_regular<'a, 'b>(
boundary: &'b [u8], input: &'a [u8],
) -> IResult<&'a [u8], MimeParserState> {
let (input, _) = tag(boundary)(input)?;
let (input, _) = take_till(|ch: u8| ch == b'\n')(input)?;
let (input, _) = char('\n')(input)?;
return Ok((input, MimeParserState::MimeHeader));
}
// Number of characters after boundary, without end of line, before changing state to streaming
const MIME_BOUNDARY_MAX_BEFORE_EOL: usize = 128;
const MIME_HEADER_MAX_LINE: usize = 4096;
fn mime_parse_boundary_missing_eol<'a, 'b>(
boundary: &'b [u8], input: &'a [u8],
) -> IResult<&'a [u8], MimeParserState> {
let (input, _) = tag(boundary)(input)?;
let (input, _) = take(MIME_BOUNDARY_MAX_BEFORE_EOL)(input)?;
return Ok((input, MimeParserState::MimeBoundaryWaitingForEol));
}
fn mime_parse_boundary<'a, 'b>(
boundary: &'b [u8], input: &'a [u8],
) -> IResult<&'a [u8], MimeParserState> {
let r = mime_parse_boundary_regular(boundary, input);
if r.is_ok() {
return r;
}
let r2 = mime_parse_skip_line(input);
if r2.is_ok() {
return r2;
}
return mime_parse_boundary_missing_eol(boundary, input);
}
fn mime_consume_until_eol(input: &[u8]) -> IResult<&[u8], bool> {
return alt((value(true, mime_parse_skip_line), value(false, rest)))(input);
}
fn mime_parse_header_line(input: &[u8]) -> IResult<&[u8], &[u8]> {
let (input, name) = take_till(|ch: u8| ch == b':')(input)?;
let (input, _) = char(':')(input)?;
return Ok((input, name));
}
// s2 is already lower case
fn rs_equals_lowercase(s1: &[u8], s2: &[u8]) -> bool {
if s1.len() == s2.len() {
for i in 0..s1.len() {
if s1[i].to_ascii_lowercase() != s2[i] {
return false;
}
}
return true;
}
return false;
}
fn mime_parse_headers<'a, 'b>(
ctx: &'b mut MimeStateHTTP, i: &'a [u8],
) -> IResult<&'a [u8], (MimeParserState, bool, bool)> {
let mut fileopen = false;
let mut errored = false;
let mut input = i;
while input.len() > 0 {
match take_until::<_, &[u8], nom7::error::Error<&[u8]>>("\r\n")(input) {
Ok((input2, line)) => {
match mime_parse_header_line(line) {
Ok((value, name)) => {
if rs_equals_lowercase(name, "content-disposition".as_bytes()) {
let mut sections_values = Vec::new();
if let Ok(filename) = mime_find_header_token(
value,
"filename".as_bytes(),
&mut sections_values,
) {
if filename.len() > 0 {
ctx.filename = Vec::with_capacity(filename.len());
fileopen = true;
for c in filename {
// unescape
if *c != b'\\' {
ctx.filename.push(*c);
}
}
}
}
}
if value.len() == 0 {
errored = true;
}
}
_ => {
if line.len() > 0 {
errored = true;
}
}
}
let (input3, _) = tag("\r\n")(input2)?;
input = input3;
if line.len() == 0 || (line.len() == 1 && line[0] == b'\r') {
return Ok((input, (MimeParserState::MimeHeaderEnd, fileopen, errored)));
}
}
_ => {
// guard against too long header lines
if input.len() > MIME_HEADER_MAX_LINE {
return Ok((
input,
(
MimeParserState::MimeBoundaryWaitingForEol,
fileopen,
errored,
),
));
}
if input.len() < i.len() {
return Ok((input, (MimeParserState::MimeHeader, fileopen, errored)));
} // else only an incomplete line, ask for more
return Err(Err::Error(make_error(input, ErrorKind::Eof)));
}
}
}
return Ok((input, (MimeParserState::MimeHeader, fileopen, errored)));
}
fn mime_consume_chunk<'a, 'b>(boundary: &'b [u8], input: &'a [u8]) -> IResult<&'a [u8], bool> {
let r: Result<(&[u8], &[u8]), Err<nom7::error::Error<&[u8]>>> = take_until("\r\n")(input);
match r {
Ok((input, line)) => {
let (input2, _) = tag("\r\n")(input)?;
if input2.len() < boundary.len() {
if input2 == &boundary[..input2.len()] {
if line.len() > 0 {
// consume as chunk up to eol (not consuming eol)
return Ok((input, false));
}
// new line beignning like boundary, with nothin to consume as chunk : request more
return Err(Err::Error(make_error(input, ErrorKind::Eof)));
}
// not like boundary : consume everything as chunk
return Ok((&input[input.len()..], false));
} // else
if &input2[..boundary.len()] == boundary {
// end of file with boundary, consume eol but do not consume boundary
return Ok((input2, true));
}
// not like boundary : consume everything as chunk
return Ok((input2, false));
}
_ => {
return Ok((&input[input.len()..], false));
}
}
}
pub const MIME_EVENT_FLAG_INVALID_HEADER: u32 = 0x01;
pub const MIME_EVENT_FLAG_NO_FILEDATA: u32 = 0x02;
fn mime_process(ctx: &mut MimeStateHTTP, i: &[u8]) -> (MimeParserResult, u32, u32) {
let mut input = i;
let mut consumed = 0;
let mut warnings = 0;
while input.len() > 0 {
match ctx.state {
MimeParserState::MimeStart => {
if let Ok((rem, next)) = mime_parse_boundary(&ctx.boundary, input) {
ctx.state = next;
consumed += (input.len() - rem.len()) as u32;
input = rem;
} else {
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
}
MimeParserState::MimeBoundaryWaitingForEol => {
if let Ok((rem, found)) = mime_consume_until_eol(input) {
if found {
ctx.state = MimeParserState::MimeHeader;
}
consumed += (input.len() - rem.len()) as u32;
input = rem;
} else {
// should never happen
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
}
MimeParserState::MimeHeader => {
if let Ok((rem, (next, fileopen, err))) = mime_parse_headers(ctx, input) {
ctx.state = next;
consumed += (input.len() - rem.len()) as u32;
input = rem;
if err {
warnings |= MIME_EVENT_FLAG_INVALID_HEADER;
}
if fileopen {
return (MimeParserResult::MimeFileOpen, consumed, warnings);
}
} else {
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
}
MimeParserState::MimeHeaderEnd => {
// check if we start with the boundary
// and transition to chunk, or empty file and back to start
if input.len() < ctx.boundary.len() {
if input == &ctx.boundary[..input.len()] {
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
ctx.state = MimeParserState::MimeChunk;
} else {
if &input[..ctx.boundary.len()] == ctx.boundary {
ctx.state = MimeParserState::MimeStart;
if ctx.filename.len() > 0 {
warnings |= MIME_EVENT_FLAG_NO_FILEDATA;
}
ctx.filename.clear();
return (MimeParserResult::MimeFileClose, consumed, warnings);
} else {
ctx.state = MimeParserState::MimeChunk;
}
}
}
MimeParserState::MimeChunk => {
if let Ok((rem, eof)) = mime_consume_chunk(&ctx.boundary, input) {
consumed += (input.len() - rem.len()) as u32;
if eof {
ctx.state = MimeParserState::MimeStart;
ctx.filename.clear();
return (MimeParserResult::MimeFileClose, consumed, warnings);
} else {
// + 2 for \r\n
if rem.len() < ctx.boundary.len() + 2 {
return (MimeParserResult::MimeFileChunk, consumed, warnings);
}
input = rem;
}
} else {
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
}
}
}
return (MimeParserResult::MimeNeedsMore, consumed, warnings);
}
pub fn mime_state_init(i: &[u8]) -> Option<MimeStateHTTP> {
let mut sections_values = Vec::new();
match mime_find_header_token(i, "boundary".as_bytes(), &mut sections_values) {
Ok(value) => {
if value.len() <= RS_MIME_MAX_TOKEN_LEN {
let mut r = MimeStateHTTP::default();
r.boundary = Vec::with_capacity(2 + value.len());
// start wih 2 additional hyphens
r.boundary.push(b'-');
r.boundary.push(b'-');
for c in value {
// unescape
if *c != b'\\' {
r.boundary.push(*c);
}
}
return Some(r);
}
}
_ => {}
}
return None;
}
#[no_mangle]
pub unsafe extern "C" fn rs_mime_state_init(
input: *const u8, input_len: u32,
) -> *mut MimeStateHTTP {
let slice = build_slice!(input, input_len as usize);
if let Some(ctx) = mime_state_init(slice) {
let boxed = Box::new(ctx);
return Box::into_raw(boxed) as *mut _;
}
return std::ptr::null_mut();
}
#[no_mangle]
pub unsafe extern "C" fn rs_mime_parse(
ctx: &mut MimeStateHTTP, input: *const u8, input_len: u32, consumed: *mut u32,
warnings: *mut u32,
) -> MimeParserResult {
let slice = build_slice!(input, input_len as usize);
let (r, c, w) = mime_process(ctx, slice);
*consumed = c;
*warnings = w;
return r;
}
#[no_mangle]
pub unsafe extern "C" fn rs_mime_state_get_filename(
ctx: &mut MimeStateHTTP, buffer: *mut *const u8, filename_len: *mut u16,
) {
if ctx.filename.len() > 0 {
*buffer = ctx.filename.as_ptr();
if ctx.filename.len() < u16::MAX.into() {
*filename_len = ctx.filename.len() as u16;
} else {
*filename_len = u16::MAX;
}
} else {
*buffer = std::ptr::null_mut();
*filename_len = 0;
}
}
#[no_mangle]
pub unsafe extern "C" fn rs_mime_state_free(ctx: &mut MimeStateHTTP) {
// Just unbox...
std::mem::drop(Box::from_raw(ctx));
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_mime_find_header_token() {
let mut outvec = Vec::new();
let undelimok = mime_find_header_token(
"attachment; filename=test;".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(undelimok, Ok("test".as_bytes()));
let delimok = mime_find_header_token(
"attachment; filename=\"test2\";".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(delimok, Ok("test2".as_bytes()));
let escaped = mime_find_header_token(
"attachment; filename=\"test\\\"2\";".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(escaped, Ok("test\\\"2".as_bytes()));
let evasion_othertoken = mime_find_header_token(
"attachment; dummy=\"filename=wrong\"; filename=real;".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(evasion_othertoken, Ok("real".as_bytes()));
let evasion_suffixtoken = mime_find_header_token(
"attachment; notafilename=wrong; filename=good;".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(evasion_suffixtoken, Ok("good".as_bytes()));
let badending = mime_find_header_token(
"attachment; filename=oksofar; badending".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(badending, Ok("oksofar".as_bytes()));
let missend = mime_find_header_token(
"attachment; filename=test".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(missend, Ok("test".as_bytes()));
let spaces = mime_find_header_token(
"attachment; filename=test me wrong".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(spaces, Ok("test me wrong".as_bytes()));
assert_eq!(outvec.len(), 0);
let multi = mime_find_header_token(
"attachment; filename*0=abc; filename*1=\"def\";".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(multi, Ok("abcdef".as_bytes()));
outvec.clear();
let multi = mime_find_header_token(
"attachment; filename*1=456; filename*0=\"123\"".as_bytes(),
"filename".as_bytes(),
&mut outvec,
);
assert_eq!(multi, Ok("123456".as_bytes()));
outvec.clear();
}
}
pub mod mime;
pub mod smtp;
pub mod smtp_log;

854
rust/src/mime/smtp.rs
Unescape Escape View File

@ -0,0 +1,854 @@
/* Copyright (C) 2024 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use super::mime;
use crate::core::StreamingBufferConfig;
use crate::filecontainer::FileContainer;
use digest::generic_array::{typenum::U16, GenericArray};
use digest::Digest;
use digest::Update;
use md5::Md5;
use std::ffi::CStr;
use std::io;
use std::os::raw::c_uchar;
#[repr(u8)]
#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)]
pub enum MimeSmtpParserState {
MimeSmtpStart = 0,
MimeSmtpHeader = 1,
MimeSmtpBody = 2,
MimeSmtpParserError = 3,
}
impl Default for MimeSmtpParserState {
fn default() -> Self {
MimeSmtpParserState::MimeSmtpStart
}
}
#[derive(Debug, Default)]
pub struct MimeHeader {
pub name: Vec<u8>,
pub value: Vec<u8>,
}
#[repr(u8)]
#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)]
pub enum MimeSmtpMd5State {
MimeSmtpMd5Disabled = 0,
MimeSmtpMd5Inited = 1,
MimeSmtpMd5Started = 2,
MimeSmtpMd5Completed = 3,
}
#[repr(u8)]
#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)]
enum MimeSmtpContentType {
Message = 0,
PlainText = 1,
Html = 2,
Unknown = 3,
}
impl Default for MimeSmtpContentType {
fn default() -> Self {
MimeSmtpContentType::Message
}
}
#[derive(Debug)]
pub struct MimeStateSMTP<'a> {
pub(crate) state_flag: MimeSmtpParserState,
pub(crate) headers: Vec<MimeHeader>,
pub(crate) main_headers_nb: usize,
filename: Vec<u8>,
pub(crate) attachments: Vec<Vec<u8>>,
pub(crate) urls: Vec<Vec<u8>>,
boundaries: Vec<Vec<u8>>,
encoding: MimeSmtpEncoding,
decoder: Option<MimeBase64Decoder>,
content_type: MimeSmtpContentType,
decoded_line: Vec<u8>,
// small buffer for end of line
// waiting to see if it is part of the boundary
bufeol: [u8; 2],
bufeolen: u8,
files: &'a mut FileContainer,
sbcfg: *const StreamingBufferConfig,
md5: md5::Md5,
pub(crate) md5_state: MimeSmtpMd5State,
pub(crate) md5_result: GenericArray<u8, U16>,
}
#[derive(Debug)]
pub struct MimeBase64Decoder {
tmp: [u8; 4],
nb: u8,
}
impl MimeBase64Decoder {
pub fn new() -> MimeBase64Decoder {
MimeBase64Decoder { tmp: [0; 4], nb: 0 }
}
}
impl Default for MimeBase64Decoder {
fn default() -> Self {
Self::new()
}
}
pub fn mime_smtp_state_init(
files: &mut FileContainer, sbcfg: *const StreamingBufferConfig,
) -> Option<MimeStateSMTP> {
let r = MimeStateSMTP {
state_flag: MimeSmtpParserState::MimeSmtpStart,
headers: Vec::new(),
main_headers_nb: 0,
filename: Vec::new(),
attachments: Vec::new(),
urls: Vec::new(),
boundaries: Vec::new(),
decoded_line: Vec::new(),
encoding: MimeSmtpEncoding::Plain,
decoder: None,
content_type: MimeSmtpContentType::Message,
bufeol: [0; 2],
bufeolen: 0,
files,
sbcfg,
md5: Md5::new(),
md5_state: MimeSmtpMd5State::MimeSmtpMd5Disabled,
md5_result: [0; 16].into(),
};
return Some(r);
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpStateInit(
files: &mut FileContainer, sbcfg: *const StreamingBufferConfig,
) -> *mut MimeStateSMTP {
if let Some(ctx) = mime_smtp_state_init(files, sbcfg) {
let boxed = Box::new(ctx);
return Box::into_raw(boxed) as *mut _;
}
return std::ptr::null_mut();
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpStateFree(ctx: &mut MimeStateSMTP) {
// Just unbox...
std::mem::drop(Box::from_raw(ctx));
}
#[repr(u8)]
#[derive(Copy, Clone, PartialOrd, PartialEq, Eq)]
pub enum MimeSmtpParserResult {
MimeSmtpNeedsMore = 0,
MimeSmtpFileOpen = 1,
MimeSmtpFileClose = 2,
MimeSmtpFileChunk = 3,
}
#[repr(u8)]
#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Eq)]
pub enum MimeSmtpEncoding {
Plain = 0,
Base64 = 1,
QuotedPrintable = 2,
}
impl Default for MimeSmtpEncoding {
fn default() -> Self {
MimeSmtpEncoding::Plain
}
}
// Cannot use BIT_U32 macros as they do not get exported by cbindgen :-/
pub const MIME_ANOM_INVALID_BASE64: u32 = 0x1;
pub const MIME_ANOM_INVALID_QP: u32 = 0x2;
pub const MIME_ANOM_LONG_LINE: u32 = 0x4;
pub const MIME_ANOM_LONG_ENC_LINE: u32 = 0x8;
pub const MIME_ANOM_LONG_HEADER_NAME: u32 = 0x10;
pub const MIME_ANOM_LONG_HEADER_VALUE: u32 = 0x20;
//unused pub const MIME_ANOM_MALFORMED_MSG: u32 = 0x40;
pub const MIME_ANOM_LONG_BOUNDARY: u32 = 0x80;
pub const MIME_ANOM_LONG_FILENAME: u32 = 0x100;
fn mime_smtp_process_headers(ctx: &mut MimeStateSMTP) -> (u32, bool) {
let mut sections_values = Vec::new();
let mut warnings = 0;
let mut encap = false;
for h in &ctx.headers[ctx.main_headers_nb..] {
if mime::slice_equals_lowercase(&h.name, b"content-disposition") {
if ctx.filename.is_empty() {
if let Some(value) =
mime::mime_find_header_token(&h.value, b"filename", &mut sections_values)
{
let value = if value.len() > mime::RS_MIME_MAX_TOKEN_LEN {
warnings |= MIME_ANOM_LONG_FILENAME;
&value[..mime::RS_MIME_MAX_TOKEN_LEN]
} else {
value
};
ctx.filename.extend_from_slice(value);
let mut newname = Vec::new();
newname.extend_from_slice(value);
ctx.attachments.push(newname);
sections_values.clear();
}
}
} else if mime::slice_equals_lowercase(&h.name, b"content-transfer-encoding") {
if mime::slice_equals_lowercase(&h.value, b"base64") {
ctx.encoding = MimeSmtpEncoding::Base64;
ctx.decoder = Some(MimeBase64Decoder::new());
} else if mime::slice_equals_lowercase(&h.value, b"quoted-printable") {
ctx.encoding = MimeSmtpEncoding::QuotedPrintable;
}
} else if mime::slice_equals_lowercase(&h.name, b"content-type") {
if ctx.filename.is_empty() {
if let Some(value) =
mime::mime_find_header_token(&h.value, b"name", &mut sections_values)
{
let value = if value.len() > mime::RS_MIME_MAX_TOKEN_LEN {
warnings |= MIME_ANOM_LONG_FILENAME;
&value[..mime::RS_MIME_MAX_TOKEN_LEN]
} else {
value
};
ctx.filename.extend_from_slice(value);
let mut newname = Vec::new();
newname.extend_from_slice(value);
ctx.attachments.push(newname);
sections_values.clear();
}
}
if let Some(value) =
mime::mime_find_header_token(&h.value, b"boundary", &mut sections_values)
{
// start wih 2 additional hyphens
let mut boundary = Vec::new();
boundary.push(b'-');
boundary.push(b'-');
boundary.extend_from_slice(value);
ctx.boundaries.push(boundary);
if value.len() > MAX_BOUNDARY_LEN {
warnings |= MIME_ANOM_LONG_BOUNDARY;
}
sections_values.clear();
}
let ct = if let Some(x) = h.value.iter().position(|&x| x == b';') {
&h.value[..x]
} else {
&h.value
};
match ct {
b"text/plain" => {
ctx.content_type = MimeSmtpContentType::PlainText;
}
b"text/html" => {
ctx.content_type = MimeSmtpContentType::Html;
}
_ => {
if ct.starts_with(b"message/") {
encap = true;
}
ctx.content_type = MimeSmtpContentType::Unknown;
}
}
}
}
return (warnings, encap);
}
extern "C" {
// Defined in util-file.h
pub fn FileAppendData(
c: *mut FileContainer, sbcfg: *const StreamingBufferConfig, data: *const c_uchar,
data_len: u32,
) -> std::os::raw::c_int;
// Defined in util-spm-bs.h
pub fn BasicSearchNocaseIndex(
data: *const c_uchar, data_len: u32, needle: *const c_uchar, needle_len: u16,
) -> u32;
}
fn hex(i: u8) -> Option<u8> {
if i.is_ascii_digit() {
return Some(i - b'0');
}
if (b'A'..=b'F').contains(&i) {
return Some(i - b'A' + 10);
}
return None;
}
const SMTP_MIME_MAX_DECODED_LINE_LENGTH: usize = 8192;
fn mime_smtp_finish_url(input: &[u8]) -> &[u8] {
if let Some(x) = input.iter().position(|&x| {
x == b' ' || x == b'"' || x == b'\'' || x == b'<' || x == b'>' || x == b']' || x == b'\t'
}) {
return &input[..x];
}
return input;
}
fn mime_smtp_extract_urls(urls: &mut Vec<Vec<u8>>, input_start: &[u8]) {
//TODO optimize later : use mpm
for s in unsafe { MIME_SMTP_CONFIG_EXTRACT_URL_SCHEMES.iter() } {
let mut input = input_start;
let mut start = unsafe {
BasicSearchNocaseIndex(
input.as_ptr(),
input.len() as u32,
s.as_ptr(),
s.len() as u16,
)
};
while (start as usize) < input.len() {
let url = mime_smtp_finish_url(&input[start as usize..]);
let mut urlv = Vec::with_capacity(url.len());
if unsafe { !MIME_SMTP_CONFIG_LOG_URL_SCHEME } {
urlv.extend_from_slice(&url[s.len()..]);
} else {
urlv.extend_from_slice(url);
}
urls.push(urlv);
input = &input[start as usize + url.len()..];
start = unsafe {
BasicSearchNocaseIndex(
input.as_ptr(),
input.len() as u32,
s.as_ptr(),
s.len() as u16,
)
};
}
}
}
fn mime_smtp_find_url_strings(ctx: &mut MimeStateSMTP, input_new: &[u8]) {
if unsafe { !MIME_SMTP_CONFIG_EXTRACT_URLS } {
return;
}
let mut input = input_new;
// use previosly buffered beginning of line if any
if !ctx.decoded_line.is_empty() {
ctx.decoded_line.extend_from_slice(input_new);
input = &ctx.decoded_line;
}
// no input, no url
if input.is_empty() {
return;
}
if input[input.len() - 1] == b'\n' || input.len() > SMTP_MIME_MAX_DECODED_LINE_LENGTH {
// easy case, no buffering to do
mime_smtp_extract_urls(&mut ctx.urls, input);
if !ctx.decoded_line.is_empty() {
ctx.decoded_line.clear()
}
} else if let Some(x) = input.iter().rev().position(|&x| x == b'\n') {
input = &input[..x];
mime_smtp_extract_urls(&mut ctx.urls, input);
if !ctx.decoded_line.is_empty() {
ctx.decoded_line.drain(0..x);
} else {
ctx.decoded_line.extend_from_slice(&input_new[x..]);
}
} // else no end of line, already buffered for next input...
}
fn mime_base64_map(input: u8) -> io::Result<u8> {
match input {
43 => Ok(62), // +
47 => Ok(63), // /
48 => Ok(52), // 0
49 => Ok(53), // 1
50 => Ok(54), // 2
51 => Ok(55), // 3
52 => Ok(56), // 4
53 => Ok(57), // 5
54 => Ok(58), // 6
55 => Ok(59), // 7
56 => Ok(60), // 8
57 => Ok(61), // 9
65 => Ok(0), // A
66 => Ok(1), // B
67 => Ok(2), // C
68 => Ok(3), // D
69 => Ok(4), // E
70 => Ok(5), // F
71 => Ok(6), // G
72 => Ok(7), // H
73 => Ok(8), // I
74 => Ok(9), // J
75 => Ok(10), // K
76 => Ok(11), // L
77 => Ok(12), // M
78 => Ok(13), // N
79 => Ok(14), // O
80 => Ok(15), // P
81 => Ok(16), // Q
82 => Ok(17), // R
83 => Ok(18), // S
84 => Ok(19), // T
85 => Ok(20), // U
86 => Ok(21), // V
87 => Ok(22), // W
88 => Ok(23), // X
89 => Ok(24), // Y
90 => Ok(25), // Z
97 => Ok(26), // a
98 => Ok(27), // b
99 => Ok(28), // c
100 => Ok(29), // d
101 => Ok(30), // e
102 => Ok(31), // f
103 => Ok(32), // g
104 => Ok(33), // h
105 => Ok(34), // i
106 => Ok(35), // j
107 => Ok(36), // k
108 => Ok(37), // l
109 => Ok(38), // m
110 => Ok(39), // n
111 => Ok(40), // o
112 => Ok(41), // p
113 => Ok(42), // q
114 => Ok(43), // r
115 => Ok(44), // s
116 => Ok(45), // t
117 => Ok(46), // u
118 => Ok(47), // v
119 => Ok(48), // w
120 => Ok(49), // x
121 => Ok(50), // y
122 => Ok(51), // z
_ => Err(io::Error::new(io::ErrorKind::InvalidData, "invalid base64")),
}
}
fn mime_base64_decode(decoder: &mut MimeBase64Decoder, input: &[u8]) -> io::Result<Vec<u8>> {
let mut i = input;
let maxlen = ((decoder.nb as usize + i.len()) * 3) / 4;
let mut r = vec![0; maxlen];
let mut offset = 0;
while !i.is_empty() {
while decoder.nb < 4 && !i.is_empty() {
if mime_base64_map(i[0]).is_ok() || i[0] == b'=' {
decoder.tmp[decoder.nb as usize] = i[0];
decoder.nb += 1;
}
i = &i[1..];
}
if decoder.nb == 4 {
decoder.tmp[0] = mime_base64_map(decoder.tmp[0])?;
decoder.tmp[1] = mime_base64_map(decoder.tmp[1])?;
if decoder.tmp[2] == b'=' {
r[offset] = (decoder.tmp[0] << 2) | (decoder.tmp[1] >> 4);
offset += 1;
} else {
decoder.tmp[2] = mime_base64_map(decoder.tmp[2])?;
if decoder.tmp[3] == b'=' {
r[offset] = (decoder.tmp[0] << 2) | (decoder.tmp[1] >> 4);
r[offset + 1] = (decoder.tmp[1] << 4) | (decoder.tmp[2] >> 2);
offset += 2;
} else {
decoder.tmp[3] = mime_base64_map(decoder.tmp[3])?;
r[offset] = (decoder.tmp[0] << 2) | (decoder.tmp[1] >> 4);
r[offset + 1] = (decoder.tmp[1] << 4) | (decoder.tmp[2] >> 2);
r[offset + 2] = (decoder.tmp[2] << 6) | decoder.tmp[3];
offset += 3;
}
}
decoder.nb = 0;
}
}
r.truncate(offset);
return Ok(r);
}
const MAX_LINE_LEN: u32 = 998; // Def in RFC 2045, excluding CRLF sequence
const MAX_ENC_LINE_LEN: usize = 76; /* Def in RFC 2045, excluding CRLF sequence */
const MAX_HEADER_NAME: usize = 75; /* 75 + ":" = 76 */
const MAX_HEADER_VALUE: usize = 2000; /* Default - arbitrary limit */
const MAX_BOUNDARY_LEN: usize = 254;
fn mime_smtp_parse_line(
ctx: &mut MimeStateSMTP, i: &[u8], full: &[u8],
) -> (MimeSmtpParserResult, u32) {
if ctx.md5_state == MimeSmtpMd5State::MimeSmtpMd5Started {
Update::update(&mut ctx.md5, full);
}
let mut warnings = 0;
match ctx.state_flag {
MimeSmtpParserState::MimeSmtpStart => {
if unsafe { MIME_SMTP_CONFIG_BODY_MD5 }
&& ctx.md5_state != MimeSmtpMd5State::MimeSmtpMd5Started
{
ctx.md5 = Md5::new();
ctx.md5_state = MimeSmtpMd5State::MimeSmtpMd5Inited;
}
if i.is_empty() {
let (w, encap_msg) = mime_smtp_process_headers(ctx);
warnings |= w;
if ctx.main_headers_nb == 0 {
ctx.main_headers_nb = ctx.headers.len();
}
if encap_msg {
ctx.state_flag = MimeSmtpParserState::MimeSmtpStart;
ctx.headers.truncate(ctx.main_headers_nb);
return (MimeSmtpParserResult::MimeSmtpNeedsMore, warnings);
}
ctx.state_flag = MimeSmtpParserState::MimeSmtpBody;
return (MimeSmtpParserResult::MimeSmtpFileOpen, warnings);
} else if let Ok((value, name)) = mime::mime_parse_header_line(i) {
ctx.state_flag = MimeSmtpParserState::MimeSmtpHeader;
let mut h = MimeHeader::default();
h.name.extend_from_slice(name);
h.value.extend_from_slice(value);
if h.name.len() > MAX_HEADER_NAME {
warnings |= MIME_ANOM_LONG_HEADER_NAME;
}
if h.value.len() > MAX_HEADER_VALUE {
warnings |= MIME_ANOM_LONG_HEADER_VALUE;
}
ctx.headers.push(h);
} // else event ?
}
MimeSmtpParserState::MimeSmtpHeader => {
if i.is_empty() {
let (w, encap_msg) = mime_smtp_process_headers(ctx);
warnings |= w;
if ctx.main_headers_nb == 0 {
ctx.main_headers_nb = ctx.headers.len();
}
if encap_msg {
ctx.state_flag = MimeSmtpParserState::MimeSmtpStart;
ctx.headers.truncate(ctx.main_headers_nb);
return (MimeSmtpParserResult::MimeSmtpNeedsMore, warnings);
}
ctx.state_flag = MimeSmtpParserState::MimeSmtpBody;
return (MimeSmtpParserResult::MimeSmtpFileOpen, warnings);
} else if i[0] == b' ' || i[0] == b'\t' {
let last = ctx.headers.len() - 1;
ctx.headers[last].value.extend_from_slice(&i[1..]);
} else if let Ok((value, name)) = mime::mime_parse_header_line(i) {
let mut h = MimeHeader::default();
h.name.extend_from_slice(name);
h.value.extend_from_slice(value);
if h.name.len() > MAX_HEADER_NAME {
warnings |= MIME_ANOM_LONG_HEADER_NAME;
}
if h.value.len() > MAX_HEADER_VALUE {
warnings |= MIME_ANOM_LONG_HEADER_VALUE;
}
ctx.headers.push(h);
}
}
MimeSmtpParserState::MimeSmtpBody => {
if ctx.md5_state == MimeSmtpMd5State::MimeSmtpMd5Inited {
ctx.md5_state = MimeSmtpMd5State::MimeSmtpMd5Started;
Update::update(&mut ctx.md5, full);
}
let boundary = ctx.boundaries.last();
if let Some(b) = boundary {
if i.len() >= b.len() && &i[..b.len()] == b {
if ctx.encoding == MimeSmtpEncoding::Base64
&& unsafe { MIME_SMTP_CONFIG_DECODE_BASE64 }
{
if let Some(ref mut decoder) = &mut ctx.decoder {
if decoder.nb > 0 {
// flush the base64 buffer with padding
let mut v = Vec::new();
for _i in 0..4 - decoder.nb {
v.push(b'=');
}
if let Ok(dec) = mime_base64_decode(decoder, &v) {
unsafe {
FileAppendData(
ctx.files,
ctx.sbcfg,
dec.as_ptr(),
dec.len() as u32,
);
}
}
}
}
}
ctx.state_flag = MimeSmtpParserState::MimeSmtpStart;
let toclose = !ctx.filename.is_empty();
ctx.filename.clear();
ctx.headers.truncate(ctx.main_headers_nb);
ctx.encoding = MimeSmtpEncoding::Plain;
if i.len() >= b.len() + 2 && i[b.len()] == b'-' && i[b.len() + 1] == b'-' {
ctx.boundaries.pop();
}
if toclose {
return (MimeSmtpParserResult::MimeSmtpFileClose, 0);
}
return (MimeSmtpParserResult::MimeSmtpNeedsMore, 0);
}
}
if ctx.filename.is_empty() {
if ctx.content_type == MimeSmtpContentType::PlainText
|| ctx.content_type == MimeSmtpContentType::Html
|| ctx.content_type == MimeSmtpContentType::Message
{
mime_smtp_find_url_strings(ctx, full);
}
return (MimeSmtpParserResult::MimeSmtpNeedsMore, 0);
}
match ctx.encoding {
MimeSmtpEncoding::Plain => {
mime_smtp_find_url_strings(ctx, full);
if ctx.bufeolen > 0 {
unsafe {
FileAppendData(
ctx.files,
ctx.sbcfg,
ctx.bufeol.as_ptr(),
ctx.bufeol.len() as u32,
);
}
}
unsafe {
FileAppendData(ctx.files, ctx.sbcfg, i.as_ptr(), i.len() as u32);
}
ctx.bufeolen = (full.len() - i.len()) as u8;
if ctx.bufeolen > 0 {
ctx.bufeol[..ctx.bufeolen as usize].copy_from_slice(&full[i.len()..]);
}
}
MimeSmtpEncoding::Base64 => {
if unsafe { MIME_SMTP_CONFIG_DECODE_BASE64 } {
if let Some(ref mut decoder) = &mut ctx.decoder {
if i.len() > MAX_ENC_LINE_LEN {
warnings |= MIME_ANOM_LONG_ENC_LINE;
}
if let Ok(dec) = mime_base64_decode(decoder, i) {
mime_smtp_find_url_strings(ctx, &dec);
unsafe {
FileAppendData(
ctx.files,
ctx.sbcfg,
dec.as_ptr(),
dec.len() as u32,
);
}
} else {
warnings |= MIME_ANOM_INVALID_BASE64;
}
}
}
}
MimeSmtpEncoding::QuotedPrintable => {
if unsafe { MIME_SMTP_CONFIG_DECODE_QUOTED } {
if i.len() > MAX_ENC_LINE_LEN {
warnings |= MIME_ANOM_LONG_ENC_LINE;
}
let mut c = 0;
let mut eol_equal = false;
let mut quoted_buffer = Vec::with_capacity(i.len());
while c < i.len() {
if i[c] == b'=' {
if c == i.len() - 1 {
eol_equal = true;
break;
} else if c + 2 >= i.len() {
// log event ?
warnings |= MIME_ANOM_INVALID_QP;
break;
}
if let Some(v) = hex(i[c + 1]) {
if let Some(v2) = hex(i[c + 2]) {
quoted_buffer.push((v << 4) | v2);
} else {
warnings |= MIME_ANOM_INVALID_QP;
}
} else {
warnings |= MIME_ANOM_INVALID_QP;
}
c += 3;
} else {
quoted_buffer.push(i[c]);
c += 1;
}
}
if !eol_equal {
quoted_buffer.extend_from_slice(&full[i.len()..]);
}
mime_smtp_find_url_strings(ctx, &quoted_buffer);
unsafe {
FileAppendData(
ctx.files,
ctx.sbcfg,
quoted_buffer.as_ptr(),
quoted_buffer.len() as u32,
);
}
}
}
}
return (MimeSmtpParserResult::MimeSmtpFileChunk, warnings);
}
_ => {}
}
return (MimeSmtpParserResult::MimeSmtpNeedsMore, warnings);
}
#[no_mangle]
pub unsafe extern "C" fn SCSmtpMimeParseLine(
input: *const u8, input_len: u32, delim_len: u8, warnings: *mut u32, ctx: &mut MimeStateSMTP,
) -> MimeSmtpParserResult {
let full_line = build_slice!(input, input_len as usize + delim_len as usize);
let line = &full_line[..input_len as usize];
let (r, w) = mime_smtp_parse_line(ctx, line, full_line);
*warnings = w;
if input_len > MAX_LINE_LEN {
*warnings |= MIME_ANOM_LONG_LINE;
}
return r;
}
fn mime_smtp_complete(ctx: &mut MimeStateSMTP) {
if ctx.md5_state == MimeSmtpMd5State::MimeSmtpMd5Started {
ctx.md5_state = MimeSmtpMd5State::MimeSmtpMd5Completed;
ctx.md5_result = ctx.md5.finalize_reset();
}
}
#[no_mangle]
pub unsafe extern "C" fn SCSmtpMimeComplete(ctx: &mut MimeStateSMTP) {
mime_smtp_complete(ctx);
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpGetState(ctx: &mut MimeStateSMTP) -> MimeSmtpParserState {
return ctx.state_flag;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpGetFilename(
ctx: &mut MimeStateSMTP, buffer: *mut *const u8, filename_len: *mut u16,
) {
if !ctx.filename.is_empty() {
*buffer = ctx.filename.as_ptr();
if ctx.filename.len() < u16::MAX.into() {
*filename_len = ctx.filename.len() as u16;
} else {
*filename_len = u16::MAX;
}
} else {
*buffer = std::ptr::null_mut();
*filename_len = 0;
}
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpGetHeader(
ctx: &mut MimeStateSMTP, str: *const std::os::raw::c_char, buffer: *mut *const u8,
buffer_len: *mut u32,
) -> bool {
let name: &CStr = CStr::from_ptr(str); //unsafe
for h in &ctx.headers[ctx.main_headers_nb..] {
if mime::slice_equals_lowercase(&h.name, name.to_bytes()) {
*buffer = h.value.as_ptr();
*buffer_len = h.value.len() as u32;
return true;
}
}
*buffer = std::ptr::null_mut();
*buffer_len = 0;
return false;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpGetHeaderName(
ctx: &mut MimeStateSMTP, buffer: *mut *const u8, buffer_len: *mut u32, num: u32,
) -> bool {
if num as usize + ctx.main_headers_nb < ctx.headers.len() {
*buffer = ctx.headers[ctx.main_headers_nb + num as usize]
.name
.as_ptr();
*buffer_len = ctx.headers[ctx.main_headers_nb + num as usize].name.len() as u32;
return true;
}
*buffer = std::ptr::null_mut();
*buffer_len = 0;
return false;
}
static mut MIME_SMTP_CONFIG_DECODE_BASE64: bool = true;
static mut MIME_SMTP_CONFIG_DECODE_QUOTED: bool = true;
static mut MIME_SMTP_CONFIG_BODY_MD5: bool = false;
static mut MIME_SMTP_CONFIG_HEADER_VALUE_DEPTH: u32 = 0;
static mut MIME_SMTP_CONFIG_EXTRACT_URLS: bool = true;
static mut MIME_SMTP_CONFIG_LOG_URL_SCHEME: bool = false;
static mut MIME_SMTP_CONFIG_EXTRACT_URL_SCHEMES: Vec<&str> = Vec::new();
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpConfigDecodeBase64(val: std::os::raw::c_int) {
MIME_SMTP_CONFIG_DECODE_BASE64 = val != 0;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpConfigDecodeQuoted(val: std::os::raw::c_int) {
MIME_SMTP_CONFIG_DECODE_QUOTED = val != 0;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpConfigExtractUrls(val: std::os::raw::c_int) {
MIME_SMTP_CONFIG_EXTRACT_URLS = val != 0;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpConfigLogUrlScheme(val: std::os::raw::c_int) {
MIME_SMTP_CONFIG_LOG_URL_SCHEME = val != 0;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpConfigBodyMd5(val: std::os::raw::c_int) {
MIME_SMTP_CONFIG_BODY_MD5 = val != 0;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpConfigHeaderValueDepth(val: u32) {
MIME_SMTP_CONFIG_HEADER_VALUE_DEPTH = val;
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpConfigExtractUrlsSchemeReset() {
MIME_SMTP_CONFIG_EXTRACT_URL_SCHEMES.clear();
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpConfigExtractUrlsSchemeAdd(
str: *const std::os::raw::c_char,
) -> std::os::raw::c_int {
let scheme: &CStr = CStr::from_ptr(str); //unsafe
if let Ok(s) = scheme.to_str() {
MIME_SMTP_CONFIG_EXTRACT_URL_SCHEMES.push(s);
return 0;
}
return -1;
}

241
rust/src/mime/smtp_log.rs
Unescape Escape View File

@ -0,0 +1,241 @@
/* Copyright (C) 2022 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use super::mime;
use crate::jsonbuilder::{JsonBuilder, JsonError};
use crate::mime::smtp::{MimeSmtpMd5State, MimeStateSMTP};
use digest::Digest;
use digest::Update;
use md5::Md5;
use std::ffi::CStr;
fn log_subject_md5(js: &mut JsonBuilder, ctx: &mut MimeStateSMTP) -> Result<(), JsonError> {
for h in &ctx.headers[..ctx.main_headers_nb] {
if mime::slice_equals_lowercase(&h.name, b"subject") {
let hash = format!("{:x}", Md5::new().chain(&h.value).finalize());
js.set_string("subject_md5", &hash)?;
break;
}
}
return Ok(());
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpLogSubjectMd5(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP,
) -> bool {
return log_subject_md5(js, ctx).is_ok();
}
fn log_body_md5(js: &mut JsonBuilder, ctx: &mut MimeStateSMTP) -> Result<(), JsonError> {
if ctx.md5_state == MimeSmtpMd5State::MimeSmtpMd5Completed {
let hash = format!("{:x}", ctx.md5_result);
js.set_string("body_md5", &hash)?;
}
return Ok(());
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpLogBodyMd5(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP,
) -> bool {
return log_body_md5(js, ctx).is_ok();
}
fn log_field_array(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, c: &str, e: &str,
) -> Result<(), JsonError> {
let mark = js.get_mark();
let mut found = false;
js.open_array(c)?;
for h in &ctx.headers[..ctx.main_headers_nb] {
if mime::slice_equals_lowercase(&h.name, e.as_bytes()) {
found = true;
js.append_string(&String::from_utf8_lossy(&h.value))?;
}
}
if found {
js.close()?;
} else {
js.restore_mark(&mark)?;
}
return Ok(());
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpLogFieldArray(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, email: *const std::os::raw::c_char,
config: *const std::os::raw::c_char,
) -> bool {
let e: &CStr = CStr::from_ptr(email); //unsafe
if let Ok(email_field) = e.to_str() {
let c: &CStr = CStr::from_ptr(config); //unsafe
if let Ok(config_field) = c.to_str() {
return log_field_array(js, ctx, config_field, email_field).is_ok();
}
}
return false;
}
enum FieldCommaState {
Start = 0, // skip leading spaces
Field = 1,
Quoted = 2, // do not take comma for split in quote
}
fn log_field_comma(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, c: &str, e: &str,
) -> Result<(), JsonError> {
for h in &ctx.headers[..ctx.main_headers_nb] {
if mime::slice_equals_lowercase(&h.name, e.as_bytes()) {
let mark = js.get_mark();
let mut has_not_empty_field = false;
js.open_array(c)?;
let mut start = 0;
let mut state = FieldCommaState::Start;
for i in 0..h.value.len() {
match state {
FieldCommaState::Start => {
if h.value[i] == b' ' || h.value[i] == b'\t' {
start += 1;
} else if h.value[i] == b'"' {
state = FieldCommaState::Quoted;
} else {
state = FieldCommaState::Field;
}
}
FieldCommaState::Field => {
if h.value[i] == b',' {
if i > start {
js.append_string(&String::from_utf8_lossy(&h.value[start..i]))?;
has_not_empty_field = true;
}
start = i + 1;
state = FieldCommaState::Start;
} else if h.value[i] == b'"' {
state = FieldCommaState::Quoted;
}
}
FieldCommaState::Quoted => {
if h.value[i] == b'"' {
state = FieldCommaState::Field;
}
}
}
}
if h.value.len() > start {
// do not log empty string
js.append_string(&String::from_utf8_lossy(&h.value[start..]))?;
has_not_empty_field = true;
}
if has_not_empty_field {
js.close()?;
} else {
js.restore_mark(&mark)?;
}
break;
}
}
return Ok(());
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpLogFieldComma(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, email: *const std::os::raw::c_char,
config: *const std::os::raw::c_char,
) -> bool {
let e: &CStr = CStr::from_ptr(email); //unsafe
if let Ok(email_field) = e.to_str() {
let c: &CStr = CStr::from_ptr(config); //unsafe
if let Ok(config_field) = c.to_str() {
return log_field_comma(js, ctx, config_field, email_field).is_ok();
}
}
return false;
}
fn log_field_string(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, c: &str, e: &str,
) -> Result<(), JsonError> {
for h in &ctx.headers[..ctx.main_headers_nb] {
if mime::slice_equals_lowercase(&h.name, e.as_bytes()) {
js.set_string(c, &String::from_utf8_lossy(&h.value))?;
break;
}
}
return Ok(());
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpLogFieldString(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, email: *const std::os::raw::c_char,
config: *const std::os::raw::c_char,
) -> bool {
let e: &CStr = CStr::from_ptr(email); //unsafe
if let Ok(email_field) = e.to_str() {
let c: &CStr = CStr::from_ptr(config); //unsafe
if let Ok(config_field) = c.to_str() {
return log_field_string(js, ctx, config_field, email_field).is_ok();
}
}
return false;
}
fn log_data_header(
js: &mut JsonBuilder, ctx: &mut MimeStateSMTP, hname: &str,
) -> Result<(), JsonError> {
for h in &ctx.headers[..ctx.main_headers_nb] {
if mime::slice_equals_lowercase(&h.name, hname.as_bytes()) {
js.set_string(hname, &String::from_utf8_lossy(&h.value))?;
break;
}
}
return Ok(());
}
fn log_data(js: &mut JsonBuilder, ctx: &mut MimeStateSMTP) -> Result<(), JsonError> {
log_data_header(js, ctx, "from")?;
log_field_comma(js, ctx, "to", "to")?;
log_field_comma(js, ctx, "cc", "cc")?;
js.set_string("status", "PARSE_DONE")?;
if !ctx.attachments.is_empty() {
js.open_array("attachment")?;
for a in &ctx.attachments {
js.append_string(&String::from_utf8_lossy(a))?;
}
js.close()?;
}
if !ctx.urls.is_empty() {
js.open_array("url")?;
for a in ctx.urls.iter().rev() {
js.append_string(&String::from_utf8_lossy(a))?;
}
js.close()?;
}
return Ok(());
}
#[no_mangle]
pub unsafe extern "C" fn SCMimeSmtpLogData(js: &mut JsonBuilder, ctx: &mut MimeStateSMTP) -> bool {
return log_data(js, ctx).is_ok();
}

2
src/Makefile.am
Unescape Escape View File

@ -521,7 +521,6 @@ noinst_HEADERS = \
util-datalink.h \
util-debug-filters.h \
util-debug.h \
util-decode-mime.h \
util-detect.h \
util-device.h \
util-dpdk.h \
@ -1117,7 +1116,6 @@ libsuricata_c_a_SOURCES = \
util-datalink.c \
util-debug.c \
util-debug-filters.c \
util-decode-mime.c \
util-detect.c \
util-device.c \
util-dpdk.c \

11
src/app-layer-htp.c
Unescape Escape View File

@ -373,7 +373,7 @@ static void HtpTxUserDataFree(HtpState *state, HtpTxUserData *htud)
HTPFree(htud->response_headers_raw, htud->response_headers_raw_len);
AppLayerDecoderEventsFreeEvents(&htud->tx_data.events);
if (htud->mime_state)
rs_mime_state_free(htud->mime_state);
SCMimeStateFree(htud->mime_state);
if (htud->tx_data.de_state != NULL) {
DetectEngineStateFree(htud->tx_data.de_state);
}
@ -1136,7 +1136,7 @@ static int HtpRequestBodySetupMultipart(htp_tx_t *tx, HtpTxUserData *htud)
htp_header_t *h = (htp_header_t *)htp_table_get_c(tx->request_headers,
"Content-Type");
if (h != NULL && bstr_len(h->value) > 0) {
htud->mime_state = rs_mime_state_init(bstr_ptr(h->value), bstr_len(h->value));
htud->mime_state = SCMimeStateInit(bstr_ptr(h->value), bstr_len(h->value));
if (htud->mime_state) {
htud->tsflags |= HTP_BOUNDARY_SET;
SCReturnInt(1);
@ -1211,7 +1211,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
// keep parsing mime and use callbacks when needed
while (cur_buf_len > 0) {
MimeParserResult r =
rs_mime_parse(htud->mime_state, cur_buf, cur_buf_len, &consumed, &warnings);
SCMimeParse(htud->mime_state, cur_buf, cur_buf_len, &consumed, &warnings);
DEBUG_VALIDATE_BUG_ON(consumed > cur_buf_len);
htud->request_body.body_parsed += consumed;
if (warnings) {
@ -1230,7 +1230,7 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
goto end;
case MimeFileOpen:
// get filename owned by mime state
rs_mime_state_get_filename(htud->mime_state, &filename, &filename_len);
SCMimeStateGetFilename(htud->mime_state, &filename, &filename_len);
if (filename_len > 0) {
htud->tsflags |= HTP_FILENAME_SET;
htud->tsflags &= ~HTP_DONTSTORE;
@ -1268,7 +1268,6 @@ static int HtpRequestBodyHandleMultipart(HtpState *hstate, HtpTxUserData *htud,
}
htud->tsflags &= ~HTP_FILENAME_SET;
break;
// TODO event on parsing error ?
}
cur_buf += consumed;
cur_buf_len -= consumed;
@ -5609,7 +5608,7 @@ static int HTPBodyReassemblyTest01(void)
printf("REASSCHUNK END: \n");
#endif
htud.mime_state = rs_mime_state_init((const uint8_t *)"multipart/form-data; boundary=toto",
htud.mime_state = SCMimeStateInit((const uint8_t *)"multipart/form-data; boundary=toto",
strlen("multipart/form-data; boundary=toto"));
FAIL_IF_NULL(htud.mime_state);
htud.tsflags |= HTP_BOUNDARY_SET;

398
src/app-layer-smtp.c
Unescape Escape View File

@ -137,7 +137,6 @@ SCEnumCharMap smtp_decoder_event_table[] = {
/* MIME Events */
{ "MIME_PARSE_FAILED", SMTP_DECODER_EVENT_MIME_PARSE_FAILED },
{ "MIME_MALFORMED_MSG", SMTP_DECODER_EVENT_MIME_MALFORMED_MSG },
{ "MIME_INVALID_BASE64", SMTP_DECODER_EVENT_MIME_INVALID_BASE64 },
{ "MIME_INVALID_QP", SMTP_DECODER_EVENT_MIME_INVALID_QP },
{ "MIME_LONG_LINE", SMTP_DECODER_EVENT_MIME_LONG_LINE },
@ -233,15 +232,6 @@ SCEnumCharMap smtp_reply_map[ ] = {
/* Create SMTP config structure */
SMTPConfig smtp_config = {
.decode_mime = true,
{
.decode_base64 = true,
.decode_quoted_printable = true,
.extract_urls = true,
.extract_urls_schemes = NULL,
.log_url_scheme = false,
.body_md5 = false,
.header_value_depth = 0,
},
.content_limit = FILEDATA_CONTENT_LIMIT,
.content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE,
.content_inspect_window = FILEDATA_CONTENT_INSPECT_WINDOW,
@ -251,6 +241,8 @@ SMTPConfig smtp_config = {
static SMTPString *SMTPStringAlloc(void);
#define SCHEME_SUFFIX_LEN 3
/**
* \brief Configure SMTP Mime Decoder by parsing out mime section of YAML
* config file
@ -277,22 +269,25 @@ static void SMTPConfigure(void) {
ret = ConfGetChildValueBool(config, "decode-base64", &val);
if (ret) {
smtp_config.mime_config.decode_base64 = val;
SCMimeSmtpConfigDecodeBase64(val);
}
ret = ConfGetChildValueBool(config, "decode-quoted-printable", &val);
if (ret) {
smtp_config.mime_config.decode_quoted_printable = val;
SCMimeSmtpConfigDecodeQuoted(val);
}
ret = ConfGetChildValueInt(config, "header-value-depth", &imval);
if (ret) {
smtp_config.mime_config.header_value_depth = (uint32_t) imval;
if (imval < 0 || imval > UINT32_MAX) {
FatalError("Invalid value for header-value-depth");
}
SCMimeSmtpConfigHeaderValueDepth((uint32_t)imval);
}
ret = ConfGetChildValueBool(config, "extract-urls", &val);
if (ret) {
smtp_config.mime_config.extract_urls = val;
SCMimeSmtpConfigExtractUrls(val);
}
/* Parse extract-urls-schemes from mime config, add '://' suffix to found schemes,
@ -302,75 +297,48 @@ static void SMTPConfigure(void) {
if (extract_urls_schemes) {
ConfNode *scheme = NULL;
SCMimeSmtpConfigExtractUrlsSchemeReset();
TAILQ_FOREACH (scheme, &extract_urls_schemes->head, next) {
/* new_val_len: scheme value from config e.g. 'http' + '://' + null terminator */
size_t new_val_len = strlen(scheme->val) + 3 + 1;
if (new_val_len > UINT16_MAX) {
size_t scheme_len = strlen(scheme->val);
if (scheme_len > UINT16_MAX - SCHEME_SUFFIX_LEN) {
FatalError("Too long value for extract-urls-schemes");
}
char *new_val = SCMalloc(new_val_len);
if (unlikely(new_val == NULL)) {
FatalError("SCMalloc failure.");
if (scheme->val[scheme_len - 1] != '/') {
scheme_len += SCHEME_SUFFIX_LEN;
char *new_val = SCMalloc(scheme_len + 1);
if (unlikely(new_val == NULL)) {
FatalError("SCMalloc failure.");
}
int r = snprintf(new_val, scheme_len + 1, "%s://", scheme->val);
if (r != (int)scheme_len) {
FatalError("snprintf failure for SMTP url extraction scheme.");
}
SCFree(scheme->val);
scheme->val = new_val;
}
int r = snprintf(new_val, new_val_len, "%s://", scheme->val);
if (r < 0 || r >= (int)new_val_len) {
FatalError("snprintf failure.");
int r = SCMimeSmtpConfigExtractUrlsSchemeAdd(scheme->val);
if (r < 0) {
FatalError("Failed to add smtp extract url scheme");
}
/* replace existing scheme value stored on the linked list with new value including
* '://' suffix */
SCFree(scheme->val);
scheme->val = new_val;
}
smtp_config.mime_config.extract_urls_schemes = extract_urls_schemes;
} else {
/* Add default extract url scheme 'http' since
* extract-urls-schemes wasn't found in the config */
ConfNode *seq_node = ConfNodeNew();
if (unlikely(seq_node == NULL)) {
FatalError("ConfNodeNew failure.");
}
ConfNode *scheme = ConfNodeNew();
if (unlikely(scheme == NULL)) {
FatalError("ConfNodeNew failure.");
}
seq_node->name = SCStrdup("extract-urls-schemes");
if (unlikely(seq_node->name == NULL)) {
FatalError("SCStrdup failure.");
}
scheme->name = SCStrdup("0");
if (unlikely(scheme->name == NULL)) {
FatalError("SCStrdup failure.");
}
scheme->val = SCStrdup("http://");
if (unlikely(scheme->val == NULL)) {
FatalError("SCStrdup failure.");
}
seq_node->is_seq = 1;
TAILQ_INSERT_TAIL(&seq_node->head, scheme, next);
TAILQ_INSERT_TAIL(&config->head, seq_node, next);
smtp_config.mime_config.extract_urls_schemes = seq_node;
SCMimeSmtpConfigExtractUrlsSchemeReset();
SCMimeSmtpConfigExtractUrlsSchemeAdd("http://");
}
ret = ConfGetChildValueBool(config, "log-url-scheme", &val);
if (ret) {
smtp_config.mime_config.log_url_scheme = val;
SCMimeSmtpConfigLogUrlScheme(val);
}
ret = ConfGetChildValueBool(config, "body-md5", &val);
if (ret) {
smtp_config.mime_config.body_md5 = val;
SCMimeSmtpConfigBodyMd5(val);
}
}
/* Pass mime config data to MimeDec API */
MimeDecSetConfig(&smtp_config.mime_config);
ConfNode *t = ConfGetNode("app-layer.protocols.smtp.inspected-tracker");
ConfNode *p = NULL;
@ -453,7 +421,6 @@ static SMTPTransaction *SMTPTransactionCreate(SMTPState *state)
}
TAILQ_INIT(&tx->rcpt_to_list);
tx->mime_state = NULL;
tx->tx_data.file_tx = STREAM_TOSERVER; // can xfer files
return tx;
}
@ -491,151 +458,6 @@ static void SMTPNewFile(SMTPTransaction *tx, File *file)
smtp_config.content_inspect_min_size);
}
int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len,
MimeDecParseState *state)
{
SCEnter();
int ret = MIME_DEC_OK;
Flow *flow = (Flow *) state->data;
SMTPState *smtp_state = (SMTPState *) flow->alstate;
SMTPTransaction *tx = smtp_state->curr_tx;
MimeDecEntity *entity = (MimeDecEntity *) state->stack->top->data;
FileContainer *files = NULL;
DEBUG_VALIDATE_BUG_ON(tx == NULL);
uint16_t flags = FileFlowToFlags(flow, STREAM_TOSERVER);
/* Find file */
if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
files = &tx->files_ts;
/* Open file if necessary */
if (state->body_begin) {
#ifdef DEBUG
if (SCLogDebugEnabled()) {
SCLogDebug("Opening file...%u bytes", len);
printf("File - ");
for (uint32_t i = 0; i < entity->filename_len; i++) {
printf("%c", entity->filename[i]);
}
printf("\n");
}
#endif
/* Set storage flag if applicable since only the first file in the
* flow seems to be processed by the 'filestore' detector */
if (files->head != NULL && (files->head->flags & FILE_STORE)) {
flags |= FILE_STORE;
}
uint32_t depth = smtp_config.content_inspect_min_size +
(smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp);
SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %"PRIu32, depth);
StreamTcpReassemblySetMinInspectDepth(flow->protoctx, STREAM_TOSERVER, depth);
uint16_t flen = (uint16_t)entity->filename_len;
if (entity->filename_len > SC_FILENAME_MAX) {
flen = SC_FILENAME_MAX;
SMTPSetEvent(smtp_state, SMTP_DECODER_EVENT_MIME_LONG_FILENAME);
}
if (FileOpenFileWithId(files, &smtp_config.sbcfg, smtp_state->file_track_id++,
(uint8_t *)entity->filename, flen, (uint8_t *)chunk, len, flags) != 0) {
ret = MIME_DEC_ERR_DATA;
SCLogDebug("FileOpenFile() failed");
} else {
SMTPNewFile(tx, files->tail);
}
/* If close in the same chunk, then pass in empty bytes */
if (state->body_end) {
SCLogDebug("Closing file...%u bytes", len);
if (files->tail->state == FILE_STATE_OPENED) {
ret = FileCloseFile(files, &smtp_config.sbcfg, (uint8_t *)NULL, 0, flags);
if (ret != 0) {
SCLogDebug("FileCloseFile() failed: %d", ret);
ret = MIME_DEC_ERR_DATA;
}
} else {
SCLogDebug("File already closed");
}
depth = smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp;
AppLayerParserTriggerRawStreamReassembly(flow, STREAM_TOSERVER);
SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u",
depth);
StreamTcpReassemblySetMinInspectDepth(flow->protoctx, STREAM_TOSERVER,
depth);
}
} else if (state->body_end) {
/* Close file */
SCLogDebug("Closing file...%u bytes", len);
if (files->tail && files->tail->state == FILE_STATE_OPENED) {
ret = FileCloseFile(files, &smtp_config.sbcfg, (uint8_t *)chunk, len, flags);
if (ret != 0) {
SCLogDebug("FileCloseFile() failed: %d", ret);
ret = MIME_DEC_ERR_DATA;
}
} else {
SCLogDebug("File already closed");
}
uint32_t depth = smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp;
AppLayerParserTriggerRawStreamReassembly(flow, STREAM_TOSERVER);
SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u",
depth);
StreamTcpReassemblySetMinInspectDepth(flow->protoctx,
STREAM_TOSERVER, depth);
} else {
/* Append data chunk to file */
SCLogDebug("Appending file...%u bytes", len);
/* 0 is ok, -2 is not stored, -1 is error */
ret = FileAppendData(files, &smtp_config.sbcfg, (uint8_t *)chunk, len);
if (ret == -2) {
ret = 0;
SCLogDebug("FileAppendData() - file no longer being extracted");
} else if (ret < 0) {
SCLogDebug("FileAppendData() failed: %d", ret);
ret = MIME_DEC_ERR_DATA;
}
if (files->tail && files->tail->content_inspected == 0 &&
files->tail->size >= smtp_config.content_inspect_min_size) {
uint32_t depth = smtp_config.content_inspect_min_size +
(smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp);
AppLayerParserTriggerRawStreamReassembly(flow, STREAM_TOSERVER);
SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u",
depth);
StreamTcpReassemblySetMinInspectDepth(flow->protoctx,
STREAM_TOSERVER, depth);
/* after the start of the body inspection, disable the depth logic */
} else if (files->tail && files->tail->content_inspected > 0) {
StreamTcpReassemblySetMinInspectDepth(flow->protoctx,
STREAM_TOSERVER, 0);
/* expand the limit as long as we get file data, as the file data is bigger on the
* wire due to base64 */
} else {
uint32_t depth = smtp_config.content_inspect_min_size +
(smtp_state->toserver_data_count - smtp_state->toserver_last_data_stamp);
SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %"PRIu32,
depth);
StreamTcpReassemblySetMinInspectDepth(flow->protoctx,
STREAM_TOSERVER, depth);
}
}
if (ret == 0) {
SCLogDebug("Successfully processed file data!");
}
} else {
SCLogDebug("Body not a Ctnt_attachment");
}
SCReturnInt(ret);
}
/**
* \internal
* \brief Get the next line from input. It doesn't do any length validation.
@ -770,39 +592,34 @@ static int SMTPProcessCommandBDAT(
SCReturnInt(0);
}
static void SetMimeEvents(SMTPState *state)
static void SetMimeEvents(SMTPState *state, uint32_t events)
{
if (state->curr_tx->mime_state->msg == NULL) {
if (events == 0) {
return;
}
/* Generate decoder events */
MimeDecEntity *msg = state->curr_tx->mime_state->msg;
if (msg->anomaly_flags & ANOM_INVALID_BASE64) {
if (events & MIME_ANOM_INVALID_BASE64) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_INVALID_BASE64);
}
if (msg->anomaly_flags & ANOM_INVALID_QP) {
if (events & MIME_ANOM_INVALID_QP) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_INVALID_QP);
}
if (msg->anomaly_flags & ANOM_LONG_LINE) {
if (events & MIME_ANOM_LONG_LINE) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_LINE);
}
if (msg->anomaly_flags & ANOM_LONG_ENC_LINE) {
if (events & MIME_ANOM_LONG_ENC_LINE) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_ENC_LINE);
}
if (msg->anomaly_flags & ANOM_LONG_HEADER_NAME) {
if (events & MIME_ANOM_LONG_HEADER_NAME) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_HEADER_NAME);
}
if (msg->anomaly_flags & ANOM_LONG_HEADER_VALUE) {
if (events & MIME_ANOM_LONG_HEADER_VALUE) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_HEADER_VALUE);
}
if (msg->anomaly_flags & ANOM_MALFORMED_MSG) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_MALFORMED_MSG);
}
if (msg->anomaly_flags & ANOM_LONG_BOUNDARY) {
if (events & MIME_ANOM_LONG_BOUNDARY) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_BOUNDARY_TOO_LONG);
}
if (msg->anomaly_flags & ANOM_LONG_FILENAME) {
if (events & MIME_ANOM_LONG_FILENAME) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_FILENAME);
}
}
@ -841,14 +658,11 @@ static int SMTPProcessCommandDATA(SMTPState *state, SMTPTransaction *tx, Flow *f
FileCloseFile(&tx->files_ts, &smtp_config.sbcfg, NULL, 0, 0);
} else if (smtp_config.decode_mime && tx->mime_state != NULL) {
/* Complete parsing task */
int ret = MimeDecParseComplete(tx->mime_state);
if (ret != MIME_DEC_OK) {
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_PARSE_FAILED);
SCLogDebug("MimeDecParseComplete() function failed");
SCSmtpMimeComplete(tx->mime_state);
if (tx->files_ts.tail && tx->files_ts.tail->state == FILE_STATE_OPENED) {
FileCloseFile(&tx->files_ts, &smtp_config.sbcfg, NULL, 0,
FileFlowToFlags(f, STREAM_TOSERVER));
}
/* Generate decoder events */
SetMimeEvents(state);
}
SMTPTransactionComplete(state);
SCLogDebug("marked tx as done");
@ -863,17 +677,78 @@ static int SMTPProcessCommandDATA(SMTPState *state, SMTPTransaction *tx, Flow *f
(state->parser_state & SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
if (smtp_config.decode_mime && tx->mime_state != NULL) {
int ret = MimeDecParseLine(line->buf, line->len, line->delim_len, tx->mime_state);
if (ret != MIME_DEC_OK) {
if (ret != MIME_DEC_ERR_STATE) {
/* Generate decoder events */
SetMimeEvents(state);
SCLogDebug("MimeDecParseLine() function returned an error code: %d", ret);
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_PARSE_FAILED);
}
/* keep the parser in its error state so we can log that,
* the parser will reject new data */
uint32_t events;
uint16_t flags = FileFlowToFlags(f, STREAM_TOSERVER);
const uint8_t *filename = NULL;
uint16_t filename_len = 0;
uint32_t depth;
/* we depend on detection engine for file pruning */
flags |= FILE_USE_DETECT;
MimeSmtpParserResult ret = SCSmtpMimeParseLine(
line->buf, line->len, line->delim_len, &events, tx->mime_state);
SetMimeEvents(state, events);
switch (ret) {
case MimeSmtpFileOpen:
// get filename owned by mime state
SCMimeSmtpGetFilename(state->curr_tx->mime_state, &filename, &filename_len);
if (filename_len == 0) {
// not an attachment
break;
}
depth = smtp_config.content_inspect_min_size +
(state->toserver_data_count - state->toserver_last_data_stamp);
SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %" PRIu32,
depth);
StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, depth);
if (filename_len > SC_FILENAME_MAX) {
filename_len = SC_FILENAME_MAX;
SMTPSetEvent(state, SMTP_DECODER_EVENT_MIME_LONG_FILENAME);
}
if (FileOpenFileWithId(&tx->files_ts, &smtp_config.sbcfg,
state->file_track_id++, filename, filename_len, NULL, 0,
flags) != 0) {
SCLogDebug("FileOpenFile() failed");
}
SMTPNewFile(state->curr_tx, tx->files_ts.tail);
break;
case MimeSmtpFileChunk:
// rust already run FileAppendData
if (tx->files_ts.tail && tx->files_ts.tail->content_inspected == 0 &&
tx->files_ts.tail->size >= smtp_config.content_inspect_min_size) {
depth = smtp_config.content_inspect_min_size +
(state->toserver_data_count - state->toserver_last_data_stamp);
AppLayerParserTriggerRawStreamReassembly(f, STREAM_TOSERVER);
SCLogDebug(
"StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u", depth);
StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, depth);
/* after the start of the body inspection, disable the depth logic */
} else if (tx->files_ts.tail && tx->files_ts.tail->content_inspected > 0) {
StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, 0);
/* expand the limit as long as we get file data, as the file data is bigger
* on the wire due to base64 */
} else {
depth = smtp_config.content_inspect_min_size +
(state->toserver_data_count - state->toserver_last_data_stamp);
SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %" PRIu32,
depth);
StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, depth);
}
break;
case MimeSmtpFileClose:
if (tx->files_ts.tail && tx->files_ts.tail->state == FILE_STATE_OPENED) {
if (FileCloseFile(&tx->files_ts, &smtp_config.sbcfg, NULL, 0, flags) != 0) {
SCLogDebug("FileCloseFile() failed: %d", ret);
}
} else {
SCLogDebug("File already closed");
}
depth = state->toserver_data_count - state->toserver_last_data_stamp;
AppLayerParserTriggerRawStreamReassembly(f, STREAM_TOSERVER);
SCLogDebug("StreamTcpReassemblySetMinInspectDepth STREAM_TOSERVER %u", depth);
StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER, depth);
}
}
}
@ -1169,8 +1044,8 @@ static int NoNewTx(SMTPState *state, const SMTPLine *line)
* -1 for errors and inconsistent states
* -2 if MIME state could not be allocated
* */
static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *pstate,
SMTPInput *input, const SMTPLine *line)
static int SMTPProcessRequest(
SMTPState *state, Flow *f, AppLayerParserState *pstate, const SMTPLine *line)
{
SCEnter();
SMTPTransaction *tx = state->curr_tx;
@ -1224,19 +1099,11 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps
}
} else if (smtp_config.decode_mime) {
DEBUG_VALIDATE_BUG_ON(tx->mime_state);
tx->mime_state = MimeDecInitParser(f, SMTPProcessDataChunk);
tx->mime_state = SCMimeSmtpStateInit(&tx->files_ts, &smtp_config.sbcfg);
if (tx->mime_state == NULL) {
return MIME_DEC_ERR_MEM;
}
/* Add new MIME message to end of list */
if (tx->msg_head == NULL) {
tx->msg_head = tx->mime_state->msg;
tx->msg_tail = tx->mime_state->msg;
}
else {
tx->msg_tail->next = tx->mime_state->msg;
tx->msg_tail = tx->mime_state->msg;
SCLogDebug("MimeDecInitParser() failed to "
"allocate data");
return -1;
}
}
state->curr_tx->is_data = true;
@ -1343,7 +1210,7 @@ static int SMTPPreProcessCommands(
/* fall back to strict line parsing for mime header parsing */
if (state->curr_tx && state->curr_tx->mime_state &&
state->curr_tx->mime_state->state_flag < HEADER_DONE)
SCMimeSmtpGetState(state->curr_tx->mime_state) < MimeSmtpBody)
return 1;
bool line_complete = false;
@ -1387,7 +1254,7 @@ static int SMTPPreProcessCommands(
input->consumed = total_consumed;
input->len -= current_line_consumed;
DEBUG_VALIDATE_BUG_ON(input->consumed + input->len != input->orig_len);
if (SMTPProcessRequest(state, f, pstate, input, line) == -1) {
if (SMTPProcessRequest(state, f, pstate, line) == -1) {
return -1;
}
line_complete = false;
@ -1438,7 +1305,7 @@ static AppLayerResult SMTPParse(uint8_t direction, Flow *f, SMTPState *state,
}
AppLayerResult res = SMTPGetLine(state, &input, &line, direction);
while (res.status == 0) {
int retval = SMTPProcessRequest(state, f, pstate, &input, &line);
int retval = SMTPProcessRequest(state, f, pstate, &line);
if (retval != 0)
SCReturnStruct(APP_LAYER_ERROR);
if (line.delim_len == 0 && line.len == SMTP_LINE_BUFFER_LIMIT) {
@ -1596,10 +1463,8 @@ static void SMTPLocalStorageFree(void *ptr)
static void SMTPTransactionFree(SMTPTransaction *tx, SMTPState *state)
{
if (tx->mime_state != NULL) {
MimeDecDeInitParser(tx->mime_state);
SCMimeSmtpStateFree(tx->mime_state);
}
/* Free list of MIME message recursively */
MimeDecFreeEntity(tx->msg_head);
if (tx->tx_data.events != NULL)
AppLayerDecoderEventsFreeEvents(&tx->tx_data.events);
@ -1915,8 +1780,6 @@ void SMTPParserCleanup(void)
static void SMTPTestInitConfig(void)
{
MimeDecSetConfig(&smtp_config.mime_config);
smtp_config.content_limit = FILEDATA_CONTENT_LIMIT;
smtp_config.content_inspect_window = FILEDATA_CONTENT_INSPECT_WINDOW;
smtp_config.content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE;
@ -4078,9 +3941,8 @@ static int SMTPParserTest14(void)
/* Enable mime decoding */
smtp_config.decode_mime = true;
smtp_config.mime_config.decode_base64 = true;
smtp_config.mime_config.decode_quoted_printable = true;
MimeDecSetConfig(&smtp_config.mime_config);
SCMimeSmtpConfigDecodeBase64(1);
SCMimeSmtpConfigDecodeQuoted(1);
/* DATA request */
r = AppLayerParserParse(NULL, alp_tctx, &f, ALPROTO_SMTP,
@ -4121,7 +3983,6 @@ static int SMTPParserTest14(void)
if (smtp_state->cmds_cnt != 0 || smtp_state->cmds_idx != 0 ||
smtp_state->curr_tx->mime_state == NULL ||
smtp_state->curr_tx->msg_head == NULL || /* MIME data structures */
smtp_state->parser_state !=
(SMTP_PARSER_STATE_FIRST_REPLY_SEEN | SMTP_PARSER_STATE_COMMAND_DATA_MODE)) {
printf("smtp parser in inconsistent state l.%d\n", __LINE__);
@ -4139,7 +4000,6 @@ static int SMTPParserTest14(void)
if (smtp_state->cmds_cnt != 1 || smtp_state->cmds_idx != 0 ||
smtp_state->cmds[0] != SMTP_COMMAND_DATA_MODE ||
smtp_state->curr_tx->mime_state == NULL ||
smtp_state->curr_tx->msg_head == NULL || /* MIME data structures */
smtp_state->parser_state != (SMTP_PARSER_STATE_FIRST_REPLY_SEEN)) {
printf("smtp parser in inconsistent state l.%d\n", __LINE__);
goto end;

14
src/app-layer-smtp.h
Unescape Escape View File

@ -24,7 +24,6 @@
#ifndef SURICATA_APP_LAYER_SMTP_H
#define SURICATA_APP_LAYER_SMTP_H
#include "util-decode-mime.h"
#include "util-streaming-buffer.h"
#include "rust.h"
@ -81,12 +80,8 @@ typedef struct SMTPTransaction_ {
// another DATA command within the same context
// will trigger an app-layer event.
bool is_data;
/** the first message contained in the session */
MimeDecEntity *msg_head;
/** the last message contained in the session */
MimeDecEntity *msg_tail;
/** the mime decoding parser state */
MimeDecParseState *mime_state;
MimeStateSMTP *mime_state;
/* MAIL FROM parameters */
uint8_t *mail_from;
@ -99,10 +94,14 @@ typedef struct SMTPTransaction_ {
TAILQ_ENTRY(SMTPTransaction_) next;
} SMTPTransaction;
/**
* \brief Structure for containing configuration options
*
*/
typedef struct SMTPConfig {
bool decode_mime;
MimeDecConfig mime_config;
uint32_t content_limit;
uint32_t content_inspect_min_size;
uint32_t content_inspect_window;
@ -158,7 +157,6 @@ typedef struct SMTPState_ {
/* Create SMTP config structure */
extern SMTPConfig smtp_config;
int SMTPProcessDataChunk(const uint8_t *chunk, uint32_t len, MimeDecParseState *state);
void *SMTPStateAlloc(void *orig_state, AppProto proto_orig);
void RegisterSMTPParsers(void);
void SMTPParserCleanup(void);

200
src/output-json-email-common.c
Unescape Escape View File

@ -83,82 +83,29 @@ struct {
{ NULL, NULL, LOG_EMAIL_DEFAULT},
};
static inline char *SkipWhiteSpaceTill(char *p, char *savep)
{
char *sp = p;
if (unlikely(p == NULL)) {
return NULL;
}
while (((*sp == '\t') || (*sp == ' ')) && (sp < savep)) {
sp++;
}
return sp;
}
static bool EveEmailJsonArrayFromCommaList(JsonBuilder *js, const uint8_t *val, size_t len)
{
char *savep = NULL;
char *p;
char *sp;
char *to_line = BytesToString((uint8_t *)val, len);
if (likely(to_line != NULL)) {
p = strtok_r(to_line, ",", &savep);
if (p == NULL) {
SCFree(to_line);
return false;
}
sp = SkipWhiteSpaceTill(p, savep);
jb_append_string(js, sp);
while ((p = strtok_r(NULL, ",", &savep)) != NULL) {
sp = SkipWhiteSpaceTill(p, savep);
jb_append_string(js, sp);
}
} else {
return false;
}
SCFree(to_line);
return true;
}
static void EveEmailLogJSONMd5(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx)
{
if (email_ctx->flags & LOG_EMAIL_SUBJECT_MD5) {
MimeDecEntity *entity = tx->msg_tail;
MimeStateSMTP *entity = tx->mime_state;
if (entity == NULL) {
return;
}
MimeDecField *field = MimeDecFindField(entity, "subject");
if (field != NULL) {
char smd5[SC_MD5_HEX_LEN + 1];
SCMd5HashBufferToHex((uint8_t *)field->value, field->value_len, smd5, sizeof(smd5));
jb_set_string(js, "subject_md5", smd5);
}
SCMimeSmtpLogSubjectMd5(js, entity);
}
if (email_ctx->flags & LOG_EMAIL_BODY_MD5) {
MimeDecParseState *mime_state = tx->mime_state;
if (mime_state && mime_state->has_md5 && (mime_state->state_flag == PARSE_DONE)) {
jb_set_hex(js, "body_md5", mime_state->md5, (uint32_t)sizeof(mime_state->md5));
MimeStateSMTP *entity = tx->mime_state;
if (entity == NULL) {
return;
}
SCMimeSmtpLogBodyMd5(js, entity);
}
}
static int JsonEmailAddToJsonArray(const uint8_t *val, size_t len, void *data)
{
JsonBuilder *ajs = data;
if (ajs == NULL)
return 0;
jb_append_string_from_bytes(ajs, val, (uint32_t)len);
return 1;
}
static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js, SMTPTransaction *tx)
{
int f = 0;
JsonBuilderMark mark = { 0, 0, 0 };
MimeDecField *field;
MimeDecEntity *entity = tx->msg_tail;
MimeStateSMTP *entity = tx->mime_state;
if (entity == NULL) {
return;
}
@ -169,31 +116,14 @@ static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js
((email_ctx->flags & LOG_EMAIL_EXTENDED) && (email_fields[f].flags & LOG_EMAIL_EXTENDED))
) {
if (email_fields[f].flags & LOG_EMAIL_ARRAY) {
jb_get_mark(js, &mark);
jb_open_array(js, email_fields[f].config_field);
int found = MimeDecFindFieldsForEach(entity, email_fields[f].email_field, JsonEmailAddToJsonArray, js);
if (found > 0) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
}
SCMimeSmtpLogFieldArray(
js, entity, email_fields[f].email_field, email_fields[f].config_field);
} else if (email_fields[f].flags & LOG_EMAIL_COMMA) {
field = MimeDecFindField(entity, email_fields[f].email_field);
if (field) {
jb_get_mark(js, &mark);
jb_open_array(js, email_fields[f].config_field);
if (EveEmailJsonArrayFromCommaList(js, field->value, field->value_len)) {
jb_close(js);
} else {
jb_restore_mark(js, &mark);
}
}
SCMimeSmtpLogFieldComma(
js, entity, email_fields[f].email_field, email_fields[f].config_field);
} else {
field = MimeDecFindField(entity, email_fields[f].email_field);
if (field != NULL) {
jb_set_string_from_bytes(
js, email_fields[f].config_field, field->value, field->value_len);
}
SCMimeSmtpLogFieldString(
js, entity, email_fields[f].email_field, email_fields[f].config_field);
}
}
@ -205,9 +135,7 @@ static void EveEmailLogJSONCustom(OutputJsonEmailCtx *email_ctx, JsonBuilder *js
static bool EveEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t tx_id, JsonBuilder *sjs)
{
SMTPState *smtp_state;
MimeDecParseState *mime_state;
MimeDecEntity *entity;
JsonBuilderMark mark = { 0, 0, 0 };
MimeStateSMTP *mime_state;
/* check if we have SMTP state or not */
AppProto proto = FlowGetAppProtocol(f);
@ -221,110 +149,14 @@ static bool EveEmailLogJsonData(const Flow *f, void *state, void *vtx, uint64_t
}
SMTPTransaction *tx = vtx;
mime_state = tx->mime_state;
entity = tx->msg_tail;
SCLogDebug("lets go mime_state %p, entity %p, state_flag %u", mime_state, entity, mime_state ? mime_state->state_flag : 0);
SCLogDebug("lets go mime_state %p", mime_state);
break;
default:
/* don't know how we got here */
SCReturnBool(false);
}
if ((mime_state != NULL)) {
if (entity == NULL) {
SCReturnBool(false);
}
jb_set_string(sjs, "status", MimeDecParseStateGetStatus(mime_state));
MimeDecField *field;
/* From: */
field = MimeDecFindField(entity, "from");
if (field != NULL) {
char *s = BytesToString((uint8_t *)field->value,
(size_t)field->value_len);
if (likely(s != NULL)) {
//printf("From: \"%s\"\n", s);
char * sp = SkipWhiteSpaceTill(s, s + strlen(s));
jb_set_string(sjs, "from", sp);
SCFree(s);
}
}
/* To: */
field = MimeDecFindField(entity, "to");
if (field != NULL) {
jb_get_mark(sjs, &mark);
jb_open_array(sjs, "to");
if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) {
jb_close(sjs);
} else {
jb_restore_mark(sjs, &mark);
}
}
/* Cc: */
field = MimeDecFindField(entity, "cc");
if (field != NULL) {
jb_get_mark(sjs, &mark);
jb_open_array(sjs, "cc");
if (EveEmailJsonArrayFromCommaList(sjs, field->value, field->value_len)) {
jb_close(sjs);
} else {
jb_restore_mark(sjs, &mark);
}
}
if (mime_state->stack == NULL || mime_state->stack->top == NULL || mime_state->stack->top->data == NULL) {
SCReturnBool(false);
}
entity = (MimeDecEntity *)mime_state->stack->top->data;
int attach_cnt = 0;
int url_cnt = 0;
JsonBuilder *js_attach = jb_new_array();
JsonBuilder *js_url = jb_new_array();
if (entity->url_list != NULL) {
MimeDecUrl *url;
bool has_ipv6_url = false;
bool has_ipv4_url = false;
bool has_exe_url = false;
for (url = entity->url_list; url != NULL; url = url->next) {
jb_append_string_from_bytes(js_url, url->url, url->url_len);
if (url->url_flags & URL_IS_EXE)
has_exe_url = true;
if (url->url_flags & URL_IS_IP6)
has_ipv6_url = true;
if (url->url_flags & URL_IS_IP4)
has_ipv6_url = true;
url_cnt += 1;
}
jb_set_bool(sjs, "has_ipv6_url", has_ipv6_url);
jb_set_bool(sjs, "has_ipv4_url", has_ipv4_url);
jb_set_bool(sjs, "has_exe_url", has_exe_url);
}
for (entity = entity->child; entity != NULL; entity = entity->next) {
if (entity->ctnt_flags & CTNT_IS_ATTACHMENT) {
jb_append_string_from_bytes(js_attach, entity->filename, entity->filename_len);
attach_cnt += 1;
}
if (entity->url_list != NULL) {
MimeDecUrl *url;
for (url = entity->url_list; url != NULL; url = url->next) {
jb_append_string_from_bytes(js_url, url->url, url->url_len);
url_cnt += 1;
}
}
}
if (attach_cnt > 0) {
jb_close(js_attach);
jb_set_object(sjs, "attachment", js_attach);
}
jb_free(js_attach);
if (url_cnt > 0) {
jb_close(js_url);
jb_set_object(sjs, "url", js_url);
}
jb_free(js_url);
SCMimeSmtpLogData(sjs, mime_state);
SCReturnBool(true);
}

2
src/runmode-unittests.c
Unescape Escape View File

@ -79,6 +79,7 @@
#include "util-memcmp.h"
#include "util-misc.h"
#include "util-signal.h"
#include "util-base64.h"
#include "reputation.h"
#include "util-atomic.h"
@ -203,7 +204,6 @@ static void RegisterUnittests(void)
SCAtomicRegisterTests();
MemrchrRegisterTests();
AppLayerUnittestsRegister();
MimeDecRegisterTests();
StreamingBufferRegisterTests();
MacSetRegisterTests();
#ifdef OS_WIN32

33
src/tests/fuzz/fuzz_mimedecparseline.c
Unescape Escape View File

@ -7,22 +7,11 @@
#include "suricata-common.h"
#include "suricata.h"
#include "util-decode-mime.h"
#include "rust.h"
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
static int initialized = 0;
static int dummy = 0;
static int MimeParserDataFromFileCB(const uint8_t *chunk, uint32_t len,
MimeDecParseState *state)
{
if (len > 0 && chunk[len-1] == 0) {
// do not get optimized away
dummy++;
}
return MIME_DEC_OK;
}
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
{
@ -36,19 +25,19 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
initialized = 1;
}
uint32_t line_count = 0;
MimeDecParseState *state = MimeDecInitParser(&line_count, MimeParserDataFromFileCB);
MimeDecEntity *msg_head = state->msg;
uint32_t events;
FileContainer *files = FileContainerAlloc();
StreamingBufferConfig sbcfg = STREAMING_BUFFER_CONFIG_INITIALIZER;
MimeStateSMTP *state = SCMimeSmtpStateInit(files, &sbcfg);
const uint8_t * buffer = data;
while (1) {
uint8_t * next = memchr(buffer, '\n', size);
if (next == NULL) {
if (state->state_flag >= BODY_STARTED)
(void)MimeDecParseLine(buffer, size, 0, state);
if (SCMimeSmtpGetState(state) >= MimeSmtpBody)
(void)SCSmtpMimeParseLine(buffer, size, 0, &events, state);
break;
} else {
(void) MimeDecParseLine(buffer, next - buffer, 1, state);
(void)SCSmtpMimeParseLine(buffer, next - buffer, 1, &events, state);
if (buffer + size < next + 1) {
break;
}
@ -57,10 +46,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size)
}
}
/* Completed */
(void)MimeDecParseComplete(state);
(void)SCSmtpMimeComplete(state);
/* De Init parser */
MimeDecDeInitParser(state);
MimeDecFreeEntity(msg_head);
SCMimeSmtpStateFree(state);
FileContainerFree(files, &sbcfg);
return 0;
}

3590
src/util-decode-mime.c
View File

File diff suppressed because it is too large Load Diff

243
src/util-decode-mime.h
Unescape Escape View File

@ -1,243 +0,0 @@
/* Copyright (C) 2012 BAE Systems
* Copyright (C) 2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
/**
* \file
*
* \author David Abarbanel <david.abarbanel@baesystems.com>
*
*/
#ifndef MIME_DECODE_H_
#define MIME_DECODE_H_
#include "conf.h"
#include "util-base64.h"
#include "util-file.h"
/* Content Flags */
#define CTNT_IS_MSG 1
#define CTNT_IS_ENV 2
#define CTNT_IS_ENCAP 4
#define CTNT_IS_BODYPART 8
#define CTNT_IS_MULTIPART 16
#define CTNT_IS_ATTACHMENT 32
#define CTNT_IS_BASE64 64
#define CTNT_IS_QP 128
#define CTNT_IS_TEXT 256
#define CTNT_IS_HTML 512
/* URL Flags */
#define URL_IS_IP4 1
#define URL_IS_IP6 2
#define URL_IS_EXE 4
/* Anomaly Flags */
#define ANOM_INVALID_BASE64 1 /* invalid base64 chars */
#define ANOM_INVALID_QP 2 /* invalid quoted-printable chars */
#define ANOM_LONG_HEADER_NAME 4 /* header is abnormally long */
#define ANOM_LONG_HEADER_VALUE 8 /* header value is abnormally long
* (includes multi-line) */
#define ANOM_LONG_LINE 16 /* Lines that exceed 998 octets */
#define ANOM_LONG_ENC_LINE 32 /* Lines that exceed 76 octets */
#define ANOM_MALFORMED_MSG 64 /* Misc msg format errors found */
#define ANOM_LONG_BOUNDARY 128 /* Boundary too long */
#define ANOM_LONG_FILENAME 256 /* filename truncated */
/* Publicly exposed size constants */
#define DATA_CHUNK_SIZE 3072 /* Should be divisible by 3 */
/* Mime Parser Constants */
#define HEADER_READY 0x01
#define HEADER_STARTED 0x02
#define HEADER_DONE 0x03
#define BODY_STARTED 0x04
#define BODY_DONE 0x05
#define BODY_END_BOUND 0x06
#define PARSE_DONE 0x07
#define PARSE_ERROR 0x08
/**
* \brief Mime Decoder Error Codes
*/
typedef enum MimeDecRetCode {
MIME_DEC_OK = 0,
MIME_DEC_MORE = 1,
MIME_DEC_ERR_DATA = -1,
MIME_DEC_ERR_MEM = -2,
MIME_DEC_ERR_PARSE = -3,
MIME_DEC_ERR_STATE = -4, /**< parser in error state */
MIME_DEC_ERR_OVERFLOW = -5,
} MimeDecRetCode;
/**
* \brief Structure for containing configuration options
*
*/
typedef struct MimeDecConfig {
bool decode_base64; /**< Decode base64 bodies */
bool decode_quoted_printable; /**< Decode quoted-printable bodies */
bool extract_urls; /**< Extract and store URLs in data structure */
ConfNode *extract_urls_schemes; /**< List of schemes of which to
extract urls */
bool log_url_scheme; /**< Log the scheme of extracted URLs */
bool body_md5; /**< Compute md5 sum of body */
uint32_t header_value_depth; /**< Depth of which to store header values
(Default is 2000) */
} MimeDecConfig;
/**
* \brief This represents a header field name and associated value
*/
typedef struct MimeDecField {
uint8_t *name; /**< Name of the header field */
uint32_t name_len; /**< Length of the name */
uint32_t value_len; /**< Length of the value */
uint8_t *value; /**< Value of the header field */
struct MimeDecField *next; /**< Pointer to next field */
} MimeDecField;
/**
* \brief This represents a URL value node in a linked list
*
* Since HTML can sometimes contain a high number of URLs, this
* structure only features the URL host name/IP or those that are
* pointing to an executable file (see url_flags to determine which).
*/
typedef struct MimeDecUrl {
uint8_t *url; /**< String representation of full or partial URL (lowercase) */
uint32_t url_len; /**< Length of the URL string */
uint32_t url_flags; /**< Flags indicating type of URL */
struct MimeDecUrl *next; /**< Pointer to next URL */
} MimeDecUrl;
/**
* \brief This represents the MIME Entity (or also top level message) in a
* child-sibling tree
*/
typedef struct MimeDecEntity {
MimeDecField *field_list; /**< Pointer to list of header fields */
MimeDecUrl *url_list; /**< Pointer to list of URLs */
uint32_t header_flags; /**< Flags indicating header characteristics */
uint32_t ctnt_flags; /**< Flags indicating type of content */
uint32_t anomaly_flags; /**< Flags indicating an anomaly in the message */
uint32_t filename_len; /**< Length of file attachment name */
uint8_t *filename; /**< Name of file attachment */
uint8_t *ctnt_type; /**< Quick access pointer to short-hand content type field */
uint32_t ctnt_type_len; /**< Length of content type field value */
uint32_t msg_id_len; /**< Quick access pointer to message Id */
uint8_t *msg_id; /**< Quick access pointer to message Id */
struct MimeDecEntity *next; /**< Pointer to list of sibling entities */
struct MimeDecEntity *child; /**< Pointer to list of child entities */
struct MimeDecEntity *last_child; /**< Pointer to tail of the list of child entities */
} MimeDecEntity;
/**
* \brief Structure contains boundary and entity for the current node (entity)
* in the stack
*
*/
typedef struct MimeDecStackNode {
MimeDecEntity *data; /**< Pointer to the entity data structure */
uint8_t *bdef; /**< Copy of boundary definition for child entity */
uint16_t bdef_len; /**< Boundary length for child entity */
bool is_encap; /**< Flag indicating entity is encapsulated in message */
struct MimeDecStackNode *next; /**< Pointer to next item on the stack */
} MimeDecStackNode;
/**
* \brief Structure holds the top of the stack along with some free reusable nodes
*
*/
typedef struct MimeDecStack {
MimeDecStackNode *top; /**< Pointer to the top of the stack */
MimeDecStackNode *free_nodes; /**< Pointer to the list of free nodes */
uint32_t free_nodes_cnt; /**< Count of free nodes in the list */
} MimeDecStack;
/**
* \brief Structure contains a list of value and lengths for robust data processing
*
*/
typedef struct DataValue {
uint8_t *value; /**< Copy of data value */
uint32_t value_len; /**< Length of data value */
struct DataValue *next; /**< Pointer to next value in the list */
} DataValue;
/**
* \brief Structure contains the current state of the MIME parser
*
*/
typedef struct MimeDecParseState {
MimeDecEntity *msg; /**< Pointer to the top-level message entity */
MimeDecStack *stack; /**< Pointer to the top of the entity stack */
uint8_t *hname; /**< Copy of the last known header name */
uint32_t hlen; /**< Length of the last known header name */
uint32_t hvlen; /**< Total length of value list */
DataValue *hvalue; /**< Pointer to the incomplete header value list */
uint8_t bvremain[B64_BLOCK]; /**< Remainder from base64-decoded line */
uint8_t bvr_len; /**< Length of remainder from base64-decoded line */
uint8_t data_chunk[DATA_CHUNK_SIZE]; /**< Buffer holding data chunk */
SCMd5 *md5_ctx;
uint8_t md5[SC_MD5_LEN];
bool has_md5;
uint8_t state_flag; /**< Flag representing current state of parser */
uint32_t data_chunk_len; /**< Length of data chunk */
int found_child; /**< Flag indicating a child entity was found */
int body_begin; /**< Currently at beginning of body */
int body_end; /**< Currently at end of body */
uint8_t current_line_delimiter_len; /**< Length of line delimiter */
void *data; /**< Pointer to data specific to the caller */
int (*DataChunkProcessorFunc) (const uint8_t *chunk, uint32_t len,
struct MimeDecParseState *state); /**< Data chunk processing function callback */
} MimeDecParseState;
/* Config functions */
void MimeDecSetConfig(MimeDecConfig *config);
MimeDecConfig * MimeDecGetConfig(void);
/* Memory functions */
void MimeDecFreeEntity(MimeDecEntity *entity);
void MimeDecFreeField(MimeDecField *field);
void MimeDecFreeUrl(MimeDecUrl *url);
/* List functions */
MimeDecField * MimeDecAddField(MimeDecEntity *entity);
MimeDecField * MimeDecFindField(const MimeDecEntity *entity, const char *name);
int MimeDecFindFieldsForEach(const MimeDecEntity *entity, const char *name, int (*DataCallback)(const uint8_t *val, const size_t, void *data), void *data);
MimeDecEntity * MimeDecAddEntity(MimeDecEntity *parent);
/* Helper functions */
//MimeDecField * MimeDecFillField(MimeDecEntity *entity, const char *name,
// uint32_t nlen, const char *value, uint32_t vlen, int copy_name_value);
/* Parser functions */
MimeDecParseState * MimeDecInitParser(void *data, int (*dcpfunc)(const uint8_t *chunk,
uint32_t len, MimeDecParseState *state));
void MimeDecDeInitParser(MimeDecParseState *state);
int MimeDecParseComplete(MimeDecParseState *state);
int MimeDecParseLine(const uint8_t *line, const uint32_t len, const uint8_t delim_len, MimeDecParseState *state);
MimeDecEntity * MimeDecParseFullMsg(const uint8_t *buf, uint32_t blen, void *data,
int (*DataChunkProcessorFunc)(const uint8_t *chunk, uint32_t len, MimeDecParseState *state));
const char *MimeDecParseStateGetStatus(MimeDecParseState *state);
/* Test functions */
void MimeDecRegisterTests(void);
#endif

35
src/util-lua-smtp.c
Unescape Escape View File

@ -68,24 +68,23 @@ static int GetMimeDecField(lua_State *luastate, Flow *flow, const char *name)
if(smtp_tx == NULL) {
return LuaCallbackError(luastate, "Transaction ending or not found");
}
/* pointer to tail of msg list of MimeDecEntities in current transaction. */
MimeDecEntity *mime = smtp_tx->msg_tail;
/* pointer to tail of msg list of MimeStateSMTP in current transaction. */
MimeStateSMTP *mime = smtp_tx->mime_state;
/* check if msg_tail was hit */
if(mime == NULL){
return LuaCallbackError(luastate, "Internal error: no fields in transaction");
}
/* extract MIME field based on specific field name. */
MimeDecField *field = MimeDecFindField(mime, name);
const uint8_t *field_value;
uint32_t field_len;
/* check MIME field */
if(field == NULL) {
if (!SCMimeSmtpGetHeader(mime, name, &field_value, &field_len)) {
return LuaCallbackError(luastate, "Error: mimefield not found");
}
/* return extracted field. */
if(field->value == NULL || field->value_len == 0){
if (field_len == 0) {
return LuaCallbackError(luastate, "Error, pointer error");
}
return LuaPushStringBuffer(luastate, field->value, field->value_len);
return LuaPushStringBuffer(luastate, field_value, field_len);
}
/**
@ -139,29 +138,23 @@ static int GetMimeList(lua_State *luastate, Flow *flow)
if(smtp_tx == NULL) {
return LuaCallbackError(luastate, "Error: no SMTP transaction found");
}
/* Create a pointer to the tail of MimeDecEntity list */
MimeDecEntity *mime = smtp_tx->msg_tail;
/* Create a pointer to the tail of MimeStateSMTP list */
MimeStateSMTP *mime = smtp_tx->mime_state;
if(mime == NULL) {
return LuaCallbackError(luastate, "Error: no mime entity found");
}
MimeDecField *field = mime->field_list;
if(field == NULL) {
return LuaCallbackError(luastate, "Error: no field_list found");
}
if(field->name == NULL || field->name_len == 0) {
return LuaCallbackError(luastate, "Error: field has no name");
}
const uint8_t *field_name;
uint32_t field_len;
/* Counter of MIME fields found */
int num = 1;
/* loop trough the list of mimeFields, printing each name found */
lua_newtable(luastate);
while (field != NULL) {
if(field->name != NULL && field->name_len != 0) {
while (SCMimeSmtpGetHeaderName(mime, &field_name, &field_len, (uint32_t)num)) {
if (field_len != 0) {
lua_pushinteger(luastate,num++);
LuaPushStringBuffer(luastate, field->name, field->name_len);
LuaPushStringBuffer(luastate, field_name, field_len);
lua_settable(luastate,-3);
}
field = field->next;
}
return 1;
}

10
src/util-spm-bs.c
Unescape Escape View File

@ -130,3 +130,13 @@ uint8_t *BasicSearchNocase(const uint8_t *haystack, uint32_t haystack_len, const
return NULL;
}
uint32_t BasicSearchNocaseIndex(
const uint8_t *haystack, uint32_t haystack_len, const uint8_t *needle, uint16_t needle_len)
{
uint8_t *r = BasicSearchNocase(haystack, haystack_len, needle, needle_len);
if (r == NULL) {
return haystack_len;
}
return (uint32_t)(r - haystack);
}

1
src/util-spm-bs.h
Unescape Escape View File

@ -29,5 +29,6 @@
uint8_t *BasicSearch(const uint8_t *, uint32_t, const uint8_t *, uint16_t);
uint8_t *BasicSearchNocase(const uint8_t *, uint32_t, const uint8_t *, uint16_t);
uint32_t BasicSearchNocaseIndex(const uint8_t *, uint32_t, const uint8_t *, uint16_t);
#endif /* SURICATA_UTIL_SPM_BS */

Write Preview
Loading…
Cancel
Save