http2: decompression for files

gzip and brotli decompression for files
pull/5780/head
Philippe Antoine 6 years ago
parent 2e46b5d100
commit d861228214

@ -14,3 +14,4 @@ alert http2 any any -> any any (msg:"SURICATA HTTP2 header frame with extra data
alert http2 any any -> any any (msg:"SURICATA HTTP2 too long frame data"; flow:established; app-layer-event:http2.long_frame_data; classtype:protocol-command-decode; sid:2290006; rev:1;)
alert http2 any any -> any any (msg:"SURICATA HTTP2 stream identifier reuse"; flow:established; app-layer-event:http2.stream_id_reuse; classtype:protocol-command-decode; sid:2290007; rev:1;)
alert http2 any any -> any any (msg:"SURICATA HTTP2 invalid HTTP1 settings during upgrade"; flow:established; app-layer-event:http2.invalid_http1_settings; classtype:protocol-command-decode; sid:2290008; rev:1;)
alert http2 any any -> any any (msg:"SURICATA HTTP2 failed decompression"; flow:established; app-layer-event:http2.failed_decompression; classtype:protocol-command-decode; sid:2290009; rev:1;)

@ -29,6 +29,8 @@ num = "0.2"
num-derive = "0.2"
num-traits = "0.2"
widestring = "0.4"
flate2 = "1.0"
brotli = "3.3.0"
der-parser = "4.0"
kerberos-parser = "0.5"

@ -0,0 +1,216 @@
/* Copyright (C) 2021 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
* Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* version 2 along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
use crate::core::STREAM_TOCLIENT;
use brotli;
use flate2::read::GzDecoder;
use std;
use std::io;
use std::io::{Cursor, Read, Write};
pub const HTTP2_DECOMPRESSION_CHUNK_SIZE: usize = 0x1000; // 4096
#[repr(u8)]
#[derive(Copy, Clone, PartialOrd, PartialEq, Debug)]
pub enum HTTP2ContentEncoding {
HTTP2ContentEncodingUnknown = 0,
HTTP2ContentEncodingGzip = 1,
HTTP2ContentEncodingBr = 2,
HTTP2ContentEncodingUnrecognized = 3,
}
//a cursor turning EOF into blocking errors
pub struct HTTP2cursor {
pub cursor: Cursor<Vec<u8>>,
}
impl HTTP2cursor {
pub fn new() -> HTTP2cursor {
HTTP2cursor {
cursor: Cursor::new(Vec::new()),
}
}
#[cfg(feature = "debug-validate")]
pub fn position(&self) -> u64 {
return self.cursor.position();
}
pub fn set_position(&mut self, pos: u64) {
return self.cursor.set_position(pos);
}
}
// we need to implement this as flate2 and brotli crates
// will read from this object
impl Read for HTTP2cursor {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
//use the cursor, except it turns eof into blocking error
let r = self.cursor.read(buf);
match r {
Err(ref err) => {
if err.kind() == io::ErrorKind::UnexpectedEof {
return Err(io::ErrorKind::WouldBlock.into());
}
}
Ok(0) => {
//regular EOF turned into blocking error
return Err(io::ErrorKind::WouldBlock.into());
}
Ok(_n) => {}
}
return r;
}
}
pub enum HTTP2Decompresser {
UNASSIGNED,
GZIP(GzDecoder<HTTP2cursor>),
BROTLI(brotli::Decompressor<HTTP2cursor>),
}
struct HTTP2DecoderHalf {
encoding: HTTP2ContentEncoding,
decoder: HTTP2Decompresser,
}
pub trait GetMutCursor {
fn get_mut(&mut self) -> &mut HTTP2cursor;
}
impl GetMutCursor for GzDecoder<HTTP2cursor> {
fn get_mut(&mut self) -> &mut HTTP2cursor {
return self.get_mut();
}
}
impl GetMutCursor for brotli::Decompressor<HTTP2cursor> {
fn get_mut(&mut self) -> &mut HTTP2cursor {
return self.get_mut();
}
}
fn http2_decompress<'a>(
decoder: &mut (impl Read + GetMutCursor), input: &'a [u8], output: &'a mut Vec<u8>,
) -> io::Result<&'a [u8]> {
match decoder.get_mut().cursor.write_all(input) {
Ok(()) => {}
Err(e) => {
return Err(e);
}
}
let mut offset = 0;
decoder.get_mut().set_position(0);
output.resize(HTTP2_DECOMPRESSION_CHUNK_SIZE, 0);
loop {
match decoder.read(&mut output[offset..]) {
Ok(0) => {
break;
}
Ok(n) => {
offset += n;
if offset == output.len() {
output.resize(output.len() + HTTP2_DECOMPRESSION_CHUNK_SIZE, 0);
}
}
Err(e) => {
if e.kind() == io::ErrorKind::WouldBlock {
break;
}
return Err(e);
}
}
}
//checks all input was consumed
debug_validate_bug_on!(decoder.get_mut().position() < (input.len() as u64));
decoder.get_mut().set_position(0);
return Ok(&output[..offset]);
}
impl HTTP2DecoderHalf {
pub fn new() -> HTTP2DecoderHalf {
HTTP2DecoderHalf {
encoding: HTTP2ContentEncoding::HTTP2ContentEncodingUnknown,
decoder: HTTP2Decompresser::UNASSIGNED,
}
}
pub fn http2_encoding_fromvec(&mut self, input: &Vec<u8>) {
//use first encoding...
if self.encoding == HTTP2ContentEncoding::HTTP2ContentEncodingUnknown {
if *input == "gzip".as_bytes().to_vec() {
self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingGzip;
self.decoder = HTTP2Decompresser::GZIP(GzDecoder::new(HTTP2cursor::new()));
} else if *input == "br".as_bytes().to_vec() {
self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingBr;
self.decoder = HTTP2Decompresser::BROTLI(brotli::Decompressor::new(
HTTP2cursor::new(),
HTTP2_DECOMPRESSION_CHUNK_SIZE,
));
} else {
self.encoding = HTTP2ContentEncoding::HTTP2ContentEncodingUnrecognized;
}
}
}
pub fn decompress<'a>(
&'a mut self, input: &'a [u8], output: &'a mut Vec<u8>,
) -> io::Result<&'a [u8]> {
match self.decoder {
HTTP2Decompresser::GZIP(ref mut gzip_decoder) => {
return http2_decompress(gzip_decoder, input, output);
}
HTTP2Decompresser::BROTLI(ref mut br_decoder) => {
return http2_decompress(br_decoder, input, output);
}
_ => {}
}
return Ok(input);
}
}
pub struct HTTP2Decoder {
decoder_tc: HTTP2DecoderHalf,
decoder_ts: HTTP2DecoderHalf,
}
impl HTTP2Decoder {
pub fn new() -> HTTP2Decoder {
HTTP2Decoder {
decoder_tc: HTTP2DecoderHalf::new(),
decoder_ts: HTTP2DecoderHalf::new(),
}
}
pub fn http2_encoding_fromvec(&mut self, input: &Vec<u8>, dir: u8) {
if dir == STREAM_TOCLIENT {
self.decoder_tc.http2_encoding_fromvec(input);
} else {
self.decoder_ts.http2_encoding_fromvec(input);
}
}
pub fn decompress<'a>(
&'a mut self, input: &'a [u8], output: &'a mut Vec<u8>, dir: u8,
) -> io::Result<&'a [u8]> {
if dir == STREAM_TOCLIENT {
return self.decoder_tc.decompress(input, output);
} else {
return self.decoder_ts.decompress(input, output);
}
}
}

@ -15,6 +15,7 @@
* 02110-1301, USA.
*/
use super::decompression;
use super::parser;
use crate::applayer::{self, *};
use crate::core::{
@ -27,6 +28,7 @@ use nom;
use std;
use std::ffi::{CStr, CString};
use std::fmt;
use std::io;
use std::mem::transmute;
static mut ALPROTO_HTTP2: AppProto = ALPROTO_UNKNOWN;
@ -124,6 +126,8 @@ pub struct HTTP2Transaction {
pub frames_tc: Vec<HTTP2Frame>,
pub frames_ts: Vec<HTTP2Frame>,
decoder: decompression::HTTP2Decoder,
de_state: Option<*mut core::DetectEngineState>,
events: *mut core::AppLayerDecoderEvents,
tx_data: AppLayerTxData,
@ -143,6 +147,7 @@ impl HTTP2Transaction {
state: HTTP2TransactionState::HTTP2StateIdle,
frames_tc: Vec::new(),
frames_ts: Vec::new(),
decoder: decompression::HTTP2Decoder::new(),
de_state: None,
events: std::ptr::null_mut(),
tx_data: AppLayerTxData::new(),
@ -160,6 +165,36 @@ impl HTTP2Transaction {
}
}
fn handle_headers(&mut self, blocks: &Vec<parser::HTTP2FrameHeaderBlock>, dir: u8) {
for i in 0..blocks.len() {
if blocks[i].name == "content-encoding".as_bytes().to_vec() {
self.decoder.http2_encoding_fromvec(&blocks[i].value, dir);
}
}
}
fn decompress<'a>(
&'a mut self, input: &'a [u8], dir: u8, sfcm: &'static SuricataFileContext, over: bool,
files: &mut FileContainer, flags: u16,
) -> io::Result<()> {
let mut output = Vec::with_capacity(decompression::HTTP2_DECOMPRESSION_CHUNK_SIZE);
let decompressed = self.decoder.decompress(input, &mut output, dir)?;
let xid: u32 = self.tx_id as u32;
self.ft.new_chunk(
sfcm,
files,
flags,
b"",
decompressed,
self.ft.tracked, //offset = append
decompressed.len() as u32,
0,
over,
&xid,
);
return Ok(());
}
fn handle_frame(
&mut self, header: &parser::HTTP2FrameHeader, data: &HTTP2FrameTypeData, dir: u8,
) {
@ -173,18 +208,21 @@ impl HTTP2Transaction {
}
self.state = HTTP2TransactionState::HTTP2StateReserved;
}
self.handle_headers(&hs.blocks, dir);
}
HTTP2FrameTypeData::CONTINUATION(_) => {
HTTP2FrameTypeData::CONTINUATION(hs) => {
if dir == STREAM_TOCLIENT
&& header.flags & parser::HTTP2_FLAG_HEADER_END_HEADERS != 0
{
self.child_stream_id = 0;
}
self.handle_headers(&hs.blocks, dir);
}
HTTP2FrameTypeData::HEADERS(_) => {
HTTP2FrameTypeData::HEADERS(hs) => {
if dir == STREAM_TOCLIENT {
self.child_stream_id = 0;
}
self.handle_headers(&hs.blocks, dir);
}
HTTP2FrameTypeData::RSTSTREAM(_) => {
self.child_stream_id = 0;
@ -253,6 +291,7 @@ pub enum HTTP2Event {
LongFrameData,
StreamIdReuse,
InvalidHTTP1Settings,
FailedDecompression,
}
impl HTTP2Event {
@ -267,6 +306,7 @@ impl HTTP2Event {
6 => Some(HTTP2Event::LongFrameData),
7 => Some(HTTP2Event::StreamIdReuse),
8 => Some(HTTP2Event::InvalidHTTP1Settings),
9 => Some(HTTP2Event::FailedDecompression),
_ => None,
}
}
@ -767,21 +807,21 @@ impl HTTP2State {
let index = self.find_tx_index(sid);
if index > 0 {
let mut tx_same = &mut self.transactions[index - 1];
let xid: u32 = tx_same.tx_id as u32;
tx_same.ft.tx_id = tx_same.tx_id - 1;
let (files, flags) = self.files.get(dir);
tx_same.ft.new_chunk(
match tx_same.decompress(
&rem[..hlsafe],
dir,
sfcm,
over,
files,
flags,
b"",
&rem[..hlsafe],
tx_same.ft.tracked, //offset = append
hlsafe as u32,
0,
over,
&xid,
);
) {
Err(_e) => {
self.set_event(HTTP2Event::FailedDecompression);
}
_ => {}
}
}
}
None => panic!("no SURICATA_HTTP2_FILE_CONFIG"),
@ -1052,6 +1092,7 @@ pub extern "C" fn rs_http2_state_get_event_info(
"long_frame_data" => HTTP2Event::LongFrameData as i32,
"stream_id_reuse" => HTTP2Event::StreamIdReuse as i32,
"invalid_http1_settings" => HTTP2Event::InvalidHTTP1Settings as i32,
"failed_decompression" => HTTP2Event::FailedDecompression as i32,
_ => -1, // unknown event
}
}
@ -1080,6 +1121,7 @@ pub extern "C" fn rs_http2_state_get_event_info_by_id(
HTTP2Event::LongFrameData => "long_frame_data\0",
HTTP2Event::StreamIdReuse => "stream_id_reuse\0",
HTTP2Event::InvalidHTTP1Settings => "invalid_http1_settings\0",
HTTP2Event::FailedDecompression => "failed_decompression\0",
};
unsafe {
*event_name = estr.as_ptr() as *const std::os::raw::c_char;

@ -15,6 +15,7 @@
* 02110-1301, USA.
*/
mod decompression;
pub mod detect;
pub mod http2;
mod huffman;

@ -694,6 +694,8 @@ static void PrintBuildInfo(void)
#endif
/* For compatibility, just say we have HAVE_NSS. */
strlcat(features, "HAVE_NSS ", sizeof(features));
/* HTTP2_DECOMPRESSION is not an optional feature in this major version */
strlcat(features, "HTTP2_DECOMPRESSION ", sizeof(features));
#ifdef HAVE_LUA
strlcat(features, "HAVE_LUA ", sizeof(features));
#endif

Loading…
Cancel
Save