Merge 49bbfcd56e into 9a37bb659b

doc/userguide/rules/differences-from-snort.rst

@@ -100,18 +100,6 @@ Automatic Protocol Detection
    -  Use ``,norm`` for normalized buffer
    -  e.g. ``urilen:>20,norm;``
 
-``http_uri`` Buffer
--------------------
-
--  In Snort, the ``http_uri`` buffer normalizes '+' characters
-   (0x2B) to spaces (0x20).
-
-   -  Suricata can do this as well but you have to explicitly
-      set ``query-plusspace-decode: yes`` in the ``libhtp`` section of Suricata's yaml file.
-
--  `https://redmine.openinfosecfoundation.org/issues/1035 <https://redmine.openinfosecfoundation.org/issues/1035>`_
--  `https://github.com/inliniac/suricata/pull/620 <https://github.com/inliniac/suricata/pull/620>`_
-
 ``http_header`` Buffer
 ----------------------
 

rust/src/detect/transforms/urldecode.rs

@@ -33,7 +33,7 @@ unsafe extern "C" fn url_decode_setup(
     return SCDetectSignatureAddTransform(s, G_TRANSFORM_URL_DECODE_ID, ptr::null_mut());
 }
 
-fn hex_value(i: u8) -> Option<u8> {
+pub(crate) fn hex_value(i: u8) -> Option<u8> {
     match i {
         0x30..=0x39 => Some(i - 0x30),
         0x41..=0x46 => Some(i - 0x41 + 10),

rust/src/http2/detect.rs

@@ -533,6 +533,20 @@ pub unsafe extern "C" fn SCHttp2TxGetResponseLine(
 #[no_mangle]
 pub unsafe extern "C" fn SCHttp2TxGetUri(
     tx: &mut HTTP2Transaction, buffer: *mut *const u8, buffer_len: *mut u32,
+) -> u8 {
+    if !tx.normalized_uri.is_empty()
+        || http2_frames_get_header_firstvalue(tx, Direction::ToServer, ":path").is_ok()
+    {
+        *buffer = tx.normalized_uri.as_ptr(); //unsafe
+        *buffer_len = tx.normalized_uri.len() as u32;
+        return 1;
+    }
+    return 0;
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn SCHttp2TxGetRawUri(
+    tx: &mut HTTP2Transaction, buffer: *mut *const u8, buffer_len: *mut u32,
 ) -> u8 {
     if let Ok(value) = http2_frames_get_header_firstvalue(tx, Direction::ToServer, ":path") {
         *buffer = value.as_ptr(); //unsafe
@@ -1028,6 +1042,15 @@ pub unsafe extern "C" fn SCHttp2TxSetUri(
     http2_tx_set_header(state, ":path".as_bytes(), slice)
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn SCHttp2TxSetNormUri(
+    state: &mut HTTP2State, buffer: *const u8, buffer_len: u32,
+) {
+    let slice = build_slice!(buffer, buffer_len as usize);
+    let tx = state.transactions.back_mut().unwrap();
+    tx.normalized_uri = slice.to_vec();
+}
+
 fn http2_tx_set_settings(state: &mut HTTP2State, input: &[u8]) {
     match STANDARD.decode(input) {
         Ok(dec) => {

rust/src/http2/http2.rs

@@ -24,6 +24,7 @@ use super::range::{SCHTPFileCloseHandleRange, SCHttpRangeFreeBlock};
 use crate::applayer::{self, *};
 use crate::conf::{conf_get, get_memval};
 use crate::core::*;
+use crate::detect::transforms::urldecode::hex_value;
 use crate::direction::Direction;
 use crate::dns::dns::DnsVariant;
 use crate::filecontainer::FileContainerWrapper;
@@ -289,6 +290,7 @@ pub struct HTTP2Transaction {
     pub progress: HTTP2Progress,
     to_drop: bool,
     child_stream_id: u32,
+    pub normalized_uri: Vec<u8>,
 
     pub frames_tc: Vec<HTTP2Frame>,
     pub frames_ts: Vec<HTTP2Frame>,
@@ -306,6 +308,70 @@ pub struct HTTP2Transaction {
     pub doh: Option<DohHttp2Tx>,
 }
 
+enum NormalizeUriState {
+    Path,
+    Percent,
+    Percent2,
+}
+
+fn http2_normalize_uri(uri: &[u8]) -> Vec<u8> {
+    let mut state = NormalizeUriState::Path;
+    let mut query = false;
+    let mut percent = 0u8;
+    let mut prevc = 0u8;
+    let mut r = Vec::with_capacity(uri.len());
+    for c in uri {
+        match state {
+            NormalizeUriState::Path => match c {
+                b'%' => state = NormalizeUriState::Percent,
+                b'+' if query => {
+                    r.push(b' ');
+                }
+                b'?' => {
+                    query = true;
+                    r.push(*c);
+                }
+                _ => {
+                    r.push(*c);
+                }
+            },
+            NormalizeUriState::Percent => {
+                if let Some(v) = hex_value(*c) {
+                    percent = v << 4;
+                    prevc = *c;
+                    state = NormalizeUriState::Percent2;
+                } else {
+                    r.push(b'%');
+                    r.push(*c);
+                    state = NormalizeUriState::Path;
+                }
+            }
+            NormalizeUriState::Percent2 => {
+                if let Some(v) = hex_value(*c) {
+                    percent |= v;
+                    r.push(percent);
+                } else {
+                    r.push(b'%');
+                    r.push(prevc);
+                    r.push(*c);
+                }
+                state = NormalizeUriState::Path;
+            }
+        }
+    }
+    match state {
+        NormalizeUriState::Percent => {
+            r.push(b'%');
+        }
+        NormalizeUriState::Percent2 => {
+            r.push(b'%');
+            r.push(prevc);
+        }
+        _ => {}
+    }
+    return r;
+}
+
 impl Transaction for HTTP2Transaction {
     fn id(&self) -> u64 {
         self.tx_id
@@ -325,6 +391,7 @@ impl HTTP2Transaction {
             stream_id: 0,
             child_stream_id: 0,
             progress: HTTP2Progress::STREAM(HTTP2StreamProgress::init()),
+            normalized_uri: Vec::new(),
             to_drop: false,
             frames_tc: Vec::new(),
             frames_ts: Vec::new(),
@@ -389,7 +456,10 @@ impl HTTP2Transaction {
                         self.doh = Some(doh);
                     }
                 }
-            } else if block.name.as_ref() == b":path" {
+            } else if block.name.as_ref() == b":path" && dir == Direction::ToServer {
+                if self.normalized_uri.is_empty() {
+                    self.normalized_uri = http2_normalize_uri(&block.value);
+                }
                 path = Some(&block.value);
             } else if block.name.eq_ignore_ascii_case(b":authority") {
                 authority = Some(&block.value);
@@ -752,7 +822,7 @@ pub struct HTTP2State {
     response_frame_size: u32,
     dynamic_headers_ts: HTTP2DynTable,
     dynamic_headers_tc: HTTP2DynTable,
-    transactions: VecDeque<HTTP2Transaction>,
+    pub transactions: VecDeque<HTTP2Transaction>,
     progress: HTTP2ConnectionState,
 
     comp_len: u64,

src/app-layer-htp.c

@@ -1937,8 +1937,6 @@ static void HTPConfigSetDefaultsPhase1(HTPCfgRec *cfg_prec)
     htp_config_set_parse_request_cookies(cfg_prec->cfg, 0);
     htp_config_set_allow_space_uri(cfg_prec->cfg, 1);
 
-    /* don't convert + to space by default */
-    htp_config_set_plusspace_decode(cfg_prec->cfg, 0);
     // enables request decompression
     htp_config_set_request_decompression(cfg_prec->cfg, 1);
     htp_config_set_lzma_layers(cfg_prec->cfg, HTP_CONFIG_DEFAULT_LZMA_LAYERS);
@@ -4474,6 +4472,7 @@ libhtp:\n\
     personality: IDS\n\
     double-decode-path: yes\n\
     double-decode-query: yes\n\
+    query-plusspace-decode: no\n\
 ";
 
     SCConfCreateContextBackup();

src/app-layer-http2.c

@@ -88,6 +88,10 @@ void SCHTTP2MimicHttp1Request(void *alstate_orig, void *h2s)
         SCHttp2TxSetUri(h2s, bstr_ptr(htp_tx_request_uri(h1tx)),
                 (uint32_t)bstr_len(htp_tx_request_uri(h1tx)));
     }
+    if (htp_tx_normalized_uri(h1tx) != NULL) {
+        SCHttp2TxSetNormUri(h2s, bstr_ptr(htp_tx_normalized_uri(h1tx)),
+                (uint32_t)bstr_len(htp_tx_normalized_uri(h1tx)));
+    }
     size_t nbheaders = htp_tx_request_headers_size(h1tx);
     for (size_t i = 0; i < nbheaders; i++) {
         const htp_header_t *h = htp_tx_request_header_index(h1tx, i);

src/detect-http-uri.c

@@ -81,6 +81,28 @@ static int DetectHttpRawUriSetupSticky(DetectEngineCtx *de_ctx, Signature *s, co
 static int g_http_raw_uri_buffer_id = 0;
 static int g_http_uri_buffer_id = 0;
 
+static InspectionBuffer *GetRawData2(DetectEngineThreadCtx *det_ctx,
+        const DetectEngineTransforms *transforms, Flow *_f, const uint8_t _flow_flags, void *txv,
+        const int list_id)
+{
+    SCEnter();
+
+    InspectionBuffer *buffer = SCInspectionBufferGet(det_ctx, list_id);
+    if (!buffer->initialized) {
+        uint32_t b_len = 0;
+        const uint8_t *b = NULL;
+
+        if (SCHttp2TxGetRawUri(txv, &b, &b_len) != 1)
+            return NULL;
+        if (b == NULL || b_len == 0)
+            return NULL;
+
+        SCInspectionBufferSetupAndApplyTransforms(det_ctx, list_id, buffer, b, b_len, transforms);
+    }
+
+    return buffer;
+}
+
 /**
  * \brief Registration function for keywords: http_uri and http.uri
  */
@@ -149,12 +171,11 @@ void DetectHttpUriRegister (void)
     DetectAppLayerMpmRegister("http_raw_uri", SIG_FLAG_TOSERVER, 2, PrefilterGenericMpmRegister,
             GetRawData, ALPROTO_HTTP1, HTP_REQUEST_PROGRESS_LINE);
 
-    // no difference between raw and decoded uri for HTTP2
     DetectAppLayerInspectEngineRegisterSubState("http_raw_uri", ALPROTO_HTTP2, SIG_FLAG_TOSERVER,
-            HTTP2TxTypeStream, HTTP2ProgHeaders, DetectEngineInspectBufferGeneric, GetData2);
+            HTTP2TxTypeStream, HTTP2ProgHeaders, DetectEngineInspectBufferGeneric, GetRawData2);
 
     DetectAppLayerMpmRegisterSubState("http_raw_uri", SIG_FLAG_TOSERVER, 2,
-            PrefilterGenericMpmRegister, GetData2, ALPROTO_HTTP2, HTTP2TxTypeStream,
+            PrefilterGenericMpmRegister, GetRawData2, ALPROTO_HTTP2, HTTP2TxTypeStream,
             HTTP2ProgHeaders);
 
     DetectBufferTypeSetDescriptionByName("http_raw_uri",