http1: decompress zstd encoding

Ticket: 7904

As for brotli, do not allow zstd as a stacked encoding

rust/htp/Cargo.toml.in

@@ -28,6 +28,7 @@ nom = "8.0.0"
 lzma-rs = { version = "0.2.0", features = ["stream"] }
 flate2 = { version = "~1.0.35", features = ["zlib-default"], default-features = false }
 brotli = "~8.0.1"
+zstd = { version = "~0.13.3",  default-features = false, features = ["arrays", "zdict_builder"] }
 lazy_static = "1.5.0"
 time = "~0.3.41"
 

rust/htp/src/decompressors.rs

@@ -4,6 +4,7 @@ use std::{
     io::{Cursor, Write},
     time::Instant,
 };
+use zstd;
 
 /// Buffer compression output to this chunk size.
 const ENCODING_CHUNK_SIZE: usize = 8192;
@@ -213,6 +214,8 @@ pub(crate) enum HtpContentEncoding {
     Lzma,
     /// Brotli compression.
     Brotli,
+    /// Zstd compression.
+    Zstd,
 }
 
 //a cursor turning EOF into blocking errors
@@ -309,6 +312,7 @@ impl Decompressor {
             | HtpContentEncoding::Deflate
             | HtpContentEncoding::Zlib
             | HtpContentEncoding::Brotli
+            | HtpContentEncoding::Zstd
             | HtpContentEncoding::Lzma => Ok(Decompressor::new(Box::new(InnerDecompressor::new(
                 encoding, self.inner, options,
             )?))),
@@ -813,6 +817,33 @@ impl BufWriter for BrotliBufWriter {
     }
 }
 
+/// Simple wrapper around an lzma implementation
+struct ZstdBufWriter<'a>(zstd::stream::write::Decoder<'a, BlockingCursor>);
+
+impl Write for ZstdBufWriter<'_> {
+    fn write(&mut self, data: &[u8]) -> std::io::Result<usize> {
+        self.0.write(data)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.0.flush()
+    }
+}
+
+impl BufWriter for ZstdBufWriter<'_> {
+    fn get_mut(&mut self) -> Option<&mut BlockingCursor> {
+        Some(self.0.get_mut())
+    }
+
+    fn finish(self: Box<Self>) -> std::io::Result<BlockingCursor> {
+        Ok(self.0.into_inner())
+    }
+
+    fn try_finish(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
 /// Structure that represents each decompressor in the chain.
 struct InnerDecompressor {
     /// Decoder implementation that will write to a temporary buffer.
@@ -847,6 +878,20 @@ impl InnerDecompressor {
                 false,
             )),
             HtpContentEncoding::Brotli => Ok((Box::new(BrotliBufWriter::new(buf)), false)),
+            HtpContentEncoding::Zstd => {
+                if let Ok(mut z) = zstd::stream::write::Decoder::new(buf) {
+                    // 8 Mbytes max window like chromium to avoid overallocating
+                    if z.window_log_max(23).is_ok() {
+                        Ok((Box::new(ZstdBufWriter(z)), false))
+                    } else {
+                        Err(std::io::Error::other(
+                            "failed to set max window for zstd decoder",
+                        ))
+                    }
+                } else {
+                    Err(std::io::Error::other("failed to create zstd decoder"))
+                }
+            }
             HtpContentEncoding::Lzma => {
                 if let Some(options) = options.lzma {
                     Ok((
@@ -1047,6 +1092,7 @@ impl Decompress for InnerDecompressor {
                 // For other encodings, we retry with deflate, zlib and gzip
                 HtpContentEncoding::Lzma => HtpContentEncoding::Deflate,
                 HtpContentEncoding::Brotli => HtpContentEncoding::Deflate,
+                HtpContentEncoding::Zstd => HtpContentEncoding::Deflate,
                 HtpContentEncoding::None => {
                     return Err(std::io::Error::other("expected a valid encoding"))
                 }

rust/htp/src/request.rs

@@ -1048,6 +1048,7 @@ impl ConnectionParser {
             | HtpContentEncoding::Deflate
             | HtpContentEncoding::Zlib
             | HtpContentEncoding::Brotli
+            | HtpContentEncoding::Zstd
             | HtpContentEncoding::Lzma => {
                 // Send data buffer to the decompressor if it exists
                 if req.request_decompressor.is_none() && data.is_none() {
@@ -1134,6 +1135,8 @@ impl ConnectionParser {
                 HtpContentEncoding::Brotli
             } else if ce.cmp_nocase_nozero(b"lzma") {
                 HtpContentEncoding::Lzma
+            } else if ce.cmp_nocase_nozero(b"zstd") {
+                HtpContentEncoding::Zstd
             } else if ce.cmp_nocase_nozero(b"inflate") || ce.cmp_nocase_nozero(b"none") {
                 HtpContentEncoding::None
             } else {
@@ -1162,6 +1165,7 @@ impl ConnectionParser {
             | HtpContentEncoding::Deflate
             | HtpContentEncoding::Zlib
             | HtpContentEncoding::Brotli
+            | HtpContentEncoding::Zstd
             | HtpContentEncoding::Lzma => {
                 self.request_prepend_decompressor(request_content_encoding_processing)?;
             }

rust/htp/src/response.rs

@@ -1071,6 +1071,7 @@ impl ConnectionParser {
             | HtpContentEncoding::Deflate
             | HtpContentEncoding::Zlib
             | HtpContentEncoding::Brotli
+            | HtpContentEncoding::Zstd
             | HtpContentEncoding::Lzma => {
                 // Send data buffer to the decompressor if it exists
                 if resp.response_decompressor.is_none() && data.is_none() {
@@ -1152,6 +1153,8 @@ impl ConnectionParser {
                 HtpContentEncoding::Lzma
             } else if ce.cmp_nocase_nozero(b"br") {
                 HtpContentEncoding::Brotli
+            } else if ce.cmp_nocase_nozero(b"zstd") {
+                HtpContentEncoding::Zstd
             } else if ce.cmp_nocase_nozero(b"inflate") || ce.cmp_nocase_nozero(b"none") {
                 HtpContentEncoding::None
             } else {
@@ -1172,6 +1175,7 @@ impl ConnectionParser {
             | HtpContentEncoding::Deflate
             | HtpContentEncoding::Zlib
             | HtpContentEncoding::Brotli
+            | HtpContentEncoding::Zstd
             | HtpContentEncoding::Lzma => {
                 self.response_prepend_decompressor(response_content_encoding_processing)?;
             }