lzma: replaces liblzma with own sdk for swf decompression

so as to avoid memory exhaustion

configure.ac

@@ -580,46 +580,6 @@
         LIBS="${TMPLIBS} -lz"
     fi
 
-  # liblzma
-    enable_liblzma=no
-
-    AC_ARG_WITH(liblzma_includes,
-            [  --with-liblzma-includes=DIR  liblzma include directory],
-            [with_liblzma_includes="$withval"],[with_liblzma_includes="no"])
-    AC_ARG_WITH(liblzma_libraries,
-            [  --with-liblzma-libraries=DIR    liblzma library directory],
-            [with_liblzma_libraries="$withval"],[with_liblzma_libraries="no"])
-
-    if test "$with_liblzma_includes" != "no"; then
-        CPPFLAGS="${CPPFLAGS} -I${with_liblzma_includes}"
-    fi
-    TMPLIBS="${LIBS}"
-
-    AC_CHECK_HEADER(lzma.h,
-        AC_CHECK_LIB(lzma,lzma_code,[
-	    AC_DEFINE([HAVE_LIBLZMA],[1],[liblzma available])
-	    LIBLZMA="yes"
-            if test "$LIBLZMA" = "yes"; then
-                if test "$with_liblzma_libraries" != "no"; then
-                    LDFLAGS="${LDFLAGS}  -L${with_liblzma_libraries}"
-                    LIBS="${TMPLIBS} -llzma"
-                else
-                    LIBS="${TMPLIBS} -llzma"
-                fi
-            fi]),LIBLZMA="no")
-
-    if test "$LIBLZMA" != "yes"; then
-        echo
-        echo "   Error! liblzma library not found."
-        echo "   Debian/Ubuntu: apt install liblzma-dev"
-        echo "   Fedora: dnf install xz-devel"
-        echo "   CentOS/RHEL: yum install xz-devel"
-        echo
-        exit 1
-    fi
-    enable_liblzma=yes
-    LIBS="${TMPLIBS} -llzma"
-
   #libpcre
     AC_ARG_WITH(libpcre_includes,
             [  --with-libpcre-includes=DIR  libpcre include directory],
@@ -2587,7 +2547,6 @@ SURICATA_BUILD_CONF="Suricata Configuration:
   libnss support:                          ${enable_nss}
   libnspr support:                         ${enable_nspr}
   libjansson support:                      ${enable_jansson}
-  liblzma support:                         ${enable_liblzma}
   hiredis support:                         ${enable_hiredis}
   hiredis async with libevent:             ${enable_hiredis_async}
   Prelude support:                         ${enable_prelude}

src/util-file-swf-decompression.c

@@ -34,7 +34,7 @@
 
 #include <zlib.h>
 
-#include <lzma.h>
+#include <htp/lzma/LzmaDec.h>
 
 #define MAX_SWF_DECOMPRESSED_LEN 50000000
 /*
@@ -123,6 +123,10 @@ int FileSwfZlibDecompression(DetectEngineThreadCtx *det_ctx,
     return ret;
 }
 
+static void *SzAlloc(ISzAllocPtr p, size_t size) { return malloc(size); }
+static void SzFree(ISzAllocPtr p, void *address) { free(address); }
+static const ISzAlloc suri_lzma_Alloc = { SzAlloc, SzFree };
+
 /* ZWS format */
 /*
  * | 4 bytes         | 4 bytes    | 4 bytes        | 5 bytes    | n bytes   | 6 bytes         |
@@ -132,42 +136,47 @@ int FileSwfLzmaDecompression(DetectEngineThreadCtx *det_ctx,
                              uint8_t *compressed_data, uint32_t compressed_data_len,
                              uint8_t *decompressed_data, uint32_t decompressed_data_len)
 {
-    int ret = 1;
-    lzma_stream strm = LZMA_STREAM_INIT;
-    lzma_ret result = lzma_alone_decoder(&strm, UINT64_MAX /* memlimit */);
-    if (result != LZMA_OK) {
+    int ret = 0;
+
+    CLzmaDec strm;
+    LzmaDec_Construct(&strm);
+    ELzmaStatus status;
+
+    if (compressed_data_len < LZMA_PROPS_SIZE + 8) {
+        DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_FORMAT_ERROR);
+        return 0;
+    }
+    ret = LzmaDec_Allocate(&strm, compressed_data, LZMA_PROPS_SIZE, &suri_lzma_Alloc);
+    if (ret != SZ_OK) {
         DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_DECODER_ERROR);
         return 0;
     }
-
-    strm.avail_in = compressed_data_len;
-    strm.next_in = compressed_data;
-    strm.avail_out = decompressed_data_len;
-    strm.next_out = decompressed_data;
-
-    result = lzma_code(&strm, LZMA_RUN);
-    switch(result) {
-        case LZMA_STREAM_END:
+    LzmaDec_Init(&strm);
+    compressed_data += LZMA_PROPS_SIZE + 8;
+    compressed_data_len -= LZMA_PROPS_SIZE + 8;
+    size_t inprocessed = compressed_data_len;
+    size_t outprocessed = decompressed_data_len;
+
+    ret = LzmaDec_DecodeToBuf(&strm, decompressed_data, &outprocessed,
+                             compressed_data, &inprocessed, LZMA_FINISH_ANY, &status, MAX_SWF_DECOMPRESSED_LEN);
+
+    switch(ret) {
+        case SZ_OK:
+            ret = 1;
             break;
-        case LZMA_OK:
-            break;
-        case LZMA_MEMLIMIT_ERROR:
+        case SZ_ERROR_MEM:
             DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_MEMLIMIT_ERROR);
             ret = 0;
             break;
-        case LZMA_OPTIONS_ERROR:
+        case SZ_ERROR_UNSUPPORTED:
             DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_OPTIONS_ERROR);
             ret = 0;
             break;
-        case LZMA_FORMAT_ERROR:
-            DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_FORMAT_ERROR);
-            ret = 0;
-            break;
-        case LZMA_DATA_ERROR:
+        case SZ_ERROR_DATA:
             DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_DATA_ERROR);
             ret = 0;
             break;
-        case LZMA_BUF_ERROR:
+        case SZ_ERROR_INPUT_EOF:
             DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_BUF_ERROR);
             ret = 0;
             break;
@@ -177,6 +186,6 @@ int FileSwfLzmaDecompression(DetectEngineThreadCtx *det_ctx,
             break;
     }
 
-    lzma_end(&strm);
+    LzmaDec_Free(&strm, &suri_lzma_Alloc);
     return ret;
 }