Util: Add CompressHelpers

src/common/heap_array.h

@@ -195,6 +195,19 @@ public:
       m_size = 0;
     }
   }
+  DynamicHeapArray(const std::span<const T> data)
+  {
+    if (!data.empty())
+    {
+      internal_resize(data.size(), nullptr, 0);
+      std::memcpy(m_data, data.data(), sizeof(T) * data.size());
+    }
+    else
+    {
+      m_data = nullptr;
+      m_size = 0;
+    }
+  }
 
   DynamicHeapArray(const this_type& copy)
   {

src/util/CMakeLists.txt

@@ -17,6 +17,8 @@ add_library(util
   cd_image_ppf.cpp
   cd_subchannel_replacement.cpp
   cd_subchannel_replacement.h
+  compress_helpers.cpp
+  compress_helpers.h
   cue_parser.cpp
   cue_parser.h
   gpu_device.cpp

src/util/compress_helpers.cpp

@@ -0,0 +1,259 @@
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
+
+#include "compress_helpers.h"
+
+#include "common/assert.h"
+#include "common/error.h"
+#include "common/file_system.h"
+#include "common/path.h"
+#include "common/string_util.h"
+
+#include <zstd.h>
+#include <zstd_errors.h>
+
+// TODO: Use streaming API to avoid mallocing the whole input buffer. But one read() call is probably still faster..
+
+namespace CompressHelpers {
+static std::optional<CompressType> GetCompressType(const std::string_view path, Error* error);
+
+template<typename T>
+static bool DecompressHelper(OptionalByteBuffer& ret, CompressType type, T data,
+                             std::optional<size_t> decompressed_size, Error* error);
+
+template<typename T>
+static bool CompressHelper(OptionalByteBuffer& ret, CompressType type, T data, int clevel, Error* error);
+} // namespace CompressHelpers
+
+std::optional<CompressHelpers::CompressType> CompressHelpers::GetCompressType(const std::string_view path, Error* error)
+{
+  const std::string_view extension = Path::GetExtension(path);
+  if (StringUtil::EqualNoCase(extension, "zst"))
+    return CompressType::Zstandard;
+
+  return CompressType::Uncompressed;
+}
+
+template<typename T>
+bool CompressHelpers::DecompressHelper(CompressHelpers::OptionalByteBuffer& ret, CompressType type, T data,
+                                       std::optional<size_t> decompressed_size, Error* error)
+{
+  if (data.size() == 0) [[unlikely]]
+  {
+    Error::SetStringView(error, "Buffer is empty.");
+    return false;
+  }
+
+  switch (type)
+  {
+    case CompressType::Uncompressed:
+    {
+      ret = ByteBuffer(std::move(data));
+      return true;
+    }
+
+    case CompressType::Zstandard:
+    {
+      size_t real_decompressed_size;
+      if (!decompressed_size.has_value())
+      {
+        const unsigned long long runtime_decompressed_size = ZSTD_getFrameContentSize(data.data(), data.size());
+        if (runtime_decompressed_size == ZSTD_CONTENTSIZE_UNKNOWN ||
+            runtime_decompressed_size == ZSTD_CONTENTSIZE_ERROR ||
+            runtime_decompressed_size >= std::numeric_limits<size_t>::max()) [[unlikely]]
+        {
+          Error::SetStringView(error, "Failed to get uncompressed size.");
+          return false;
+        }
+
+        real_decompressed_size = static_cast<size_t>(runtime_decompressed_size);
+      }
+      else
+      {
+        real_decompressed_size = decompressed_size.value();
+      }
+
+      ret = DynamicHeapArray<u8>(real_decompressed_size);
+
+      const size_t result = ZSTD_decompress(ret->data(), ret->size(), data.data(), data.size());
+      if (ZSTD_isError(result)) [[unlikely]]
+      {
+        const char* errstr = ZSTD_getErrorString(ZSTD_getErrorCode(result));
+        Error::SetStringFmt(error, "ZSTD_decompress() failed: {}", errstr ? errstr : "<unknown>");
+        ret.reset();
+        return false;
+      }
+      else if (result != real_decompressed_size) [[unlikely]]
+      {
+        Error::SetStringFmt(error, "ZSTD_decompress() only returned {} of {} bytes.", result, real_decompressed_size);
+        ret.reset();
+        return false;
+      }
+
+      return true;
+    }
+    break;
+
+      DefaultCaseIsUnreachable()
+  }
+}
+
+template<typename T>
+bool CompressHelpers::CompressHelper(OptionalByteBuffer& ret, CompressType type, T data, int clevel, Error* error)
+{
+  if (data.size() == 0) [[unlikely]]
+  {
+    Error::SetStringView(error, "Buffer is empty.");
+    return false;
+  }
+
+  switch (type)
+  {
+    case CompressType::Uncompressed:
+    {
+      ret = ByteBuffer(std::move(data));
+      return true;
+    }
+
+    case CompressType::Zstandard:
+    {
+      const size_t compressed_size = ZSTD_compressBound(data.size());
+      if (compressed_size == 0) [[unlikely]]
+      {
+        Error::SetStringView(error, "ZSTD_compressBound() failed.");
+        return false;
+      }
+
+      ret = ByteBuffer(compressed_size);
+
+      const size_t result = ZSTD_compress(ret->data(), compressed_size, data.data(), data.size(),
+                                          (clevel < 0) ? 0 : std::clamp(clevel, 1, 22));
+      if (ZSTD_isError(result)) [[unlikely]]
+      {
+        const char* errstr = ZSTD_getErrorString(ZSTD_getErrorCode(result));
+        Error::SetStringFmt(error, "ZSTD_compress() failed: {}", errstr ? errstr : "<unknown>");
+        return false;
+      }
+
+      ret->resize(result);
+      return true;
+    }
+
+      DefaultCaseIsUnreachable()
+  }
+}
+
+CompressHelpers::OptionalByteBuffer CompressHelpers::DecompressBuffer(CompressType type, std::span<const u8> data,
+                                                                      std::optional<size_t> decompressed_size,
+                                                                      Error* error)
+{
+  CompressHelpers::OptionalByteBuffer ret;
+  DecompressHelper(ret, type, data, decompressed_size, error);
+  return ret;
+}
+
+CompressHelpers::OptionalByteBuffer CompressHelpers::DecompressBuffer(CompressType type, OptionalByteBuffer data,
+                                                                      std::optional<size_t> decompressed_size,
+                                                                      Error* error)
+{
+  OptionalByteBuffer ret;
+  if (data.has_value())
+  {
+    DecompressHelper(ret, type, std::move(data.value()), decompressed_size, error);
+  }
+  else
+  {
+    if (error && !error->IsValid())
+      error->SetStringView("Data buffer is empty.");
+  }
+
+  return ret;
+}
+
+CompressHelpers::OptionalByteBuffer CompressHelpers::DecompressFile(std::string_view path, std::span<const u8> data,
+                                                                    std::optional<size_t> decompressed_size,
+                                                                    Error* error)
+{
+  OptionalByteBuffer ret;
+  const std::optional<CompressType> type = GetCompressType(path, error);
+  if (type.has_value())
+    ret = DecompressBuffer(type.value(), data, decompressed_size, error);
+  return ret;
+}
+
+CompressHelpers::OptionalByteBuffer CompressHelpers::DecompressFile(std::string_view path, OptionalByteBuffer data,
+                                                                    std::optional<size_t> decompressed_size,
+                                                                    Error* error)
+{
+  OptionalByteBuffer ret;
+  const std::optional<CompressType> type = GetCompressType(path, error);
+  if (type.has_value())
+    ret = DecompressBuffer(type.value(), std::move(data), decompressed_size, error);
+  return ret;
+}
+
+CompressHelpers::OptionalByteBuffer
+CompressHelpers::DecompressFile(const char* path, std::optional<size_t> decompressed_size, Error* error)
+{
+  OptionalByteBuffer ret;
+  const std::optional<CompressType> type = GetCompressType(path, error);
+  if (type.has_value())
+    ret = DecompressFile(type.value(), path, decompressed_size, error);
+  return ret;
+}
+
+CompressHelpers::OptionalByteBuffer CompressHelpers::DecompressFile(CompressType type, const char* path,
+                                                                    std::optional<size_t> decompressed_size,
+                                                                    Error* error)
+{
+  OptionalByteBuffer ret;
+  OptionalByteBuffer data = FileSystem::ReadBinaryFile(path, error);
+  if (data.has_value())
+    ret = DecompressBuffer(type, std::move(data), decompressed_size, error);
+  return ret;
+}
+
+CompressHelpers::OptionalByteBuffer CompressHelpers::CompressToBuffer(CompressType type, std::span<const u8> data,
+                                                                      int clevel, Error* error)
+{
+  OptionalByteBuffer ret;
+  CompressHelper(ret, type, data, clevel, error);
+  return ret;
+}
+
+CompressHelpers::OptionalByteBuffer CompressHelpers::CompressToBuffer(CompressType type, const void* data,
+                                                                      size_t data_size, int clevel, Error* error)
+{
+  OptionalByteBuffer ret;
+  CompressHelper(ret, type, std::span<const u8>(static_cast<const u8*>(data), data_size), clevel, error);
+  return ret;
+}
+
+CompressHelpers::OptionalByteBuffer CompressHelpers::CompressToBuffer(CompressType type, OptionalByteBuffer data,
+                                                                      int clevel, Error* error)
+{
+  OptionalByteBuffer ret;
+  CompressHelper(ret, type, std::move(data.value()), clevel, error);
+  return ret;
+}
+
+bool CompressHelpers::CompressToFile(const char* path, std::span<const u8> data, int clevel, bool atomic_write,
+                                     Error* error)
+{
+  const std::optional<CompressType> type = GetCompressType(path, error);
+  if (!type.has_value())
+    return false;
+
+  return CompressToFile(type.value(), path, data, clevel, atomic_write, error);
+}
+
+bool CompressHelpers::CompressToFile(CompressType type, const char* path, std::span<const u8> data, int clevel,
+                                     bool atomic_write, Error* error)
+{
+  const OptionalByteBuffer cdata = CompressToBuffer(type, data, clevel, error);
+  if (!cdata.has_value())
+    return false;
+
+  return atomic_write ? FileSystem::WriteAtomicRenamedFile(path, cdata->data(), cdata->size(), error) :
+                        FileSystem::WriteBinaryFile(path, cdata->data(), cdata->size(), error);
+}

src/util/compress_helpers.h

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
+
+#pragma once
+
+#include "common/heap_array.h"
+
+#include <optional>
+#include <span>
+
+class Error;
+
+namespace CompressHelpers {
+enum class CompressType
+{
+  Uncompressed,
+  Zstandard,
+  Count
+};
+
+using ByteBuffer = DynamicHeapArray<u8>;
+using OptionalByteBuffer = std::optional<ByteBuffer>;
+
+OptionalByteBuffer DecompressBuffer(CompressType type, std::span<const u8> data,
+                                    std::optional<size_t> decompressed_size = std::nullopt, Error* error = nullptr);
+OptionalByteBuffer DecompressBuffer(CompressType type, OptionalByteBuffer data,
+                                    std::optional<size_t> decompressed_size = std::nullopt, Error* error = nullptr);
+OptionalByteBuffer DecompressFile(std::string_view path, std::span<const u8> data,
+                                  std::optional<size_t> decompressed_size = std::nullopt, Error* error = nullptr);
+OptionalByteBuffer DecompressFile(std::string_view path, OptionalByteBuffer data,
+                                  std::optional<size_t> decompressed_size = std::nullopt, Error* error = nullptr);
+OptionalByteBuffer DecompressFile(const char* path, std::optional<size_t> decompressed_size = std::nullopt,
+                                  Error* error = nullptr);
+OptionalByteBuffer DecompressFile(CompressType type, const char* path,
+                                  std::optional<size_t> decompressed_size = std::nullopt, Error* error = nullptr);
+
+OptionalByteBuffer CompressToBuffer(CompressType type, const void* data, size_t data_size, int clevel = -1,
+                                    Error* error = nullptr);
+OptionalByteBuffer CompressToBuffer(CompressType type, std::span<const u8> data, int clevel = -1,
+                                    Error* error = nullptr);
+OptionalByteBuffer CompressToBuffer(CompressType type, OptionalByteBuffer data, int clevel = -1,
+                                    Error* error = nullptr);
+bool CompressToFile(const char* path, std::span<const u8> data, int clevel = -1, bool atomic_write = true,
+                    Error* error = nullptr);
+bool CompressToFile(CompressType type, const char* path, std::span<const u8> data, int clevel = -1,
+                    bool atomic_write = true, Error* error = nullptr);
+} // namespace CompressHelpers

src/util/util.vcxproj

@@ -2,6 +2,7 @@
 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="..\..\dep\msvc\vsprops\Configurations.props" />
   <ItemGroup>
+    <ClInclude Include="compress_helpers.h" />
     <ClInclude Include="image.h" />
     <ClInclude Include="imgui_animated.h" />
     <ClInclude Include="audio_stream.h" />
@@ -115,6 +116,7 @@
     <ClCompile Include="cd_image_mds.cpp" />
     <ClCompile Include="cd_image_memory.cpp" />
     <ClCompile Include="cd_image_pbp.cpp" />
+    <ClCompile Include="compress_helpers.cpp" />
     <ClCompile Include="cubeb_audio_stream.cpp" />
     <ClCompile Include="cue_parser.cpp" />
     <ClCompile Include="cd_image_ppf.cpp" />

src/util/util.vcxproj.filters

@@ -72,6 +72,7 @@
     <ClInclude Include="image.h" />
     <ClInclude Include="sockets.h" />
     <ClInclude Include="media_capture.h" />
+    <ClInclude Include="compress_helpers.h" />
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="state_wrapper.cpp" />
@@ -152,6 +153,7 @@
     <ClCompile Include="sdl_audio_stream.cpp" />
     <ClCompile Include="sockets.cpp" />
     <ClCompile Include="media_capture.cpp" />
+    <ClCompile Include="compress_helpers.cpp" />
   </ItemGroup>
   <ItemGroup>
     <None Include="metal_shaders.metal" />