From 60b06c0e28617ed588e01a9655b05954a8dc4f3c Mon Sep 17 00:00:00 2001 From: Stenzek Date: Sat, 12 Sep 2026 23:57:15 +1000 Subject: [PATCH] ObjectArchive: Use binary keys instead of strings --- src/duckstation-qt/asyncpixmaploader.cpp | 2 +- src/util-tests/object_archive_tests.cpp | 181 +++++++++++++++++++---- src/util/http_cache.cpp | 17 ++- src/util/http_cache.h | 3 + src/util/object_archive.cpp | 103 +++++++------ src/util/object_archive.h | 24 +-- 6 files changed, 242 insertions(+), 88 deletions(-) diff --git a/src/duckstation-qt/asyncpixmaploader.cpp b/src/duckstation-qt/asyncpixmaploader.cpp index 8e3453a00..edd8b352c 100644 --- a/src/duckstation-qt/asyncpixmaploader.cpp +++ b/src/duckstation-qt/asyncpixmaploader.cpp @@ -32,7 +32,7 @@ bool AsyncPixmapLoader::isQueueNeeded(std::string_view url_or_path) // Don't try to async load when we don't have cache. const auto cache = HTTPCache::GetCacheArchive(); - return (cache->IsOpen() && !cache->Contains(url_or_path)); + return (cache->IsOpen() && !cache->Contains(HTTPCache::URLToCacheKey(url_or_path))); } static std::string_view GetExtensionFromURL(std::string_view url) diff --git a/src/util-tests/object_archive_tests.cpp b/src/util-tests/object_archive_tests.cpp index e6f780b7c..e12ff7347 100644 --- a/src/util-tests/object_archive_tests.cpp +++ b/src/util-tests/object_archive_tests.cpp @@ -13,6 +13,7 @@ #include #include +#include #include #include #include @@ -74,6 +75,41 @@ private: std::string m_blob_path; }; +class TempArchivePath +{ +public: + TempArchivePath() + { + const std::string base = Path::Combine(FileSystem::GetWorkingDirectory(), "duckstation_oa_path_test"); + std::FILE* fp = FileSystem::OpenTemporaryCFile(base, &m_base_path); + if (fp) + { + std::fclose(fp); + FileSystem::DeleteFile(m_base_path.c_str()); + } + } + + ~TempArchivePath() + { + if (!m_base_path.empty()) + { + FileSystem::DeleteFile(fmt::format("{}.idx", m_base_path).c_str()); + FileSystem::DeleteFile(fmt::format("{}.bin", m_base_path).c_str()); + } + } + + bool IsValid() const { return !m_base_path.empty(); } + const std::string& GetPath() const { return m_base_path; } + +private: + std::string m_base_path; +}; + +static ObjectArchive::KeySpan StringToCacheKey(std::string_view sv) +{ + return ObjectArchive::KeySpan(reinterpret_cast(sv.data()), sv.size()); +} + } // namespace static constexpr u32 TEST_VERSION = 1; @@ -95,6 +131,54 @@ TEST(ObjectArchive, CreateAndOpen) EXPECT_EQ(archive.GetSize(), 0u); } +TEST(ObjectArchive, OpenPathInvalidationStatus) +{ + TempArchivePath path; + ASSERT_TRUE(path.IsValid()); + + const u8 payload[] = {0xCA, 0xFE}; + Error error; + bool was_invalidated = true; + { + ObjectArchive archive; + ASSERT_TRUE(archive.OpenPath(path.GetPath(), TEST_VERSION, &error, &was_invalidated)) << error.GetDescription(); + EXPECT_FALSE(was_invalidated); + ASSERT_TRUE(archive.Insert(StringToCacheKey("persist"), payload, ObjectArchive::CompressType::Uncompressed, &error)) + << error.GetDescription(); + } + + was_invalidated = true; + { + ObjectArchive archive; + ASSERT_TRUE(archive.OpenPath(path.GetPath(), TEST_VERSION, &error, &was_invalidated)) << error.GetDescription(); + EXPECT_FALSE(was_invalidated); + EXPECT_TRUE(archive.Contains(StringToCacheKey("persist"))); + } + + was_invalidated = false; + { + ObjectArchive archive; + ASSERT_TRUE(archive.OpenPath(path.GetPath(), TEST_VERSION + 1, &error, &was_invalidated)) << error.GetDescription(); + EXPECT_TRUE(was_invalidated); + EXPECT_EQ(archive.GetSize(), 0u); + } + + const std::string index_path = fmt::format("{}.idx", path.GetPath()); + FileSystem::ManagedCFilePtr index_file = FileSystem::OpenManagedCFile(index_path.c_str(), "r+b", &error); + ASSERT_TRUE(index_file) << error.GetDescription(); + const u32 invalid_signature = 0; + ASSERT_EQ(std::fwrite(&invalid_signature, sizeof(invalid_signature), 1, index_file.get()), 1u); + index_file.reset(); + + was_invalidated = false; + { + ObjectArchive archive; + ASSERT_TRUE(archive.OpenPath(path.GetPath(), TEST_VERSION + 1, &error, &was_invalidated)) << error.GetDescription(); + EXPECT_TRUE(was_invalidated); + EXPECT_EQ(archive.GetSize(), 0u); + } +} + TEST(ObjectArchive, InsertToClosedArchive) { ObjectArchive archive; @@ -102,7 +186,7 @@ TEST(ObjectArchive, InsertToClosedArchive) const u8 data[] = {1, 2, 3}; Error error; - EXPECT_FALSE(archive.Insert("key", data, sizeof(data), ObjectArchive::CompressType::Uncompressed, &error)); + EXPECT_FALSE(archive.Insert(StringToCacheKey("key"), data, ObjectArchive::CompressType::Uncompressed, &error)); } TEST(ObjectArchive, EmptyKeyRejected) @@ -116,7 +200,7 @@ TEST(ObjectArchive, EmptyKeyRejected) ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription(); const u8 data[] = {0xAA}; - EXPECT_FALSE(archive.Insert("", std::span(data), ObjectArchive::CompressType::Uncompressed, &error)); + EXPECT_FALSE(archive.Insert({}, std::span(data), ObjectArchive::CompressType::Uncompressed, &error)); } // --------------------------------------------------------------------------- @@ -134,15 +218,61 @@ TEST(ObjectArchive, InsertAndLookupRoundTrip) ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription(); const u8 payload[] = {0xDE, 0xAD, 0xBE, 0xEF, 0x01, 0x02, 0x03, 0x04}; - ASSERT_TRUE(archive.Insert("test_key", payload, sizeof(payload), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("test_key"), payload, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); - auto result = archive.Lookup("test_key", &error); + auto result = archive.Lookup(StringToCacheKey("test_key"), &error); ASSERT_TRUE(result.has_value()) << error.GetDescription(); ASSERT_EQ(result->size(), sizeof(payload)); EXPECT_EQ(std::memcmp(result->data(), payload, sizeof(payload)), 0); } +TEST(ObjectArchive, BinaryKeyRoundTrip) +{ + TempArchiveFiles files; + ASSERT_TRUE(files.IsValid()); + + static constexpr std::array key1 = {0x00, 0x61, 0x00, 0x62, 0x00, 0xFF}; + static constexpr std::array key2 = {0x00, 0x61, 0x00, 0x62, 0x01, 0xFF}; + static constexpr std::array payload1 = {0x12, 0x34, 0x56}; + static constexpr std::array payload2 = {0xAB, 0xCD}; + const std::string_view key1_string(reinterpret_cast(key1.data()), key1.size()); + + { + ObjectArchive archive; + auto [idx, blob] = files.Release(); + Error error; + ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription(); + ASSERT_TRUE(archive.Insert(key1, payload1, ObjectArchive::CompressType::Uncompressed, &error)) + << error.GetDescription(); + ASSERT_TRUE(archive.Insert(key2, payload2, ObjectArchive::CompressType::Uncompressed, &error)) + << error.GetDescription(); + + EXPECT_TRUE(archive.Contains(key1)); + EXPECT_TRUE(archive.Contains(key2)); + EXPECT_TRUE(archive.Contains(StringToCacheKey(key1_string))); + EXPECT_FALSE(archive.Contains(ObjectArchive::KeySpan(key1).first(key1.size() - 1))); + } + + ASSERT_TRUE(files.Reopen()); + { + ObjectArchive archive; + auto [idx, blob] = files.Release(); + Error error; + ASSERT_TRUE(archive.OpenFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription(); + + const std::optional result1 = archive.Lookup(StringToCacheKey(key1_string), &error); + ASSERT_TRUE(result1.has_value()) << error.GetDescription(); + ASSERT_EQ(result1->size(), payload1.size()); + EXPECT_EQ(std::memcmp(result1->data(), payload1.data(), payload1.size()), 0); + + const std::optional result2 = archive.Lookup(ObjectArchive::KeySpan(key2), &error); + ASSERT_TRUE(result2.has_value()) << error.GetDescription(); + ASSERT_EQ(result2->size(), payload2.size()); + EXPECT_EQ(std::memcmp(result2->data(), payload2.data(), payload2.size()), 0); + } +} + // --------------------------------------------------------------------------- // Round-trip (compressed) // --------------------------------------------------------------------------- @@ -163,10 +293,10 @@ TEST(ObjectArchive, InsertAndLookupCompressed) payload[i] = static_cast(i & 0xFF); ASSERT_TRUE( - archive.Insert("compressed_key", std::span(payload), ObjectArchive::CompressType::Zstandard, &error)) + archive.Insert(StringToCacheKey("compressed_key"), payload, ObjectArchive::CompressType::Zstandard, &error)) << error.GetDescription(); - auto result = archive.Lookup("compressed_key", &error); + auto result = archive.Lookup(StringToCacheKey("compressed_key"), &error); ASSERT_TRUE(result.has_value()) << error.GetDescription(); ASSERT_EQ(result->size(), payload.size()); EXPECT_EQ(std::memcmp(result->data(), payload.data(), payload.size()), 0); @@ -188,9 +318,9 @@ TEST(ObjectArchive, DuplicateKeyRejected) const u8 data1[] = {1}; const u8 data2[] = {2}; - ASSERT_TRUE(archive.Insert("dup", std::span(data1), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("dup"), data1, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); - EXPECT_FALSE(archive.Insert("dup", std::span(data2), ObjectArchive::CompressType::Uncompressed, &error)); + EXPECT_FALSE(archive.Insert(StringToCacheKey("dup"), data2, ObjectArchive::CompressType::Uncompressed, &error)); } // --------------------------------------------------------------------------- @@ -209,10 +339,10 @@ TEST(ObjectArchive, MissingKeyReturnsNullopt) // Insert one key so the index is non-empty. const u8 data[] = {0x42}; - ASSERT_TRUE(archive.Insert("exists", std::span(data), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("exists"), data, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); - auto result = archive.Lookup("does_not_exist", &error); + auto result = archive.Lookup(StringToCacheKey("does_not_exist"), &error); EXPECT_FALSE(result.has_value()); } @@ -234,25 +364,25 @@ TEST(ObjectArchive, MultipleKeysCorrectIsolation) const u8 m_data[] = {0xBB, 0xCC}; const u8 z_data[] = {0xDD, 0xEE, 0xFF}; - ASSERT_TRUE(archive.Insert("aaa", std::span(a_data), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("aaa"), a_data, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); - ASSERT_TRUE(archive.Insert("mmm", std::span(m_data), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("mmm"), m_data, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); - ASSERT_TRUE(archive.Insert("zzz", std::span(z_data), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("zzz"), z_data, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); - auto ra = archive.Lookup("aaa", &error); + auto ra = archive.Lookup(StringToCacheKey("aaa"), &error); ASSERT_TRUE(ra.has_value()) << error.GetDescription(); ASSERT_EQ(ra->size(), sizeof(a_data)); EXPECT_EQ((*ra)[0], 0xAA); - auto rm = archive.Lookup("mmm", &error); + auto rm = archive.Lookup(StringToCacheKey("mmm"), &error); ASSERT_TRUE(rm.has_value()) << error.GetDescription(); ASSERT_EQ(rm->size(), sizeof(m_data)); EXPECT_EQ((*rm)[0], 0xBB); EXPECT_EQ((*rm)[1], 0xCC); - auto rz = archive.Lookup("zzz", &error); + auto rz = archive.Lookup(StringToCacheKey("zzz"), &error); ASSERT_TRUE(rz.has_value()) << error.GetDescription(); ASSERT_EQ(rz->size(), sizeof(z_data)); EXPECT_EQ((*rz)[0], 0xDD); @@ -273,7 +403,7 @@ TEST(ObjectArchive, ClearAndReinsert) ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription(); const u8 data1[] = {0x11, 0x22}; - ASSERT_TRUE(archive.Insert("key1", std::span(data1), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("key1"), data1, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); EXPECT_EQ(archive.GetSize(), 1u); @@ -281,16 +411,16 @@ TEST(ObjectArchive, ClearAndReinsert) EXPECT_EQ(archive.GetSize(), 0u); // After clear, lookup should fail. - auto result = archive.Lookup("key1", &error); + auto result = archive.Lookup(StringToCacheKey("key1"), &error); EXPECT_FALSE(result.has_value()); // Re-insertion should succeed. const u8 data2[] = {0x33, 0x44, 0x55}; - ASSERT_TRUE(archive.Insert("key2", std::span(data2), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("key2"), data2, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); EXPECT_EQ(archive.GetSize(), 1u); - auto result2 = archive.Lookup("key2", &error); + auto result2 = archive.Lookup(StringToCacheKey("key2"), &error); ASSERT_TRUE(result2.has_value()) << error.GetDescription(); ASSERT_EQ(result2->size(), sizeof(data2)); EXPECT_EQ(std::memcmp(result2->data(), data2, sizeof(data2)), 0); @@ -313,8 +443,7 @@ TEST(ObjectArchive, CloseAndReopenPersistence) auto [idx, blob] = files.Release(); Error error; ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription(); - ASSERT_TRUE( - archive.Insert("persist", std::span(payload), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("persist"), payload, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); archive.Close(); } @@ -328,7 +457,7 @@ TEST(ObjectArchive, CloseAndReopenPersistence) ASSERT_TRUE(archive.OpenFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription(); EXPECT_EQ(archive.GetSize(), 1u); - auto result = archive.Lookup("persist", &error); + auto result = archive.Lookup(StringToCacheKey("persist"), &error); ASSERT_TRUE(result.has_value()) << error.GetDescription(); ASSERT_EQ(result->size(), sizeof(payload)); EXPECT_EQ(std::memcmp(result->data(), payload, sizeof(payload)), 0); @@ -352,7 +481,7 @@ TEST(ObjectArchive, VersionMismatchCreatesEmpty) ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription(); const u8 data[] = {0x01, 0x02}; - ASSERT_TRUE(archive.Insert("v1_key", std::span(data), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey("v1_key"), data, ObjectArchive::CompressType::Uncompressed, &error)) << error.GetDescription(); archive.Close(); } @@ -415,7 +544,7 @@ TEST(ObjectArchive, LargeNumberOfObjectsUnsorted) std::memset(payload, 0, sizeof(payload)); std::memcpy(payload, &i, sizeof(i)); - ASSERT_TRUE(archive.Insert(key, payload, sizeof(payload), ObjectArchive::CompressType::Uncompressed, &error)) + ASSERT_TRUE(archive.Insert(StringToCacheKey(key), payload, ObjectArchive::CompressType::Uncompressed, &error)) << "Failed to insert key '" << key << "': " << error.GetDescription(); } @@ -433,7 +562,7 @@ TEST(ObjectArchive, LargeNumberOfObjectsUnsorted) for (const size_t i : lookup_order) { const std::string key = fmt::format("object_{:04}", i); - auto result = archive.Lookup(key, &error); + auto result = archive.Lookup(StringToCacheKey(key), &error); ASSERT_TRUE(result.has_value()) << "Lookup failed for key '" << key << "': " << error.GetDescription(); ASSERT_EQ(result->size(), 8u); diff --git a/src/util/http_cache.cpp b/src/util/http_cache.cpp index 030eccb3b..4e199b388 100644 --- a/src/util/http_cache.cpp +++ b/src/util/http_cache.cpp @@ -89,6 +89,11 @@ void HTTPCache::Shutdown() s_locals.cache_archive.Close(); } +std::span HTTPCache::URLToCacheKey(std::string_view key) +{ + return std::span(reinterpret_cast(key.data()), key.size()); +} + HTTPCache::CacheArchivePtr HTTPCache::GetCacheArchive() { std::unique_lock lock(s_locals.cache_mutex); @@ -113,7 +118,7 @@ HTTPCache::LookupResult HTTPCache::Lookup(std::string_view url, Error* error) const auto cache = GetCacheArchive(); Error lookup_error; - std::optional image_data = cache->Lookup(url, &lookup_error); + std::optional image_data = cache->Lookup(URLToCacheKey(url), &lookup_error); if (image_data.has_value()) { return LookupResult(LookupStatus::Hit, std::move(*image_data)); @@ -139,7 +144,7 @@ HTTPCache::LookupResult HTTPCache::LookupOrFetch(std::string_view url, Error* er const auto cache = GetCacheArchive(); Error lookup_error; - image_data = cache->Lookup(url, &lookup_error); + image_data = cache->Lookup(URLToCacheKey(url), &lookup_error); if (!image_data.has_value() && lookup_error.GetDescription() != ObjectArchive::ERROR_DESCRIPTION_DOES_NOT_EXIST) [[unlikely]] { @@ -221,7 +226,7 @@ void HTTPCache::DownloadCallback(const std::string& url, s32 status_code, const // TODO: only compress if it's images Error insert_error; - if (!cache->Insert(url, data, ObjectArchive::CompressType::Uncompressed, &insert_error)) + if (!cache->Insert(URLToCacheKey(url), data, ObjectArchive::CompressType::Uncompressed, &insert_error)) { if (insert_error.GetDescription() != ObjectArchive::ERROR_DESCRIPTION_ALREADY_EXISTS) ERROR_LOG("Failed to insert downloaded data for URL '{}' into cache: {}", url, insert_error.GetDescription()); @@ -230,14 +235,14 @@ void HTTPCache::DownloadCallback(const std::string& url, s32 status_code, const bool HTTPCache::Contains(std::string_view url) { - return GetCacheArchive()->Contains(url); + return GetCacheArchive()->Contains(URLToCacheKey(url)); } void HTTPCache::Prefetch(std::string_view url) { // skip early if already cached, or cannot prefetch const auto cache = GetCacheArchive(); - if (!cache->IsOpen() || cache->Contains(url)) [[unlikely]] + if (!cache->IsOpen() || cache->Contains(URLToCacheKey(url))) [[unlikely]] return; // queue a download with no callback, which will cause it to be cached when it completes @@ -255,7 +260,7 @@ void HTTPCache::Prefetch(std::string_view url, PrefetchCallback callback) } // skip early if already cached - if (cache->Contains(url)) + if (cache->Contains(URLToCacheKey(url))) { callback(true); return; diff --git a/src/util/http_cache.h b/src/util/http_cache.h index 54d5e0a7f..81efa11bc 100644 --- a/src/util/http_cache.h +++ b/src/util/http_cache.h @@ -49,6 +49,9 @@ std::string_view GetURLFilename(std::string_view url); /// Shuts down the HTTP cache, releasing the cache archive. void Shutdown(); +/// Converts a URL to a cache key. +std::span URLToCacheKey(std::string_view key); + /// Returns a locked pointer to the shared cache archive, opening it on first use. CacheArchivePtr GetCacheArchive(); diff --git a/src/util/object_archive.cpp b/src/util/object_archive.cpp index 4a59b6f6b..fcb39e62a 100644 --- a/src/util/object_archive.cpp +++ b/src/util/object_archive.cpp @@ -41,6 +41,18 @@ struct CacheIndexEntryHeader }; #pragma pack(pop) +ALWAYS_INLINE bool KeyLess(ObjectArchive::KeySpan lhs, ObjectArchive::KeySpan rhs) +{ + const size_t common = std::min(lhs.size(), rhs.size()); + const int cmp = std::memcmp(lhs.data(), rhs.data(), common); + return (cmp < 0 || (cmp == 0 && lhs.size() < rhs.size())); +} + +ALWAYS_INLINE bool KeyEqual(ObjectArchive::KeySpan lhs, ObjectArchive::KeySpan rhs) +{ + return (lhs.size() == rhs.size() && std::memcmp(lhs.data(), rhs.data(), lhs.size()) == 0); +} + } // namespace static constexpr u32 EXPECTED_SIGNATURE = 0x41435544; // DUCA @@ -69,6 +81,7 @@ void ObjectArchive::Close() m_blob_file = nullptr; } m_index.clear(); + m_key_pool.clear(); } bool ObjectArchive::Clear(Error* error) @@ -89,13 +102,17 @@ bool ObjectArchive::Clear(Error* error) } m_index.clear(); + m_key_pool.clear(); return true; } -bool ObjectArchive::OpenPath(std::string_view base_path, u32 data_version, Error* error) +bool ObjectArchive::OpenPath(std::string_view base_path, u32 data_version, Error* error, bool* was_invalidated) { Close(); + if (was_invalidated) + *was_invalidated = false; + const std::string index_filename = fmt::format("{}.idx", base_path); const std::string blob_filename = fmt::format("{}.bin", base_path); @@ -107,6 +124,9 @@ bool ObjectArchive::OpenPath(std::string_view base_path, u32 data_version, Error ERROR_LOG("Failed to open existing object archive index '{}': {}", Path::GetFileName(index_filename), open_error.GetDescription()); + + if (was_invalidated) + *was_invalidated = true; } return CreateNew(index_filename, blob_filename, data_version, error); @@ -240,12 +260,12 @@ bool ObjectArchive::ReadExisting(u32 version, Error* error) return false; } - // preallocate string storage, this will overshoot a bit since we don't know the actual key sizes, but it should be + // preallocate key storage, this will overshoot a bit since we don't know the actual key sizes, but it should be // good enough to avoid fragmentation and multiple resizes in most cases. - m_key_pool.Reserve(static_cast(index_file_size)); + m_key_pool.reserve(static_cast(index_file_size)); Timer timer; - std::string key; + std::vector key; for (;;) { CacheIndexEntryHeader key_header; @@ -265,9 +285,10 @@ bool ObjectArchive::ReadExisting(u32 version, Error* error) return false; } - const BumpStringPool::Offset offset = m_key_pool.AddString(key); - m_index.emplace_back(key_header.file_offset, key_header.compressed_size, key_header.uncompressed_size, - static_cast(offset), key_size, static_cast(key_header.compress_type)); + const u32 offset = static_cast(m_key_pool.size()); + m_key_pool.insert(m_key_pool.end(), key.begin(), key.end()); + m_index.emplace_back(key_header.file_offset, key_header.compressed_size, key_header.uncompressed_size, offset, + key_size, static_cast(key_header.compress_type)); } // ensure we don't write before seeking @@ -279,14 +300,14 @@ bool ObjectArchive::ReadExisting(u32 version, Error* error) // ensure index is sorted, the file is written out of order so it probably won't be std::sort(m_index.begin(), m_index.end(), - [this](const CacheIndexData& a, const CacheIndexData& b) { return (GetKeyString(a) < GetKeyString(b)); }); + [this](const CacheIndexData& a, const CacheIndexData& b) { return KeyLess(GetKeySpan(a), GetKeySpan(b)); }); // there shouldn't be any duplicates for (size_t i = 1; i < m_index.size(); i++) { - if (GetKeyString(m_index[i - 1]) == GetKeyString(m_index[i])) + if (KeyEqual(GetKeySpan(m_index[i - 1]), GetKeySpan(m_index[i]))) { - Error::SetStringFmt(error, "Duplicate key '{}' in index file, corrupt file?", GetKeyString(m_index[i])); + Error::SetStringView(error, "Duplicate key in index file, corrupt file?"); Close(); return false; } @@ -296,17 +317,17 @@ bool ObjectArchive::ReadExisting(u32 version, Error* error) return true; } -std::string_view ObjectArchive::GetKeyString(const CacheIndexData& data) const +ObjectArchive::KeySpan ObjectArchive::GetKeySpan(const CacheIndexData& data) const { - return m_key_pool.GetString(data.key_offset, data.key_size); + return KeySpan(m_key_pool.data() + data.key_offset, data.key_size); } -std::optional ObjectArchive::Lookup(std::string_view key, Error* error) +std::optional ObjectArchive::Lookup(KeySpan key, Error* error) { - const auto iter = std::lower_bound( - m_index.begin(), m_index.end(), key, - [this](const CacheIndexData& entry, const std::string_view& key) { return (GetKeyString(entry) < key); }); - if (iter == m_index.end() || GetKeyString(*iter) != key) + const auto iter = + std::lower_bound(m_index.begin(), m_index.end(), key, + [this](const CacheIndexData& entry, KeySpan key) { return KeyLess(GetKeySpan(entry), key); }); + if (iter == m_index.end() || !KeyEqual(GetKeySpan(*iter), key)) { Error::SetStringView(error, ERROR_DESCRIPTION_DOES_NOT_EXIST); return std::nullopt; @@ -316,7 +337,7 @@ std::optional ObjectArchive::Lookup(std::string_view if (std::fseek(m_blob_file, iter->file_offset, SEEK_SET) != 0 || std::fread(data.data(), iter->compressed_size, 1, m_blob_file) != 1) [[unlikely]] { - ERROR_LOG("failed to read {} byte object at offset {} for key '{}'", iter->compressed_size, iter->file_offset, key); + ERROR_LOG("failed to read {} byte object at offset {}", iter->compressed_size, iter->file_offset); Error::SetErrno(error, errno); return std::nullopt; } @@ -335,24 +356,18 @@ std::optional ObjectArchive::Lookup(std::string_view return std::optional(std::move(uncompressed_data)); } -bool ObjectArchive::Contains(std::string_view key) const +bool ObjectArchive::Contains(KeySpan key) const { if (key.empty() || key.size() > MAX_KEY_SIZE || !IsOpen()) [[unlikely]] return false; - const auto iter = std::lower_bound( - m_index.begin(), m_index.end(), key, - [this](const CacheIndexData& entry, const std::string_view& key) { return (GetKeyString(entry) < key); }); - return (iter != m_index.end() && GetKeyString(*iter) == key); -} - -bool ObjectArchive::Insert(std::string_view key, std::span data, CompressType compression, Error* error) -{ - return Insert(key, data.data(), data.size(), compression, error); + const auto iter = + std::lower_bound(m_index.begin(), m_index.end(), key, + [this](const CacheIndexData& entry, KeySpan key) { return KeyLess(GetKeySpan(entry), key); }); + return (iter != m_index.end() && KeyEqual(GetKeySpan(*iter), key)); } -bool ObjectArchive::Insert(std::string_view key, const void* data, size_t data_size, CompressType compression, - Error* error) +bool ObjectArchive::Insert(KeySpan key, std::span data, CompressType compression, Error* error) { if (key.empty() || key.size() > MAX_KEY_SIZE) [[unlikely]] { @@ -365,29 +380,28 @@ bool ObjectArchive::Insert(std::string_view key, const void* data, size_t data_s return false; } - const auto iter = std::lower_bound( - m_index.begin(), m_index.end(), key, - [this](const CacheIndexData& entry, const std::string_view& key) { return (GetKeyString(entry) < key); }); - if (iter != m_index.end() && GetKeyString(*iter) == key) + const auto iter = + std::lower_bound(m_index.begin(), m_index.end(), key, + [this](const CacheIndexData& entry, KeySpan key) { return KeyLess(GetKeySpan(entry), key); }); + if (iter != m_index.end() && KeyEqual(GetKeySpan(*iter), key)) { Error::SetStringView(error, ERROR_DESCRIPTION_ALREADY_EXISTS); return false; } DynamicHeapArray compress_buffer; - const void* write_data = data; - size_t write_size = data_size; + const void* write_data = data.data(); + size_t write_size = data.size(); if (compression != CompressType::Uncompressed) { - if (!CompressHelpers::CompressToBuffer(compress_buffer, compression, - std::span(static_cast(data), data_size), -1, error)) + if (!CompressHelpers::CompressToBuffer(compress_buffer, compression, data, -1, error)) { - ERROR_LOG("Compress {} byte object failed", data_size); + ERROR_LOG("Compress {} byte object failed", data.size()); return false; } - DEV_LOG("Cached compressed object: {} -> {} bytes ({:.1f}%)", data_size, compress_buffer.size(), - (static_cast(data_size) / static_cast(compress_buffer.size())) * 100.0f); + DEV_LOG("Cached compressed object: {} -> {} bytes ({:.1f}%)", data.size(), compress_buffer.size(), + (static_cast(data.size()) / static_cast(compress_buffer.size())) * 100.0f); write_data = compress_buffer.data(); write_size = compress_buffer.size(); @@ -406,10 +420,11 @@ bool ObjectArchive::Insert(std::string_view key, const void* data, size_t data_s CacheIndexData idata; idata.file_offset = static_cast(file_offset); idata.compressed_size = static_cast(write_size); - idata.uncompressed_size = static_cast(data_size); - idata.key_offset = static_cast(m_key_pool.AddString(key)); + idata.uncompressed_size = static_cast(data.size()); + idata.key_offset = static_cast(m_key_pool.size()); idata.key_size = static_cast(key.size()); idata.compress_type = compression; + m_key_pool.insert(m_key_pool.end(), key.begin(), key.end()); CacheIndexEntryHeader key_header = {}; key_header.file_offset = idata.file_offset; @@ -424,7 +439,7 @@ bool ObjectArchive::Insert(std::string_view key, const void* data, size_t data_s [[unlikely]] { Error::SetErrno(error, "fwrite() failed: ", errno); - ERROR_LOG("Failed to write {} byte object", data_size); + ERROR_LOG("Failed to write {} byte object", data.size()); return false; } diff --git a/src/util/object_archive.h b/src/util/object_archive.h index 1ad4e3821..719f372fc 100644 --- a/src/util/object_archive.h +++ b/src/util/object_archive.h @@ -6,11 +6,12 @@ #include "compress_helpers.h" #include "common/heap_array.h" -#include "common/string_pool.h" #include +#include #include #include +#include class Error; @@ -19,6 +20,7 @@ class ObjectArchive public: using ObjectData = DynamicHeapArray; using CompressType = CompressHelpers::CompressType; + using KeySpan = std::span; ObjectArchive(); ~ObjectArchive(); @@ -35,8 +37,9 @@ public: /// Opens or creates an archive at the given base path. The index and blob files will be named /// "{base_path}.idx" and "{base_path}.bin" respectively. If the files already exist and match - /// the given data version, they are opened; otherwise a new archive is created. - bool OpenPath(std::string_view base_path, u32 data_version, Error* error); + /// the given data version, they are opened; otherwise a new archive is created. If was_invalidated + /// is provided, it is set when an existing archive is replaced. + bool OpenPath(std::string_view base_path, u32 data_version, Error* error, bool* was_invalidated = nullptr); /// Opens an existing cache file. Ownership of the index_file and blob_file pointers are transferred to the /// ObjectArchive and they will be closed when the ObjectArchive is closed or goes out of scope. @@ -53,18 +56,17 @@ public: /// Closes the archive, releasing the index and blob file handles and clearing the in-memory index. void Close(); - /// Looks up an object by key. Returns the decompressed object data on success, or std::nullopt - /// if the key is not found or an I/O error occurs. - std::optional Lookup(std::string_view key, Error* error); + /// Looks up an object by key. String keys are treated as their exact byte sequence, including embedded nulls. + /// Returns the decompressed object data on success, or std::nullopt if the key is not found or an I/O error occurs. + std::optional Lookup(KeySpan key, Error* error); /// Returns true if the specified key exists in the archive. - bool Contains(std::string_view key) const; + bool Contains(KeySpan key) const; /// Inserts an object into the archive under the given key. The data may optionally be compressed /// using the specified compression type. Returns false if the key already exists, the archive is /// not open, or an I/O error occurs. - bool Insert(std::string_view key, std::span data, CompressType compression, Error* error); - bool Insert(std::string_view key, const void* data, size_t data_size, CompressType compression, Error* error); + bool Insert(KeySpan key, std::span data, CompressType compression, Error* error); /// Returns the total size of all objects in the cache. u64 GetTotalObjectSize() const; @@ -89,10 +91,10 @@ private: bool OpenExisting(const std::string& index_path, const std::string& blob_path, u32 version, Error* error); bool ReadExisting(u32 version, Error* error); - std::string_view GetKeyString(const CacheIndexData& data) const; + KeySpan GetKeySpan(const CacheIndexData& data) const; CacheIndex m_index; - BumpStringPool m_key_pool; + std::vector m_key_pool; std::FILE* m_index_file = nullptr; std::FILE* m_blob_file = nullptr;