aiden
/
duckstation
mirror of https://github.com/stenzek/duckstation
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

ObjectArchive: Use binary keys instead of strings

Browse Source
pull/3797/merge
Stenzek 2 weeks ago
parent 0d2e66235c
commit 60b06c0e28
No known key found for this signature in database
6 changed files with 242 additions and 88 deletions
Show Stats Download Patch File Download Diff File

2
src/duckstation-qt/asyncpixmaploader.cpp
Unescape Escape View File

@ -32,7 +32,7 @@ bool AsyncPixmapLoader::isQueueNeeded(std::string_view url_or_path)
// Don't try to async load when we don't have cache.
const auto cache = HTTPCache::GetCacheArchive();
return (cache->IsOpen() && !cache->Contains(url_or_path));
return (cache->IsOpen() && !cache->Contains(HTTPCache::URLToCacheKey(url_or_path)));
}
static std::string_view GetExtensionFromURL(std::string_view url)

181
src/util-tests/object_archive_tests.cpp
Unescape Escape View File

@ -13,6 +13,7 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <array>
#include <cstring>
#include <numeric>
#include <vector>
@ -74,6 +75,41 @@ private:
std::string m_blob_path;
};
class TempArchivePath
{
public:
TempArchivePath()
{
const std::string base = Path::Combine(FileSystem::GetWorkingDirectory(), "duckstation_oa_path_test");
std::FILE* fp = FileSystem::OpenTemporaryCFile(base, &m_base_path);
if (fp)
{
std::fclose(fp);
FileSystem::DeleteFile(m_base_path.c_str());
}
}
~TempArchivePath()
{
if (!m_base_path.empty())
{
FileSystem::DeleteFile(fmt::format("{}.idx", m_base_path).c_str());
FileSystem::DeleteFile(fmt::format("{}.bin", m_base_path).c_str());
}
}
bool IsValid() const { return !m_base_path.empty(); }
const std::string& GetPath() const { return m_base_path; }
private:
std::string m_base_path;
};
static ObjectArchive::KeySpan StringToCacheKey(std::string_view sv)
{
return ObjectArchive::KeySpan(reinterpret_cast<const u8*>(sv.data()), sv.size());
}
} // namespace
static constexpr u32 TEST_VERSION = 1;
@ -95,6 +131,54 @@ TEST(ObjectArchive, CreateAndOpen)
EXPECT_EQ(archive.GetSize(), 0u);
}
TEST(ObjectArchive, OpenPathInvalidationStatus)
{
TempArchivePath path;
ASSERT_TRUE(path.IsValid());
const u8 payload[] = {0xCA, 0xFE};
Error error;
bool was_invalidated = true;
{
ObjectArchive archive;
ASSERT_TRUE(archive.OpenPath(path.GetPath(), TEST_VERSION, &error, &was_invalidated)) << error.GetDescription();
EXPECT_FALSE(was_invalidated);
ASSERT_TRUE(archive.Insert(StringToCacheKey("persist"), payload, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
}
was_invalidated = true;
{
ObjectArchive archive;
ASSERT_TRUE(archive.OpenPath(path.GetPath(), TEST_VERSION, &error, &was_invalidated)) << error.GetDescription();
EXPECT_FALSE(was_invalidated);
EXPECT_TRUE(archive.Contains(StringToCacheKey("persist")));
}
was_invalidated = false;
{
ObjectArchive archive;
ASSERT_TRUE(archive.OpenPath(path.GetPath(), TEST_VERSION + 1, &error, &was_invalidated)) << error.GetDescription();
EXPECT_TRUE(was_invalidated);
EXPECT_EQ(archive.GetSize(), 0u);
}
const std::string index_path = fmt::format("{}.idx", path.GetPath());
FileSystem::ManagedCFilePtr index_file = FileSystem::OpenManagedCFile(index_path.c_str(), "r+b", &error);
ASSERT_TRUE(index_file) << error.GetDescription();
const u32 invalid_signature = 0;
ASSERT_EQ(std::fwrite(&invalid_signature, sizeof(invalid_signature), 1, index_file.get()), 1u);
index_file.reset();
was_invalidated = false;
{
ObjectArchive archive;
ASSERT_TRUE(archive.OpenPath(path.GetPath(), TEST_VERSION + 1, &error, &was_invalidated)) << error.GetDescription();
EXPECT_TRUE(was_invalidated);
EXPECT_EQ(archive.GetSize(), 0u);
}
}
TEST(ObjectArchive, InsertToClosedArchive)
{
ObjectArchive archive;
@ -102,7 +186,7 @@ TEST(ObjectArchive, InsertToClosedArchive)
const u8 data[] = {1, 2, 3};
Error error;
EXPECT_FALSE(archive.Insert("key", data, sizeof(data), ObjectArchive::CompressType::Uncompressed, &error));
EXPECT_FALSE(archive.Insert(StringToCacheKey("key"), data, ObjectArchive::CompressType::Uncompressed, &error));
}
TEST(ObjectArchive, EmptyKeyRejected)
@ -116,7 +200,7 @@ TEST(ObjectArchive, EmptyKeyRejected)
ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription();
const u8 data[] = {0xAA};
EXPECT_FALSE(archive.Insert("", std::span<const u8>(data), ObjectArchive::CompressType::Uncompressed, &error));
EXPECT_FALSE(archive.Insert({}, std::span<const u8>(data), ObjectArchive::CompressType::Uncompressed, &error));
}
// ---------------------------------------------------------------------------
@ -134,15 +218,61 @@ TEST(ObjectArchive, InsertAndLookupRoundTrip)
ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription();
const u8 payload[] = {0xDE, 0xAD, 0xBE, 0xEF, 0x01, 0x02, 0x03, 0x04};
ASSERT_TRUE(archive.Insert("test_key", payload, sizeof(payload), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("test_key"), payload, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
auto result = archive.Lookup("test_key", &error);
auto result = archive.Lookup(StringToCacheKey("test_key"), &error);
ASSERT_TRUE(result.has_value()) << error.GetDescription();
ASSERT_EQ(result->size(), sizeof(payload));
EXPECT_EQ(std::memcmp(result->data(), payload, sizeof(payload)), 0);
}
TEST(ObjectArchive, BinaryKeyRoundTrip)
{
TempArchiveFiles files;
ASSERT_TRUE(files.IsValid());
static constexpr std::array<u8, 6> key1 = {0x00, 0x61, 0x00, 0x62, 0x00, 0xFF};
static constexpr std::array<u8, 6> key2 = {0x00, 0x61, 0x00, 0x62, 0x01, 0xFF};
static constexpr std::array<u8, 3> payload1 = {0x12, 0x34, 0x56};
static constexpr std::array<u8, 2> payload2 = {0xAB, 0xCD};
const std::string_view key1_string(reinterpret_cast<const char*>(key1.data()), key1.size());
{
ObjectArchive archive;
auto [idx, blob] = files.Release();
Error error;
ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription();
ASSERT_TRUE(archive.Insert(key1, payload1, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
ASSERT_TRUE(archive.Insert(key2, payload2, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
EXPECT_TRUE(archive.Contains(key1));
EXPECT_TRUE(archive.Contains(key2));
EXPECT_TRUE(archive.Contains(StringToCacheKey(key1_string)));
EXPECT_FALSE(archive.Contains(ObjectArchive::KeySpan(key1).first(key1.size() - 1)));
}
ASSERT_TRUE(files.Reopen());
{
ObjectArchive archive;
auto [idx, blob] = files.Release();
Error error;
ASSERT_TRUE(archive.OpenFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription();
const std::optional<ObjectArchive::ObjectData> result1 = archive.Lookup(StringToCacheKey(key1_string), &error);
ASSERT_TRUE(result1.has_value()) << error.GetDescription();
ASSERT_EQ(result1->size(), payload1.size());
EXPECT_EQ(std::memcmp(result1->data(), payload1.data(), payload1.size()), 0);
const std::optional<ObjectArchive::ObjectData> result2 = archive.Lookup(ObjectArchive::KeySpan(key2), &error);
ASSERT_TRUE(result2.has_value()) << error.GetDescription();
ASSERT_EQ(result2->size(), payload2.size());
EXPECT_EQ(std::memcmp(result2->data(), payload2.data(), payload2.size()), 0);
}
}
// ---------------------------------------------------------------------------
// Round-trip (compressed)
// ---------------------------------------------------------------------------
@ -163,10 +293,10 @@ TEST(ObjectArchive, InsertAndLookupCompressed)
payload[i] = static_cast<u8>(i & 0xFF);
ASSERT_TRUE(
archive.Insert("compressed_key", std::span<const u8>(payload), ObjectArchive::CompressType::Zstandard, &error))
archive.Insert(StringToCacheKey("compressed_key"), payload, ObjectArchive::CompressType::Zstandard, &error))
<< error.GetDescription();
auto result = archive.Lookup("compressed_key", &error);
auto result = archive.Lookup(StringToCacheKey("compressed_key"), &error);
ASSERT_TRUE(result.has_value()) << error.GetDescription();
ASSERT_EQ(result->size(), payload.size());
EXPECT_EQ(std::memcmp(result->data(), payload.data(), payload.size()), 0);
@ -188,9 +318,9 @@ TEST(ObjectArchive, DuplicateKeyRejected)
const u8 data1[] = {1};
const u8 data2[] = {2};
ASSERT_TRUE(archive.Insert("dup", std::span<const u8>(data1), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("dup"), data1, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
EXPECT_FALSE(archive.Insert("dup", std::span<const u8>(data2), ObjectArchive::CompressType::Uncompressed, &error));
EXPECT_FALSE(archive.Insert(StringToCacheKey("dup"), data2, ObjectArchive::CompressType::Uncompressed, &error));
}
// ---------------------------------------------------------------------------
@ -209,10 +339,10 @@ TEST(ObjectArchive, MissingKeyReturnsNullopt)
// Insert one key so the index is non-empty.
const u8 data[] = {0x42};
ASSERT_TRUE(archive.Insert("exists", std::span<const u8>(data), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("exists"), data, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
auto result = archive.Lookup("does_not_exist", &error);
auto result = archive.Lookup(StringToCacheKey("does_not_exist"), &error);
EXPECT_FALSE(result.has_value());
}
@ -234,25 +364,25 @@ TEST(ObjectArchive, MultipleKeysCorrectIsolation)
const u8 m_data[] = {0xBB, 0xCC};
const u8 z_data[] = {0xDD, 0xEE, 0xFF};
ASSERT_TRUE(archive.Insert("aaa", std::span<const u8>(a_data), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("aaa"), a_data, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
ASSERT_TRUE(archive.Insert("mmm", std::span<const u8>(m_data), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("mmm"), m_data, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
ASSERT_TRUE(archive.Insert("zzz", std::span<const u8>(z_data), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("zzz"), z_data, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
auto ra = archive.Lookup("aaa", &error);
auto ra = archive.Lookup(StringToCacheKey("aaa"), &error);
ASSERT_TRUE(ra.has_value()) << error.GetDescription();
ASSERT_EQ(ra->size(), sizeof(a_data));
EXPECT_EQ((*ra)[0], 0xAA);
auto rm = archive.Lookup("mmm", &error);
auto rm = archive.Lookup(StringToCacheKey("mmm"), &error);
ASSERT_TRUE(rm.has_value()) << error.GetDescription();
ASSERT_EQ(rm->size(), sizeof(m_data));
EXPECT_EQ((*rm)[0], 0xBB);
EXPECT_EQ((*rm)[1], 0xCC);
auto rz = archive.Lookup("zzz", &error);
auto rz = archive.Lookup(StringToCacheKey("zzz"), &error);
ASSERT_TRUE(rz.has_value()) << error.GetDescription();
ASSERT_EQ(rz->size(), sizeof(z_data));
EXPECT_EQ((*rz)[0], 0xDD);
@ -273,7 +403,7 @@ TEST(ObjectArchive, ClearAndReinsert)
ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription();
const u8 data1[] = {0x11, 0x22};
ASSERT_TRUE(archive.Insert("key1", std::span<const u8>(data1), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("key1"), data1, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
EXPECT_EQ(archive.GetSize(), 1u);
@ -281,16 +411,16 @@ TEST(ObjectArchive, ClearAndReinsert)
EXPECT_EQ(archive.GetSize(), 0u);
// After clear, lookup should fail.
auto result = archive.Lookup("key1", &error);
auto result = archive.Lookup(StringToCacheKey("key1"), &error);
EXPECT_FALSE(result.has_value());
// Re-insertion should succeed.
const u8 data2[] = {0x33, 0x44, 0x55};
ASSERT_TRUE(archive.Insert("key2", std::span<const u8>(data2), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("key2"), data2, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
EXPECT_EQ(archive.GetSize(), 1u);
auto result2 = archive.Lookup("key2", &error);
auto result2 = archive.Lookup(StringToCacheKey("key2"), &error);
ASSERT_TRUE(result2.has_value()) << error.GetDescription();
ASSERT_EQ(result2->size(), sizeof(data2));
EXPECT_EQ(std::memcmp(result2->data(), data2, sizeof(data2)), 0);
@ -313,8 +443,7 @@ TEST(ObjectArchive, CloseAndReopenPersistence)
auto [idx, blob] = files.Release();
Error error;
ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription();
ASSERT_TRUE(
archive.Insert("persist", std::span<const u8>(payload), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("persist"), payload, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
archive.Close();
}
@ -328,7 +457,7 @@ TEST(ObjectArchive, CloseAndReopenPersistence)
ASSERT_TRUE(archive.OpenFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription();
EXPECT_EQ(archive.GetSize(), 1u);
auto result = archive.Lookup("persist", &error);
auto result = archive.Lookup(StringToCacheKey("persist"), &error);
ASSERT_TRUE(result.has_value()) << error.GetDescription();
ASSERT_EQ(result->size(), sizeof(payload));
EXPECT_EQ(std::memcmp(result->data(), payload, sizeof(payload)), 0);
@ -352,7 +481,7 @@ TEST(ObjectArchive, VersionMismatchCreatesEmpty)
ASSERT_TRUE(archive.CreateFile(idx, blob, TEST_VERSION, &error)) << error.GetDescription();
const u8 data[] = {0x01, 0x02};
ASSERT_TRUE(archive.Insert("v1_key", std::span<const u8>(data), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey("v1_key"), data, ObjectArchive::CompressType::Uncompressed, &error))
<< error.GetDescription();
archive.Close();
}
@ -415,7 +544,7 @@ TEST(ObjectArchive, LargeNumberOfObjectsUnsorted)
std::memset(payload, 0, sizeof(payload));
std::memcpy(payload, &i, sizeof(i));
ASSERT_TRUE(archive.Insert(key, payload, sizeof(payload), ObjectArchive::CompressType::Uncompressed, &error))
ASSERT_TRUE(archive.Insert(StringToCacheKey(key), payload, ObjectArchive::CompressType::Uncompressed, &error))
<< "Failed to insert key '" << key << "': " << error.GetDescription();
}
@ -433,7 +562,7 @@ TEST(ObjectArchive, LargeNumberOfObjectsUnsorted)
for (const size_t i : lookup_order)
{
const std::string key = fmt::format("object_{:04}", i);
auto result = archive.Lookup(key, &error);
auto result = archive.Lookup(StringToCacheKey(key), &error);
ASSERT_TRUE(result.has_value()) << "Lookup failed for key '" << key << "': " << error.GetDescription();
ASSERT_EQ(result->size(), 8u);

17
src/util/http_cache.cpp
Unescape Escape View File

@ -89,6 +89,11 @@ void HTTPCache::Shutdown()
s_locals.cache_archive.Close();
}
std::span<const u8> HTTPCache::URLToCacheKey(std::string_view key)
{
return std::span<const u8>(reinterpret_cast<const u8*>(key.data()), key.size());
}
HTTPCache::CacheArchivePtr HTTPCache::GetCacheArchive()
{
std::unique_lock lock(s_locals.cache_mutex);
@ -113,7 +118,7 @@ HTTPCache::LookupResult HTTPCache::Lookup(std::string_view url, Error* error)
const auto cache = GetCacheArchive();
Error lookup_error;
std::optional<ObjectArchive::ObjectData> image_data = cache->Lookup(url, &lookup_error);
std::optional<ObjectArchive::ObjectData> image_data = cache->Lookup(URLToCacheKey(url), &lookup_error);
if (image_data.has_value())
{
return LookupResult(LookupStatus::Hit, std::move(*image_data));
@ -139,7 +144,7 @@ HTTPCache::LookupResult HTTPCache::LookupOrFetch(std::string_view url, Error* er
const auto cache = GetCacheArchive();
Error lookup_error;
image_data = cache->Lookup(url, &lookup_error);
image_data = cache->Lookup(URLToCacheKey(url), &lookup_error);
if (!image_data.has_value() && lookup_error.GetDescription() != ObjectArchive::ERROR_DESCRIPTION_DOES_NOT_EXIST)
[[unlikely]]
{
@ -221,7 +226,7 @@ void HTTPCache::DownloadCallback(const std::string& url, s32 status_code, const
// TODO: only compress if it's images
Error insert_error;
if (!cache->Insert(url, data, ObjectArchive::CompressType::Uncompressed, &insert_error))
if (!cache->Insert(URLToCacheKey(url), data, ObjectArchive::CompressType::Uncompressed, &insert_error))
{
if (insert_error.GetDescription() != ObjectArchive::ERROR_DESCRIPTION_ALREADY_EXISTS)
ERROR_LOG("Failed to insert downloaded data for URL '{}' into cache: {}", url, insert_error.GetDescription());
@ -230,14 +235,14 @@ void HTTPCache::DownloadCallback(const std::string& url, s32 status_code, const
bool HTTPCache::Contains(std::string_view url)
{
return GetCacheArchive()->Contains(url);
return GetCacheArchive()->Contains(URLToCacheKey(url));
}
void HTTPCache::Prefetch(std::string_view url)
{
// skip early if already cached, or cannot prefetch
const auto cache = GetCacheArchive();
if (!cache->IsOpen() || cache->Contains(url)) [[unlikely]]
if (!cache->IsOpen() || cache->Contains(URLToCacheKey(url))) [[unlikely]]
return;
// queue a download with no callback, which will cause it to be cached when it completes
@ -255,7 +260,7 @@ void HTTPCache::Prefetch(std::string_view url, PrefetchCallback callback)
}
// skip early if already cached
if (cache->Contains(url))
if (cache->Contains(URLToCacheKey(url)))
{
callback(true);
return;

3
src/util/http_cache.h
Unescape Escape View File

@ -49,6 +49,9 @@ std::string_view GetURLFilename(std::string_view url);
/// Shuts down the HTTP cache, releasing the cache archive.
void Shutdown();
/// Converts a URL to a cache key.
std::span<const u8> URLToCacheKey(std::string_view key);
/// Returns a locked pointer to the shared cache archive, opening it on first use.
CacheArchivePtr GetCacheArchive();

103
src/util/object_archive.cpp
Unescape Escape View File

@ -41,6 +41,18 @@ struct CacheIndexEntryHeader
};
#pragma pack(pop)
ALWAYS_INLINE bool KeyLess(ObjectArchive::KeySpan lhs, ObjectArchive::KeySpan rhs)
{
const size_t common = std::min(lhs.size(), rhs.size());
const int cmp = std::memcmp(lhs.data(), rhs.data(), common);
return (cmp < 0 || (cmp == 0 && lhs.size() < rhs.size()));
}
ALWAYS_INLINE bool KeyEqual(ObjectArchive::KeySpan lhs, ObjectArchive::KeySpan rhs)
{
return (lhs.size() == rhs.size() && std::memcmp(lhs.data(), rhs.data(), lhs.size()) == 0);
}
} // namespace
static constexpr u32 EXPECTED_SIGNATURE = 0x41435544; // DUCA
@ -69,6 +81,7 @@ void ObjectArchive::Close()
m_blob_file = nullptr;
}
m_index.clear();
m_key_pool.clear();
}
bool ObjectArchive::Clear(Error* error)
@ -89,13 +102,17 @@ bool ObjectArchive::Clear(Error* error)
}
m_index.clear();
m_key_pool.clear();
return true;
}
bool ObjectArchive::OpenPath(std::string_view base_path, u32 data_version, Error* error)
bool ObjectArchive::OpenPath(std::string_view base_path, u32 data_version, Error* error, bool* was_invalidated)
{
Close();
if (was_invalidated)
*was_invalidated = false;
const std::string index_filename = fmt::format("{}.idx", base_path);
const std::string blob_filename = fmt::format("{}.bin", base_path);
@ -107,6 +124,9 @@ bool ObjectArchive::OpenPath(std::string_view base_path, u32 data_version, Error
ERROR_LOG("Failed to open existing object archive index '{}': {}", Path::GetFileName(index_filename),
open_error.GetDescription());
if (was_invalidated)
*was_invalidated = true;
}
return CreateNew(index_filename, blob_filename, data_version, error);
@ -240,12 +260,12 @@ bool ObjectArchive::ReadExisting(u32 version, Error* error)
return false;
}
// preallocate string storage, this will overshoot a bit since we don't know the actual key sizes, but it should be
// preallocate key storage, this will overshoot a bit since we don't know the actual key sizes, but it should be
// good enough to avoid fragmentation and multiple resizes in most cases.
m_key_pool.Reserve(static_cast<size_t>(index_file_size));
m_key_pool.reserve(static_cast<size_t>(index_file_size));
Timer timer;
std::string key;
std::vector<u8> key;
for (;;)
{
CacheIndexEntryHeader key_header;
@ -265,9 +285,10 @@ bool ObjectArchive::ReadExisting(u32 version, Error* error)
return false;
}
const BumpStringPool::Offset offset = m_key_pool.AddString(key);
m_index.emplace_back(key_header.file_offset, key_header.compressed_size, key_header.uncompressed_size,
static_cast<u32>(offset), key_size, static_cast<CompressType>(key_header.compress_type));
const u32 offset = static_cast<u32>(m_key_pool.size());
m_key_pool.insert(m_key_pool.end(), key.begin(), key.end());
m_index.emplace_back(key_header.file_offset, key_header.compressed_size, key_header.uncompressed_size, offset,
key_size, static_cast<CompressType>(key_header.compress_type));
}
// ensure we don't write before seeking
@ -279,14 +300,14 @@ bool ObjectArchive::ReadExisting(u32 version, Error* error)
// ensure index is sorted, the file is written out of order so it probably won't be
std::sort(m_index.begin(), m_index.end(),
[this](const CacheIndexData& a, const CacheIndexData& b) { return (GetKeyString(a) < GetKeyString(b)); });
[this](const CacheIndexData& a, const CacheIndexData& b) { return KeyLess(GetKeySpan(a), GetKeySpan(b)); });
// there shouldn't be any duplicates
for (size_t i = 1; i < m_index.size(); i++)
{
if (GetKeyString(m_index[i - 1]) == GetKeyString(m_index[i]))
if (KeyEqual(GetKeySpan(m_index[i - 1]), GetKeySpan(m_index[i])))
{
Error::SetStringFmt(error, "Duplicate key '{}' in index file, corrupt file?", GetKeyString(m_index[i]));
Error::SetStringView(error, "Duplicate key in index file, corrupt file?");
Close();
return false;
}
@ -296,17 +317,17 @@ bool ObjectArchive::ReadExisting(u32 version, Error* error)
return true;
}
std::string_view ObjectArchive::GetKeyString(const CacheIndexData& data) const
ObjectArchive::KeySpan ObjectArchive::GetKeySpan(const CacheIndexData& data) const
{
return m_key_pool.GetString(data.key_offset, data.key_size);
return KeySpan(m_key_pool.data() + data.key_offset, data.key_size);
}
std::optional<ObjectArchive::ObjectData> ObjectArchive::Lookup(std::string_view key, Error* error)
std::optional<ObjectArchive::ObjectData> ObjectArchive::Lookup(KeySpan key, Error* error)
{
const auto iter = std::lower_bound(
m_index.begin(), m_index.end(), key,
[this](const CacheIndexData& entry, const std::string_view& key) { return (GetKeyString(entry) < key); });
if (iter == m_index.end() || GetKeyString(*iter) != key)
const auto iter =
std::lower_bound(m_index.begin(), m_index.end(), key,
[this](const CacheIndexData& entry, KeySpan key) { return KeyLess(GetKeySpan(entry), key); });
if (iter == m_index.end() || !KeyEqual(GetKeySpan(*iter), key))
{
Error::SetStringView(error, ERROR_DESCRIPTION_DOES_NOT_EXIST);
return std::nullopt;
@ -316,7 +337,7 @@ std::optional<ObjectArchive::ObjectData> ObjectArchive::Lookup(std::string_view
if (std::fseek(m_blob_file, iter->file_offset, SEEK_SET) != 0 ||
std::fread(data.data(), iter->compressed_size, 1, m_blob_file) != 1) [[unlikely]]
{
ERROR_LOG("failed to read {} byte object at offset {} for key '{}'", iter->compressed_size, iter->file_offset, key);
ERROR_LOG("failed to read {} byte object at offset {}", iter->compressed_size, iter->file_offset);
Error::SetErrno(error, errno);
return std::nullopt;
}
@ -335,24 +356,18 @@ std::optional<ObjectArchive::ObjectData> ObjectArchive::Lookup(std::string_view
return std::optional<ObjectData>(std::move(uncompressed_data));
}
bool ObjectArchive::Contains(std::string_view key) const
bool ObjectArchive::Contains(KeySpan key) const
{
if (key.empty() || key.size() > MAX_KEY_SIZE || !IsOpen()) [[unlikely]]
return false;
const auto iter = std::lower_bound(
m_index.begin(), m_index.end(), key,
[this](const CacheIndexData& entry, const std::string_view& key) { return (GetKeyString(entry) < key); });
return (iter != m_index.end() && GetKeyString(*iter) == key);
}
bool ObjectArchive::Insert(std::string_view key, std::span<const u8> data, CompressType compression, Error* error)
{
return Insert(key, data.data(), data.size(), compression, error);
const auto iter =
std::lower_bound(m_index.begin(), m_index.end(), key,
[this](const CacheIndexData& entry, KeySpan key) { return KeyLess(GetKeySpan(entry), key); });
return (iter != m_index.end() && KeyEqual(GetKeySpan(*iter), key));
}
bool ObjectArchive::Insert(std::string_view key, const void* data, size_t data_size, CompressType compression,
Error* error)
bool ObjectArchive::Insert(KeySpan key, std::span<const u8> data, CompressType compression, Error* error)
{
if (key.empty() || key.size() > MAX_KEY_SIZE) [[unlikely]]
{
@ -365,29 +380,28 @@ bool ObjectArchive::Insert(std::string_view key, const void* data, size_t data_s
return false;
}
const auto iter = std::lower_bound(
m_index.begin(), m_index.end(), key,
[this](const CacheIndexData& entry, const std::string_view& key) { return (GetKeyString(entry) < key); });
if (iter != m_index.end() && GetKeyString(*iter) == key)
const auto iter =
std::lower_bound(m_index.begin(), m_index.end(), key,
[this](const CacheIndexData& entry, KeySpan key) { return KeyLess(GetKeySpan(entry), key); });
if (iter != m_index.end() && KeyEqual(GetKeySpan(*iter), key))
{
Error::SetStringView(error, ERROR_DESCRIPTION_ALREADY_EXISTS);
return false;
}
DynamicHeapArray<u8> compress_buffer;
const void* write_data = data;
size_t write_size = data_size;
const void* write_data = data.data();
size_t write_size = data.size();
if (compression != CompressType::Uncompressed)
{
if (!CompressHelpers::CompressToBuffer(compress_buffer, compression,
std::span<const u8>(static_cast<const u8*>(data), data_size), -1, error))
if (!CompressHelpers::CompressToBuffer(compress_buffer, compression, data, -1, error))
{
ERROR_LOG("Compress {} byte object failed", data_size);
ERROR_LOG("Compress {} byte object failed", data.size());
return false;
}
DEV_LOG("Cached compressed object: {} -> {} bytes ({:.1f}%)", data_size, compress_buffer.size(),
(static_cast<float>(data_size) / static_cast<float>(compress_buffer.size())) * 100.0f);
DEV_LOG("Cached compressed object: {} -> {} bytes ({:.1f}%)", data.size(), compress_buffer.size(),
(static_cast<float>(data.size()) / static_cast<float>(compress_buffer.size())) * 100.0f);
write_data = compress_buffer.data();
write_size = compress_buffer.size();
@ -406,10 +420,11 @@ bool ObjectArchive::Insert(std::string_view key, const void* data, size_t data_s
CacheIndexData idata;
idata.file_offset = static_cast<u32>(file_offset);
idata.compressed_size = static_cast<u32>(write_size);
idata.uncompressed_size = static_cast<u32>(data_size);
idata.key_offset = static_cast<u32>(m_key_pool.AddString(key));
idata.uncompressed_size = static_cast<u32>(data.size());
idata.key_offset = static_cast<u32>(m_key_pool.size());
idata.key_size = static_cast<u32>(key.size());
idata.compress_type = compression;
m_key_pool.insert(m_key_pool.end(), key.begin(), key.end());
CacheIndexEntryHeader key_header = {};
key_header.file_offset = idata.file_offset;
@ -424,7 +439,7 @@ bool ObjectArchive::Insert(std::string_view key, const void* data, size_t data_s
[[unlikely]]
{
Error::SetErrno(error, "fwrite() failed: ", errno);
ERROR_LOG("Failed to write {} byte object", data_size);
ERROR_LOG("Failed to write {} byte object", data.size());
return false;
}

24
src/util/object_archive.h
Unescape Escape View File

@ -6,11 +6,12 @@
#include "compress_helpers.h"
#include "common/heap_array.h"
#include "common/string_pool.h"
#include <optional>
#include <span>
#include <string>
#include <string_view>
#include <vector>
class Error;
@ -19,6 +20,7 @@ class ObjectArchive
public:
using ObjectData = DynamicHeapArray<u8>;
using CompressType = CompressHelpers::CompressType;
using KeySpan = std::span<const u8>;
ObjectArchive();
~ObjectArchive();
@ -35,8 +37,9 @@ public:
/// Opens or creates an archive at the given base path. The index and blob files will be named
/// "{base_path}.idx" and "{base_path}.bin" respectively. If the files already exist and match
/// the given data version, they are opened; otherwise a new archive is created.
bool OpenPath(std::string_view base_path, u32 data_version, Error* error);
/// the given data version, they are opened; otherwise a new archive is created. If was_invalidated
/// is provided, it is set when an existing archive is replaced.
bool OpenPath(std::string_view base_path, u32 data_version, Error* error, bool* was_invalidated = nullptr);
/// Opens an existing cache file. Ownership of the index_file and blob_file pointers are transferred to the
/// ObjectArchive and they will be closed when the ObjectArchive is closed or goes out of scope.
@ -53,18 +56,17 @@ public:
/// Closes the archive, releasing the index and blob file handles and clearing the in-memory index.
void Close();
/// Looks up an object by key. Returns the decompressed object data on success, or std::nullopt
/// if the key is not found or an I/O error occurs.
std::optional<ObjectData> Lookup(std::string_view key, Error* error);
/// Looks up an object by key. String keys are treated as their exact byte sequence, including embedded nulls.
/// Returns the decompressed object data on success, or std::nullopt if the key is not found or an I/O error occurs.
std::optional<ObjectData> Lookup(KeySpan key, Error* error);
/// Returns true if the specified key exists in the archive.
bool Contains(std::string_view key) const;
bool Contains(KeySpan key) const;
/// Inserts an object into the archive under the given key. The data may optionally be compressed
/// using the specified compression type. Returns false if the key already exists, the archive is
/// not open, or an I/O error occurs.
bool Insert(std::string_view key, std::span<const u8> data, CompressType compression, Error* error);
bool Insert(std::string_view key, const void* data, size_t data_size, CompressType compression, Error* error);
bool Insert(KeySpan key, std::span<const u8> data, CompressType compression, Error* error);
/// Returns the total size of all objects in the cache.
u64 GetTotalObjectSize() const;
@ -89,10 +91,10 @@ private:
bool OpenExisting(const std::string& index_path, const std::string& blob_path, u32 version, Error* error);
bool ReadExisting(u32 version, Error* error);
std::string_view GetKeyString(const CacheIndexData& data) const;
KeySpan GetKeySpan(const CacheIndexData& data) const;
CacheIndex m_index;
BumpStringPool m_key_pool;
std::vector<u8> m_key_pool;
std::FILE* m_index_file = nullptr;
std::FILE* m_blob_file = nullptr;

Write Preview
Loading…
Cancel
Save