GameList: Support parsing Redump.org dat files

src/core/CMakeLists.txt

@@ -73,7 +73,7 @@ set(RECOMPILER_SRCS
 
 target_include_directories(core PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..")
 target_include_directories(core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/..")
-target_link_libraries(core PUBLIC Threads::Threads YBaseLib common imgui)
+target_link_libraries(core PUBLIC Threads::Threads YBaseLib common imgui tinyxml2)
 target_link_libraries(core PRIVATE glad simpleini stb)
 
 if(WIN32)

src/core/core.vcxproj

@@ -110,6 +110,9 @@
     <ProjectReference Include="..\..\dep\stb\stb.vcxproj">
       <Project>{ed601289-ac1a-46b8-a8ed-17db9eb73423}</Project>
     </ProjectReference>
+    <ProjectReference Include="..\..\dep\tinyxml2\tinyxml2.vcxproj">
+      <Project>{933118a9-68c5-47b4-b151-b03c93961623}</Project>
+    </ProjectReference>
     <ProjectReference Include="..\..\dep\YBaseLib\Source\YBaseLib.vcxproj">
       <Project>{b56ce698-7300-4fa5-9609-942f1d05c5a2}</Project>
     </ProjectReference>
@@ -262,10 +265,10 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <MinimalRebuild>false</MinimalRebuild>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -284,10 +287,10 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <MinimalRebuild>false</MinimalRebuild>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -306,10 +309,10 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_ITERATOR_DEBUG_LEVEL=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUGFAST;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_ITERATOR_DEBUG_LEVEL=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUGFAST;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <MinimalRebuild>false</MinimalRebuild>
@@ -331,10 +334,10 @@
       </PrecompiledHeader>
       <WarningLevel>Level4</WarningLevel>
       <Optimization>Disabled</Optimization>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_ITERATOR_DEBUG_LEVEL=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUGFAST;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_ITERATOR_DEBUG_LEVEL=1;_CRT_SECURE_NO_WARNINGS;WIN32;_DEBUGFAST;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <SDLCheck>true</SDLCheck>
       <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <BasicRuntimeChecks>Default</BasicRuntimeChecks>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <MinimalRebuild>false</MinimalRebuild>
@@ -357,8 +360,8 @@
       </PrecompiledHeader>
       <Optimization>MaxSpeed</Optimization>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -380,8 +383,8 @@
       </PrecompiledHeader>
       <Optimization>MaxSpeed</Optimization>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <WholeProgramOptimization>true</WholeProgramOptimization>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -404,8 +407,8 @@
       </PrecompiledHeader>
       <Optimization>MaxSpeed</Optimization>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <WholeProgramOptimization>false</WholeProgramOptimization>
       <LanguageStandard>stdcpp17</LanguageStandard>
@@ -427,8 +430,8 @@
       </PrecompiledHeader>
       <Optimization>MaxSpeed</Optimization>
       <IntrinsicFunctions>true</IntrinsicFunctions>
-      <PreprocessorDefinitions>WITH_RECOMPILER=1;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
-      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+      <PreprocessorDefinitions>WITH_RECOMPILER=1;TINYXML2_IMPORT;_CRT_SECURE_NO_WARNINGS;WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <AdditionalIncludeDirectories>$(SolutionDir)dep\msvc\include;$(SolutionDir)dep\YBaseLib\Include;$(SolutionDir)dep\glad\include;$(SolutionDir)dep\stb\include;$(SolutionDir)dep\imgui\include;$(SolutionDir)dep\simpleini\include;$(SolutionDir)dep\xbyak\xbyak;$(SolutionDir)dep\tinyxml2\include;$(SolutionDir)src;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
       <WholeProgramOptimization>true</WholeProgramOptimization>
       <LanguageStandard>stdcpp17</LanguageStandard>

src/core/game_list.cpp

@@ -6,6 +6,7 @@
 #include "common/iso_reader.h"
 #include <algorithm>
 #include <cctype>
+#include <tinyxml2.h>
 #include <utility>
 Log_SetChannel(GameList);
 
@@ -78,24 +79,37 @@ std::string GameList::GetGameCodeForImage(CDImage* cdi)
   // cdrom:\SCES_123.45;1
   std::string code = iter->second;
   std::string::size_type pos = code.rfind('\\');
-  if (pos == std::string::npos)
-    return {};
-  code.erase(0, pos + 1);
-  pos = code.find(';');
-  if (pos == std::string::npos)
-    return {};
+  if (pos != std::string::npos)
+  {
+    code.erase(0, pos + 1);
+  }
+  else
+  {
+    // cdrom:SCES_123.45;1
+    pos = code.rfind(':');
+    if (pos != std::string::npos)
+      code.erase(0, pos + 1);
+  }
 
-  code.erase(pos);
+  pos = code.find(';');
+  if (pos != std::string::npos)
+    code.erase(pos);
 
   // SCES_123.45 -> SCES-12345
   for (pos = 0; pos < code.size();)
   {
-    if (code[pos] == '_')
-      code[pos++] = '-';
-    else if (code[pos] == '.')
+    if (code[pos] == '.')
+    {
       code.erase(pos, 1);
+      continue;
+    }
+
+    if (code[pos] == '_')
+      code[pos] = '-';
     else
-      pos++;
+      code[pos] = static_cast<char>(std::toupper(code[pos]));
+
+    pos++;
   }
 
   return code;
@@ -137,11 +151,24 @@ bool GameList::GetGameListEntry(const char* path, GameListEntry* entry)
     return false;
 
   std::string game_code = GetGameCodeForImage(cdi.get());
+  cdi.reset();
 
   entry->path = path;
-  entry->title = game_code;
   entry->code = game_code;
-  entry->region = GetRegionForCode(game_code).value_or(ConsoleRegion::NTSC_U);
+
+  auto iter = m_database.find(game_code);
+  if (iter != m_database.end())
+  {
+    entry->title = iter->second.title;
+    entry->region = iter->second.region;
+  }
+  else
+  {
+    Log_WarningPrintf("'%s' not found in database", game_code.c_str());
+    entry->title = game_code;
+    entry->region = GetRegionForCode(game_code).value_or(ConsoleRegion::NTSC_U);
+  }
+
   return true;
 }
 
@@ -183,3 +210,105 @@ void GameList::ScanDirectory(const char* path, bool recursive)
     }
   }
 }
+
+class RedumpDatVisitor final : public tinyxml2::XMLVisitor
+{
+public:
+  RedumpDatVisitor(GameList::DatabaseMap& database) : m_database(database) {}
+
+  static std::string FixupSerial(const std::string_view str)
+  {
+    std::string ret;
+    ret.reserve(str.length());
+    for (size_t i = 0; i < str.length(); i++)
+    {
+      if (str[i] == '.' || str[i] == '#')
+        continue;
+      else if (str[i] == ',')
+        break;
+      else if (str[i] == '_' || str[i] == ' ')
+        ret.push_back('-');
+      else
+        ret.push_back(static_cast<char>(std::toupper(str[i])));
+    }
+
+    return ret;
+  }
+
+  bool VisitEnter(const tinyxml2::XMLElement& element, const tinyxml2::XMLAttribute* firstAttribute) override
+  {
+    // recurse into gamelist
+    if (Y_stricmp(element.Name(), "datafile") == 0)
+      return true;
+
+    if (Y_stricmp(element.Name(), "game") != 0)
+      return false;
+
+    const char* name = element.Attribute("name");
+    if (!name)
+      return false;
+
+    const tinyxml2::XMLElement* serial_elem = element.FirstChildElement("serial");
+    if (!serial_elem)
+      return false;
+
+    const char* serial_text = serial_elem->GetText();
+    if (!serial_text)
+      return false;
+
+    // Handle entries like <serial>SCES-00984, SCES-00984#</serial>
+    const char* start = serial_text;
+    const char* end = std::strchr(start, ',');
+    for (;;)
+    {
+      std::string code = FixupSerial(end ? std::string_view(start, end - start) : std::string_view(start));
+      auto iter = m_database.find(code);
+      if (iter == m_database.end())
+      {
+        GameList::GameDatabaseEntry gde;
+        gde.code = std::move(code);
+        gde.region = GameList::GetRegionForCode(gde.code).value_or(ConsoleRegion::NTSC_U);
+        gde.title = name;
+        m_database.emplace(gde.code, std::move(gde));
+      }
+
+      if (!end)
+        break;
+
+      start = end + 1;
+      while (std::isspace(*start))
+        start++;
+
+      end = std::strchr(start, ',');
+    }
+
+    return false;
+  }
+
+private:
+  GameList::DatabaseMap& m_database;
+};
+
+bool GameList::ParseRedumpDatabase(const char* redump_dat_path)
+{
+  tinyxml2::XMLDocument doc;
+  tinyxml2::XMLError error = doc.LoadFile(redump_dat_path);
+  if (error != tinyxml2::XML_SUCCESS)
+  {
+    Log_ErrorPrintf("Failed to parse redump dat '%s': %s", redump_dat_path,
+                    tinyxml2::XMLDocument::ErrorIDToName(error));
+    return false;
+  }
+
+  const tinyxml2::XMLElement* datafile_elem = doc.FirstChildElement("datafile");
+  if (!datafile_elem)
+  {
+    Log_ErrorPrintf("Failed to get datafile element in '%s'", redump_dat_path);
+    return false;
+  }
+
+  RedumpDatVisitor visitor(m_database);
+  datafile_elem->Accept(&visitor);
+  Log_InfoPrintf("Loaded %zu entries from Redump.org database '%s'", m_database.size(), redump_dat_path);
+  return true;
+}

src/core/game_list.h

@@ -10,6 +10,25 @@ class CDImage;
 class GameList
 {
 public:
+  struct GameDatabaseEntry
+  {
+    std::string code;
+    std::string title;
+    ConsoleRegion region;
+  };
+
+  using DatabaseMap = std::unordered_map<std::string, GameDatabaseEntry>;
+
+  struct GameListEntry
+  {
+    std::string path;
+    std::string code;
+    std::string title;
+    ConsoleRegion region;
+  };
+
+  using EntryList = std::vector<GameListEntry>;
+
   GameList();
   ~GameList();
 
@@ -17,20 +36,19 @@ public:
   static std::string GetGameCodeForPath(const char* image_path);
   static std::optional<ConsoleRegion> GetRegionForCode(std::string_view code);
 
+  const DatabaseMap& GetDatabase() const { return m_database; }
+  const EntryList& GetEntries() const { return m_entries; }
+
   void AddDirectory(const char* path, bool recursive);
 
+  bool ParseRedumpDatabase(const char* redump_dat_path);
+
 private:
-  struct GameListEntry
-  {
-    std::string path;
-    std::string code;
-    std::string title;
-    ConsoleRegion region;
-  };
 
   bool GetGameListEntry(const char* path, GameListEntry* entry);
 
   void ScanDirectory(const char* path, bool recursive);
 
-  std::vector<GameListEntry> m_entries;
+  DatabaseMap m_database;
+  EntryList m_entries;
 };