GameList: Handle non-extension-suffixed urls based on content type

src/common/file_system.cpp

@@ -69,7 +69,7 @@ static inline bool FileSystemCharacterIsSane(char32_t c, bool strip_slashes)
   if (c == '*')
     return false;
 
-  // macos doesn't allow colons, apparently
+    // macos doesn't allow colons, apparently
 #ifdef __APPLE__
   if (c == U':')
     return false;
@@ -145,6 +145,37 @@ std::string Path::SanitizeFileName(const std::string_view& str, bool strip_slash
   return ret;
 }
 
+void Path::SanitizeFileName(std::string* str, bool strip_slashes /* = true */)
+{
+  const size_t len = str->length();
+
+  char small_buf[128];
+  std::unique_ptr<char[]> large_buf;
+  char* str_copy = small_buf;
+  if (len >= std::size(small_buf))
+  {
+    large_buf = std::make_unique<char[]>(len + 1);
+    str_copy = large_buf.get();
+  }
+  std::memcpy(str_copy, str->c_str(), sizeof(char) * (len + 1));
+  str->clear();
+
+  size_t pos = 0;
+  while (pos < len)
+  {
+    char32_t ch;
+    pos += StringUtil::DecodeUTF8(str_copy + pos, pos - len, &ch);
+    ch = FileSystemCharacterIsSane(ch, strip_slashes) ? ch : U'_';
+    StringUtil::EncodeAndAppendUTF8(*str, ch);
+  }
+
+#ifdef _WIN32
+  // Windows: Can't end filename with a period.
+  if (str->length() > 0 && str->back() == '.')
+    str->back() = '_';
+#endif
+}
+
 bool Path::IsAbsolute(const std::string_view& path)
 {
 #ifdef _WIN32

src/common/http_downloader.cpp

@@ -1,6 +1,7 @@
 #include "http_downloader.h"
 #include "assert.h"
 #include "log.h"
+#include "string_util.h"
 #include "timer.h"
 Log_SetChannel(HTTPDownloader);
 
@@ -100,7 +101,7 @@ void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
       m_pending_http_requests.erase(m_pending_http_requests.begin() + index);
       lock.unlock();
 
-      req->callback(-1, Request::Data());
+      req->callback(-1, std::string(), Request::Data());
 
       CloseRequest(req);
 
@@ -122,7 +123,7 @@ void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
 
     // run callback with lock unheld
     lock.unlock();
-    req->callback(req->status_code, req->data);
+    req->callback(req->status_code, std::move(req->content_type), std::move(req->data));
     CloseRequest(req);
     lock.lock();
   }
@@ -253,4 +254,97 @@ std::string HTTPDownloader::URLDecode(const std::string_view& str)
   return std::string(str);
 }
 
+std::string HTTPDownloader::GetExtensionForContentType(const std::string& content_type)
+{
+  // Based on https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+  static constexpr const char* table[][2] = {
+    {"audio/aac", "aac"},
+    {"application/x-abiword", "abw"},
+    {"application/x-freearc", "arc"},
+    {"image/avif", "avif"},
+    {"video/x-msvideo", "avi"},
+    {"application/vnd.amazon.ebook", "azw"},
+    {"application/octet-stream", "bin"},
+    {"image/bmp", "bmp"},
+    {"application/x-bzip", "bz"},
+    {"application/x-bzip2", "bz2"},
+    {"application/x-cdf", "cda"},
+    {"application/x-csh", "csh"},
+    {"text/css", "css"},
+    {"text/csv", "csv"},
+    {"application/msword", "doc"},
+    {"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"},
+    {"application/vnd.ms-fontobject", "eot"},
+    {"application/epub+zip", "epub"},
+    {"application/gzip", "gz"},
+    {"image/gif", "gif"},
+    {"text/html", "htm"},
+    {"image/vnd.microsoft.icon", "ico"},
+    {"text/calendar", "ics"},
+    {"application/java-archive", "jar"},
+    {"image/jpeg", "jpg"},
+    {"text/javascript", "js"},
+    {"application/json", "json"},
+    {"application/ld+json", "jsonld"},
+    {"audio/midi audio/x-midi", "mid"},
+    {"text/javascript", "mjs"},
+    {"audio/mpeg", "mp3"},
+    {"video/mp4", "mp4"},
+    {"video/mpeg", "mpeg"},
+    {"application/vnd.apple.installer+xml", "mpkg"},
+    {"application/vnd.oasis.opendocument.presentation", "odp"},
+    {"application/vnd.oasis.opendocument.spreadsheet", "ods"},
+    {"application/vnd.oasis.opendocument.text", "odt"},
+    {"audio/ogg", "oga"},
+    {"video/ogg", "ogv"},
+    {"application/ogg", "ogx"},
+    {"audio/opus", "opus"},
+    {"font/otf", "otf"},
+    {"image/png", "png"},
+    {"application/pdf", "pdf"},
+    {"application/x-httpd-php", "php"},
+    {"application/vnd.ms-powerpoint", "ppt"},
+    {"application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx"},
+    {"application/vnd.rar", "rar"},
+    {"application/rtf", "rtf"},
+    {"application/x-sh", "sh"},
+    {"image/svg+xml", "svg"},
+    {"application/x-tar", "tar"},
+    {"image/tiff", "tif"},
+    {"video/mp2t", "ts"},
+    {"font/ttf", "ttf"},
+    {"text/plain", "txt"},
+    {"application/vnd.visio", "vsd"},
+    {"audio/wav", "wav"},
+    {"audio/webm", "weba"},
+    {"video/webm", "webm"},
+    {"image/webp", "webp"},
+    {"font/woff", "woff"},
+    {"font/woff2", "woff2"},
+    {"application/xhtml+xml", "xhtml"},
+    {"application/vnd.ms-excel", "xls"},
+    {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"},
+    {"application/xml", "xml"},
+    {"text/xml", "xml"},
+    {"application/vnd.mozilla.xul+xml", "xul"},
+    {"application/zip", "zip"},
+    {"video/3gpp", "3gp"},
+    {"audio/3gpp", "3gp"},
+    {"video/3gpp2", "3g2"},
+    {"audio/3gpp2", "3g2"},
+    {"application/x-7z-compressed", "7z"},
+  };
+
+  std::string ret;
+  for (size_t i = 0; i < std::size(table); i++)
+  {
+    if (StringUtil::Strncasecmp(table[i][0], content_type.data(), content_type.length()) == 0)
+    {
+      ret = table[i][1];
+      break;
+    }
+  }
+  return ret;
+}
+
 } // namespace Common

src/common/http_downloader.h

@@ -21,7 +21,7 @@ public:
   struct Request
   {
     using Data = std::vector<u8>;
-    using Callback = std::function<void(s32 status_code, const Data& data)>;
+    using Callback = std::function<void(s32 status_code, std::string content_type, Data data)>;
 
     enum class Type
     {
@@ -42,6 +42,7 @@ public:
     Callback callback;
     std::string url;
     std::string post_data;
+    std::string content_type;
     Data data;
     u64 start_time;
     s32 status_code = 0;
@@ -56,6 +57,7 @@ public:
   static std::unique_ptr<HTTPDownloader> Create(const char* user_agent = DEFAULT_USER_AGENT);
   static std::string URLEncode(const std::string_view& str);
   static std::string URLDecode(const std::string_view& str);
+  static std::string GetExtensionForContentType(const std::string& content_type);
 
   void SetTimeout(float timeout);
   void SetMaxActiveRequests(u32 max_active_requests);

src/common/http_downloader_curl.cpp

@@ -88,6 +88,11 @@ void HTTPDownloaderCurl::ProcessRequest(Request* req)
     long response_code = 0;
     curl_easy_getinfo(req->handle, CURLINFO_RESPONSE_CODE, &response_code);
     req->status_code = static_cast<s32>(response_code);
+
+    char* content_type = nullptr;
+    if (!curl_easy_getinfo(req->handle, CURLINFO_CONTENT_TYPE, &content_type) && content_type)
+      req->content_type = content_type;
+
     Log_DevPrintf("Request for '%s' returned status code %d and %zu bytes", req->url.c_str(), req->status_code,
                   req->data.size());
   }
@@ -159,4 +164,4 @@ void HTTPDownloaderCurl::CloseRequest(HTTPDownloader::Request* request)
     req->closed.store(true);
 }
 
-} // namespace FrontendCommon
+} // namespace Common

src/common/http_downloader_winhttp.cpp

@@ -130,6 +130,20 @@ void CALLBACK HTTPDownloaderWinHttp::HTTPStatusCallback(HINTERNET hRequest, DWOR
         req->content_length = 0;
       }
 
+      DWORD content_type_length = 0;
+      if (!WinHttpQueryHeaders(hRequest, WINHTTP_QUERY_CONTENT_TYPE, WINHTTP_HEADER_NAME_BY_INDEX,
+                               WINHTTP_NO_OUTPUT_BUFFER, &content_type_length, WINHTTP_NO_HEADER_INDEX) &&
+          GetLastError() == ERROR_INSUFFICIENT_BUFFER && content_type_length >= sizeof(content_type_length))
+      {
+        std::wstring content_type_wstring;
+        content_type_wstring.resize((content_type_length / sizeof(wchar_t)) - 1);
+        if (WinHttpQueryHeaders(hRequest, WINHTTP_QUERY_CONTENT_TYPE, WINHTTP_HEADER_NAME_BY_INDEX,
+                                content_type_wstring.data(), &content_type_length, WINHTTP_NO_HEADER_INDEX))
+        {
+          req->content_type = StringUtil::WideStringToUTF8String(content_type_wstring);
+        }
+      }
+
       Log_DevPrintf("Status code %d, content-length is %u", req->status_code, req->content_length);
       req->data.reserve(req->content_length);
       req->state = Request::State::Receiving;
@@ -224,7 +238,7 @@ bool HTTPDownloaderWinHttp::StartRequest(HTTPDownloader::Request* request)
   if (!WinHttpCrackUrl(url_wide.c_str(), static_cast<DWORD>(url_wide.size()), 0, &uc))
   {
     Log_ErrorPrintf("WinHttpCrackUrl() failed: %u", GetLastError());
-    req->callback(-1, req->data);
+    req->callback(-1, std::string(), req->data);
     delete req;
     return false;
   }
@@ -236,7 +250,7 @@ bool HTTPDownloaderWinHttp::StartRequest(HTTPDownloader::Request* request)
   if (!req->hConnection)
   {
     Log_ErrorPrintf("Failed to start HTTP request for '%s': %u", req->url.c_str(), GetLastError());
-    req->callback(-1, req->data);
+    req->callback(-1, std::string(), req->data);
     delete req;
     return false;
   }
@@ -297,4 +311,4 @@ void HTTPDownloaderWinHttp::CloseRequest(HTTPDownloader::Request* request)
   delete req;
 }
 
-} // namespace FrontendCommon
+} // namespace Common

src/common/path.h

@@ -23,6 +23,7 @@ void Canonicalize(std::string* path);
 
 /// Sanitizes a filename for use in a filesystem.
 std::string SanitizeFileName(const std::string_view& str, bool strip_slashes = true);
+void SanitizeFileName(std::string* str, bool strip_slashes = true);
 
 /// Returns true if the specified path is an absolute path (C:\Path on Windows or /path on Unix).
 bool IsAbsolute(const std::string_view& path);

src/duckstation-qt/qtprogresscallback.cpp

@@ -115,7 +115,8 @@ void QtModalProgressCallback::checkForDelayedShow()
   }
 }
 
-QtAsyncProgressThread::QtAsyncProgressThread(QWidget* parent) : QThread(parent) {}
+// NOTE: We deliberately don't set the thread parent, because otherwise we can't move it.
+QtAsyncProgressThread::QtAsyncProgressThread(QWidget* parent) : QThread() {}
 
 QtAsyncProgressThread::~QtAsyncProgressThread() = default;
 

src/frontend-common/achievements.cpp

@@ -67,26 +67,29 @@ static Achievement* GetMutableAchievementByID(u32 id);
 static void ClearGameInfo(bool clear_achievements = true, bool clear_leaderboards = true);
 static void ClearGameHash();
 static std::string GetUserAgent();
-static void LoginCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
-static void LoginASyncCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
+static void LoginCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
+static void LoginASyncCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
 static void SendLogin(const char* username, const char* password, Common::HTTPDownloader* http_downloader,
                       Common::HTTPDownloader::Request::Callback callback);
 static void DownloadImage(std::string url, std::string cache_filename);
 static void DisplayAchievementSummary();
-static void GetUserUnlocksCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
+static void GetUserUnlocksCallback(s32 status_code, std::string content_type,
+                                   Common::HTTPDownloader::Request::Data data);
 static void GetUserUnlocks();
-static void GetPatchesCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
-static void GetLbInfoCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
+static void GetPatchesCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
+static void GetLbInfoCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
 static void GetPatches(u32 game_id);
 static std::string GetGameHash(CDImage* image);
 static void SetChallengeMode(bool enabled);
 static void SendGetGameId();
-static void GetGameIdCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
-static void SendPlayingCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
+static void GetGameIdCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
+static void SendPlayingCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
 static void UpdateRichPresence();
-static void SendPingCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
-static void UnlockAchievementCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
-static void SubmitLeaderboardCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
+static void SendPingCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
+static void UnlockAchievementCallback(s32 status_code, std::string content_type,
+                                      Common::HTTPDownloader::Request::Data data);
+static void SubmitLeaderboardCallback(s32 status_code, std::string content_type,
+                                      Common::HTTPDownloader::Request::Data data);
 
 static bool s_active = false;
 static bool s_logged_in = false;
@@ -177,7 +180,7 @@ public:
     if (error != RC_OK)
     {
       FormattedError("%s failed: error %d (%s)", RAPIStructName<T>(), error, rc_error_str(error));
-      callback(-1, Common::HTTPDownloader::Request::Data());
+      callback(-1, std::string(), Common::HTTPDownloader::Request::Data());
       return;
     }
 
@@ -825,7 +828,7 @@ void Achievements::EnsureCacheDirectoriesExist()
   }
 }
 
-void Achievements::LoginCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::LoginCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data)
 {
   std::unique_lock lock(s_achievements_mutex);
 
@@ -858,11 +861,12 @@ void Achievements::LoginCallback(s32 status_code, Common::HTTPDownloader::Reques
   }
 }
 
-void Achievements::LoginASyncCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::LoginASyncCallback(s32 status_code, std::string content_type,
+                                      Common::HTTPDownloader::Request::Data data)
 {
   ImGuiFullscreen::CloseBackgroundProgressDialog("cheevos_async_login");
 
-  LoginCallback(status_code, std::move(data));
+  LoginCallback(status_code, std::move(content_type), std::move(data));
 }
 
 void Achievements::SendLogin(const char* username, const char* password, Common::HTTPDownloader* http_downloader,
@@ -943,7 +947,8 @@ void Achievements::Logout()
 
 void Achievements::DownloadImage(std::string url, std::string cache_filename)
 {
-  auto callback = [cache_filename](s32 status_code, Common::HTTPDownloader::Request::Data data) {
+  auto callback = [cache_filename](s32 status_code, std::string content_type,
+                                   Common::HTTPDownloader::Request::Data data) {
     if (status_code != HTTP_OK)
       return;
 
@@ -998,7 +1003,8 @@ void Achievements::DisplayAchievementSummary()
   });
 }
 
-void Achievements::GetUserUnlocksCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::GetUserUnlocksCallback(s32 status_code, std::string content_type,
+                                          Common::HTTPDownloader::Request::Data data)
 {
   if (!System::IsValid())
     return;
@@ -1046,7 +1052,8 @@ void Achievements::GetUserUnlocks()
   request.Send(GetUserUnlocksCallback);
 }
 
-void Achievements::GetPatchesCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::GetPatchesCallback(s32 status_code, std::string content_type,
+                                      Common::HTTPDownloader::Request::Data data)
 {
   if (!System::IsValid())
     return;
@@ -1185,7 +1192,8 @@ void Achievements::GetPatchesCallback(s32 status_code, Common::HTTPDownloader::R
   }
 }
 
-void Achievements::GetLbInfoCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::GetLbInfoCallback(s32 status_code, std::string content_type,
+                                     Common::HTTPDownloader::Request::Data data)
 {
   if (!System::IsValid())
     return;
@@ -1275,7 +1283,8 @@ std::string Achievements::GetGameHash(CDImage* image)
   return hash_str;
 }
 
-void Achievements::GetGameIdCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::GetGameIdCallback(s32 status_code, std::string content_type,
+                                     Common::HTTPDownloader::Request::Data data)
 {
   if (!System::IsValid())
     return;
@@ -1393,7 +1402,8 @@ void Achievements::SendGetGameId()
   request.Send(GetGameIdCallback);
 }
 
-void Achievements::SendPlayingCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::SendPlayingCallback(s32 status_code, std::string content_type,
+                                       Common::HTTPDownloader::Request::Data data)
 {
   if (!System::IsValid())
     return;
@@ -1445,7 +1455,8 @@ void Achievements::UpdateRichPresence()
   Host::OnAchievementsRefreshed();
 }
 
-void Achievements::SendPingCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::SendPingCallback(s32 status_code, std::string content_type,
+                                    Common::HTTPDownloader::Request::Data data)
 {
   if (!System::IsValid())
     return;
@@ -1634,7 +1645,8 @@ void Achievements::DeactivateAchievement(Achievement* achievement)
   Log_DevPrintf("Deactivated achievement %s (%u)", achievement->title.c_str(), achievement->id);
 }
 
-void Achievements::UnlockAchievementCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::UnlockAchievementCallback(s32 status_code, std::string content_type,
+                                             Common::HTTPDownloader::Request::Data data)
 {
   if (!System::IsValid())
     return;
@@ -1649,7 +1661,8 @@ void Achievements::UnlockAchievementCallback(s32 status_code, Common::HTTPDownlo
                  response.new_player_score);
 }
 
-void Achievements::SubmitLeaderboardCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
+void Achievements::SubmitLeaderboardCallback(s32 status_code, std::string content_type,
+                                             Common::HTTPDownloader::Request::Data data)
 {
   if (!System::IsValid())
     return;

src/frontend-common/game_list.cpp

@@ -779,7 +779,7 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
     std::string filename(Common::HTTPDownloader::URLDecode(url));
     downloader->CreateRequest(
       std::move(url), [use_serial, &save_callback, entry_path = std::move(entry_path),
-                       filename = std::move(filename)](s32 status_code, Common::HTTPDownloader::Request::Data data) {
+                       filename = std::move(filename)](s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data) {
         if (status_code != Common::HTTPDownloader::HTTP_OK || data.empty())
           return;
 
@@ -788,12 +788,26 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
         if (!entry || !GetCoverImagePathForEntry(entry).empty())
           return;
 
-        std::string write_path(GetNewCoverImagePathForEntry(entry, filename.c_str(), use_serial));
+        // prefer the content type from the response for the extension
+        // otherwise, if it's missing, and the request didn't have an extension.. fall back to jpegs.
+        std::string template_filename;
+        std::string content_type_extension(Common::HTTPDownloader::GetExtensionForContentType(content_type));
+
+        // don't treat the domain name as an extension..
+        const std::string::size_type last_slash = filename.find('/');
+        const std::string::size_type last_dot = filename.find('.');
+        if (!content_type_extension.empty())
+          template_filename = fmt::format("cover.{}", content_type_extension);
+        else if (last_slash != std::string::npos && last_dot != std::string::npos && last_dot > last_slash)
+          template_filename = Path::GetFileName(filename);
+        else
+          template_filename = "cover.jpg";
+
+        std::string write_path(GetNewCoverImagePathForEntry(entry, template_filename.c_str(), use_serial));
         if (write_path.empty())
           return;
 
-        FileSystem::WriteBinaryFile(write_path.c_str(), data.data(), data.size());
-        if (save_callback)
+        if (FileSystem::WriteBinaryFile(write_path.c_str(), data.data(), data.size()) && save_callback)
           save_callback(entry, std::move(write_path));
       });
     downloader->WaitForAllRequests();