diff --git a/src/duckstation-qt/qthost.cpp b/src/duckstation-qt/qthost.cpp
index 661f57982..0760f600a 100644
--- a/src/duckstation-qt/qthost.cpp
+++ b/src/duckstation-qt/qthost.cpp
@@ -104,6 +104,9 @@ static constexpr int BACKGROUND_CONTROLLER_POLLING_INTERVAL_WITHOUT_DEVICES = 10
 /// Poll at half the vsync rate for FSUI to reduce the chance of getting a press+release in the same frame.
 static constexpr int FULLSCREEN_UI_CONTROLLER_POLLING_INTERVAL = 8;
 
+/// Poll at 10ms when downloads are active to ensure the speed is not impacted.
+static constexpr int DOWNLOAD_CONTROLLER_POLLING_INTERVAL = 10;
+
 /// Poll at 1ms when running GDB server. We can get rid of this once we move networking to its own thread.
 static constexpr int GDB_SERVER_POLLING_INTERVAL = 1;
 
@@ -2110,6 +2113,8 @@ int CoreThread::getBackgroundControllerPollInterval() const
 
   if (m_video_thread_run_idle)
     return FULLSCREEN_UI_CONTROLLER_POLLING_INTERVAL;
+  else if (HTTPCache::IsDownloaderActive())
+    return DOWNLOAD_CONTROLLER_POLLING_INTERVAL;
   else if (InputManager::GetPollableDeviceCount() > 0)
     return BACKGROUND_CONTROLLER_POLLING_INTERVAL_WITH_DEVICES;
   else
@@ -2150,6 +2155,11 @@ void CoreThread::updateFullscreenUITheme()
     VideoThread::RunOnThread(&FullscreenUI::UpdateTheme);
 }
 
+void Host::OnHTTPCacheDownloaderActiveChanged(bool active)
+{
+  g_core_thread->updateBackgroundControllerPollInterval();
+}
+
 void CoreThread::stop()
 {
   Assert(isRunning() && !isCurrentThread() && g_core_thread == this);
diff --git a/src/duckstation-regtest/regtest_host.cpp b/src/duckstation-regtest/regtest_host.cpp
index 982cfa285..bde40a479 100644
--- a/src/duckstation-regtest/regtest_host.cpp
+++ b/src/duckstation-regtest/regtest_host.cpp
@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2025 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2026 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: CC-BY-NC-ND-4.0
 
 #include "core/achievements.h"
@@ -24,6 +24,7 @@
 
 #include "util/cd_image.h"
 #include "util/gpu_device.h"
+#include "util/http_cache.h"
 #include "util/imgui_manager.h"
 #include "util/input_manager.h"
 #include "util/translation.h"
@@ -312,6 +313,11 @@ void Host::OnMediaCaptureStopped()
   //
 }
 
+void Host::OnHTTPCacheDownloaderActiveChanged(bool active)
+{
+  //
+}
+
 void Host::PumpMessagesOnCoreThread()
 {
   RegTestHost::ProcessCoreThreadEvents();
diff --git a/src/util/http_cache.cpp b/src/util/http_cache.cpp
index ccb09e340..6eb3f70e5 100644
--- a/src/util/http_cache.cpp
+++ b/src/util/http_cache.cpp
@@ -39,11 +39,12 @@ namespace {
 struct Locals
 {
   std::unique_ptr<HTTPDownloader> downloader;
+  bool downloader_was_active = false;
+  bool tried_initialize_cache_archive = false;
   ObjectArchive cache_archive;
   std::deque<std::pair<std::string, FetchCallback>> pending_downloads;
   std::mutex cache_mutex;
   std::once_flag downloader_once_flag;
-  bool tried_initialize_cache_archive = false;
 };
 
 } // namespace
@@ -109,9 +110,9 @@ void HTTPCache::Shutdown()
     s_locals.cache_archive.Close();
 }
 
-bool HTTPCache::HasAnyRequests()
+bool HTTPCache::IsDownloaderActive()
 {
-  return (s_locals.downloader && s_locals.downloader->HasAnyRequests());
+  return s_locals.downloader_was_active;
 }
 
 void HTTPCache::PollRequests()
@@ -120,7 +121,12 @@ void HTTPCache::PollRequests()
   if (!s_locals.downloader)
     return;
 
-  s_locals.downloader->PollRequests();
+  const bool active = s_locals.downloader->PollRequests();
+  if (s_locals.downloader_was_active != active)
+  {
+    s_locals.downloader_was_active = active;
+    Host::OnHTTPCacheDownloaderActiveChanged(active);
+  }
 }
 
 void HTTPCache::WaitForAllRequests()
diff --git a/src/util/http_cache.h b/src/util/http_cache.h
index 4632846b1..745ca30a9 100644
--- a/src/util/http_cache.h
+++ b/src/util/http_cache.h
@@ -3,10 +3,10 @@
 
 #pragma once
 
+#include "common/heap_array.h"
 #include "common/locked_ptr.h"
 #include "common/optional_with_status.h"
 #include "common/types.h"
-#include "common/heap_array.h"
 
 #include <functional>
 #include <mutex>
@@ -54,7 +54,7 @@ std::string GetUserAgent();
 void Shutdown();
 
 /// Returns true if idle updates are necessary (e.g. outstanding requests).
-bool HasAnyRequests();
+bool IsDownloaderActive();
 
 /// Processes completed HTTP requests and invokes their callbacks. Should be called regularly on the main thread.
 void PollRequests();
@@ -99,3 +99,11 @@ void Prefetch(std::string_view url, PrefetchCallback callback);
 bool Clear(Error* error);
 
 } // namespace HTTPCache
+
+namespace Host {
+
+/// Called by the HTTPDownloader implementation when the active state of the downloader changes.
+/// active is set if there are any requests in-progress.
+void OnHTTPCacheDownloaderActiveChanged(bool active);
+
+} // namespace Host
diff --git a/src/util/http_downloader.cpp b/src/util/http_downloader.cpp
index 6f058dca7..0ee65dbdd 100644
--- a/src/util/http_downloader.cpp
+++ b/src/util/http_downloader.cpp
@@ -200,10 +200,11 @@ void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
   }
 }
 
-void HTTPDownloader::PollRequests()
+bool HTTPDownloader::PollRequests()
 {
   std::unique_lock lock(m_pending_http_request_lock);
   LockedPollRequests(lock);
+  return !m_pending_http_requests.empty();
 }
 
 void HTTPDownloader::WaitForAllRequests()
diff --git a/src/util/http_downloader.h b/src/util/http_downloader.h
index c74701856..e3e242410 100644
--- a/src/util/http_downloader.h
+++ b/src/util/http_downloader.h
@@ -90,7 +90,7 @@ public:
   void CreatePostRequest(std::string url, std::string post_data, Request::Callback callback,
                          ProgressCallback* progress = nullptr, HeaderList additional_headers = {},
                          std::optional<u16> timeout_seconds = {});
-  void PollRequests();
+  bool PollRequests();
   void WaitForAllRequests();
   void WaitForAllRequestsWithYield(std::function<void()> before_sleep_cb, std::function<void()> after_sleep_cb);
   bool HasAnyRequests();