GPU: Slightly faster path for DMA writes

src/core/dma.cpp

@@ -799,64 +799,26 @@ TickCount DMA::TransferMemoryToDevice(u32 address, u32 increment, u32 word_count
   address &= mask;
 
   const u32* src_pointer = reinterpret_cast<u32*>(Bus::g_ram + address);
-  if constexpr (channel != Channel::GPU)
+  if (static_cast<s32>(increment) < 0 || ((address + (increment * word_count)) & mask) <= address) [[unlikely]]
   {
-    if (static_cast<s32>(increment) < 0 || ((address + (increment * word_count)) & mask) <= address) [[unlikely]]
-    {
-      // Use temp buffer if it's wrapping around
-      if (s_state.transfer_buffer.size() < word_count)
-        s_state.transfer_buffer.resize(word_count);
-      src_pointer = s_state.transfer_buffer.data();
+    // Use temp buffer if it's wrapping around
+    if (s_state.transfer_buffer.size() < word_count)
+      s_state.transfer_buffer.resize(word_count);
+    src_pointer = s_state.transfer_buffer.data();
 
-      u8* ram_pointer = Bus::g_ram;
-      for (u32 i = 0; i < word_count; i++)
-      {
-        std::memcpy(&s_state.transfer_buffer[i], &ram_pointer[address], sizeof(u32));
-        address = (address + increment) & mask;
-      }
+    u8* ram_pointer = Bus::g_ram;
+    for (u32 i = 0; i < word_count; i++)
+    {
+      std::memcpy(&s_state.transfer_buffer[i], &ram_pointer[address], sizeof(u32));
+      address = (address + increment) & mask;
     }
   }
 
   switch (channel)
   {
     case Channel::GPU:
-    {
-      if (GPU::BeginDMAWrite()) [[likely]]
-      {
-        if (GPUDump::Recorder* dump = GPU::GetGPUDump()) [[unlikely]]
-        {
-          // No wraparound?
-          dump->BeginGP0Packet(word_count);
-          if (((address + (increment * (word_count - 1))) & mask) >= address) [[likely]]
-          {
-            dump->WriteWords(reinterpret_cast<const u32*>(&Bus::g_ram[address]), word_count);
-          }
-          else
-          {
-            u32 dump_address = address;
-            for (u32 i = 0; i < word_count; i++)
-            {
-              u32 value;
-              std::memcpy(&value, &Bus::g_ram[dump_address], sizeof(u32));
-              dump->WriteWord(value);
-              dump_address = (dump_address + increment) & mask;
-            }
-          }
-          dump->EndGP0Packet();
-        }
-
-        u8* ram_pointer = Bus::g_ram;
-        for (u32 i = 0; i < word_count; i++)
-        {
-          u32 value;
-          std::memcpy(&value, &ram_pointer[address], sizeof(u32));
-          GPU::DMAWrite(address, value);
-          address = (address + increment) & mask;
-        }
-        GPU::EndDMAWrite();
-      }
-    }
-    break;
+      GPU::DMAWrite(src_pointer, address, increment, word_count);
+      break;
 
     case Channel::SPU:
       SPU::DMAWrite(src_pointer, word_count);

src/core/gpu.cpp

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: CC-BY-NC-ND-4.0
 
 #include "gpu.h"
+#include "bus.h"
 #include "core.h"
 #include "cpu_pgxp.h"
 #include "dma.h"
@@ -897,7 +898,7 @@ void GPU::WriteRegister(u32 offset, u32 value)
   }
 }
 
-void GPU::DMARead(u32* words, u32 word_count)
+void GPU::DMARead(u32* RESTRICT words, u32 word_count)
 {
   if (s_locals.GPUSTAT.dma_direction != GPUDMADirection::GPUREADtoCPU)
   {
@@ -910,19 +911,45 @@ void GPU::DMARead(u32* words, u32 word_count)
     words[i] = ReadGPUREAD();
 }
 
-bool GPU::BeginDMAWrite()
+void GPU::DMAWrite(const u32* RESTRICT words, u32 address, u32 increment, u32 word_count)
 {
-  return (s_locals.GPUSTAT.dma_direction == GPUDMADirection::CPUtoGP0 ||
-          s_locals.GPUSTAT.dma_direction == GPUDMADirection::FIFO);
-}
+  if (s_locals.GPUSTAT.dma_direction != GPUDMADirection::CPUtoGP0 &&
+      s_locals.GPUSTAT.dma_direction != GPUDMADirection::FIFO)
+  {
+    return;
+  }
 
-void GPU::DMAWrite(u32 address, u32 value)
-{
-  s_locals.fifo.Push((ZeroExtend64(address) << 32) | ZeroExtend64(value));
-}
+  if (GPUDump::Recorder* dump = GPU::GetGPUDump()) [[unlikely]]
+  {
+    // No wraparound?
+    dump->BeginGP0Packet(word_count);
+    dump->WriteWords(words, word_count);
+    dump->EndGP0Packet();
+  }
+
+  const u32 mask = Bus::g_ram_mask;
+  if (const u32 contig_words = std::min(word_count, s_locals.fifo.GetContiguousSpace()); contig_words > 0)
+  {
+    const u32* RESTRICT const contig_words_end = words + contig_words;
+    u32* RESTRICT fifo_ptr = reinterpret_cast<u32*>(s_locals.fifo.GetWritePointer());
+    while (words < contig_words_end)
+    {
+      *(fifo_ptr++) = *(words++);
+      *(fifo_ptr++) = address;
+      address = (address + increment) & mask;
+    }
+
+    s_locals.fifo.AdvanceTail(contig_words);
+    word_count -= contig_words;
+  }
+
+  const u32* RESTRICT const words_end = words + word_count;
+  while (words < words_end)
+  {
+    s_locals.fifo.Push((ZeroExtend64(address) << 32) | ZeroExtend64(*(words++)));
+    address = (address + increment) & mask;
+  }
 
-void GPU::EndDMAWrite()
-{
   ExecuteCommands();
 }
 
@@ -2638,17 +2665,37 @@ static constexpr u32 ReplaceZero(u32 value, u32 value_for_zero)
 
 ALWAYS_INLINE u32 GPU::FifoPop()
 {
-  return Truncate32(s_locals.fifo.Pop());
+  // Avoid the memcpy() call in debug builds.
+  u32 ret;
+#ifdef _DEBUG
+  std::memcpy(&ret, &s_locals.fifo.Peek(), sizeof(u32));
+#else
+  ret = Truncate32(s_locals.fifo.Peek());
+#endif
+  s_locals.fifo.RemoveOne();
+  return ret;
 }
 
 ALWAYS_INLINE u32 GPU::FifoPeek()
 {
-  return Truncate32(s_locals.fifo.Peek());
+  u32 ret;
+#ifdef _DEBUG
+  std::memcpy(&ret, &s_locals.fifo.Peek(), sizeof(u32));
+#else
+  ret = Truncate32(s_locals.fifo.Peek());
+#endif
+  return ret;
 }
 
 ALWAYS_INLINE u32 GPU::FifoPeek(u32 i)
 {
-  return Truncate32(s_locals.fifo.Peek(i));
+  u32 ret;
+#ifdef _DEBUG
+  std::memcpy(&ret, &s_locals.fifo.Peek(i), sizeof(u32));
+#else
+  ret = Truncate32(s_locals.fifo.Peek(i));
+#endif
+  return ret;
 }
 
 void GPU::ExecuteCommands()

src/core/gpu.h

@@ -49,10 +49,8 @@ u32 ReadRegister(u32 offset);
 void WriteRegister(u32 offset, u32 value);
 
 // DMA access
-void DMARead(u32* words, u32 word_count);
-bool BeginDMAWrite();
-void DMAWrite(u32 address, u32 value);
-void EndDMAWrite();
+void DMARead(u32* RESTRICT words, u32 word_count);
+void DMAWrite(const u32* RESTRICT words, u32 address, u32 increment, u32 word_count);
 
 /// Writing to GPU dump.
 GPUDump::Recorder* GetGPUDump();