MediaCapture: Improve ffmpeg hardware encoding

- Don't silently fall back to software encoding if a hardware encoder is
  supported.
- Fix forced yuv420p input to encoders, select an appropriate input
  based on what is supported.
- Allocate hardware surfaces for each frame to ensure we're not
  trampling on in-flight data.
- Check supported video dimensions and return an appropriate error if
  they are out of bounds.

src/util/media_capture.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2026 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: CC-BY-NC-ND-4.0
 
 #include "media_capture.h"
@@ -56,6 +56,8 @@ extern "C" {
 #include "libavformat/version.h"
 #include "libavutil/dict.h"
 #include "libavutil/ffversion.h"
+#include "libavutil/hwcontext.h"
+#include "libavutil/mem.h"
 #include "libavutil/opt.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/version.h"
@@ -1943,6 +1945,7 @@ bool MediaCaptureMF::ProcessAudioPackets(s64 video_pts, Error* error)
   X(av_channel_layout_copy)                                                                                            \
   X(av_opt_set_chlayout)                                                                                               \
   X(av_frame_alloc)                                                                                                    \
+  X(av_frame_unref)                                                                                                    \
   X(av_frame_get_buffer)                                                                                               \
   X(av_frame_free)                                                                                                     \
   X(av_frame_make_writable)                                                                                            \
@@ -1955,15 +1958,20 @@ bool MediaCaptureMF::ProcessAudioPackets(s64 video_pts, Error* error)
   X(av_opt_set_sample_fmt)                                                                                             \
   X(av_compare_ts)                                                                                                     \
   X(av_get_pix_fmt)                                                                                                    \
+  X(av_get_pix_fmt_name)                                                                                               \
   X(av_get_bytes_per_sample)                                                                                           \
   X(av_sample_fmt_is_planar)                                                                                           \
   X(av_d2q)                                                                                                            \
   X(av_hwdevice_get_type_name)                                                                                         \
   X(av_hwdevice_ctx_create)                                                                                            \
+  X(av_hwdevice_get_hwframe_constraints)                                                                               \
   X(av_hwframe_ctx_alloc)                                                                                              \
   X(av_hwframe_ctx_init)                                                                                               \
+  X(av_hwframe_constraints_free)                                                                                       \
   X(av_hwframe_transfer_data)                                                                                          \
+  X(av_hwframe_transfer_get_formats)                                                                                   \
   X(av_hwframe_get_buffer)                                                                                             \
+  X(av_free)                                                                                                           \
   X(av_buffer_ref)                                                                                                     \
   X(av_buffer_unref)
 
@@ -2272,111 +2280,306 @@ bool MediaCaptureFFmpeg::InternalBeginCapture(float fps, float aspect, u32 sampl
       }
     }
 
-    // Select output pixel format.
+    // Query the codec's software pixel formats before trying hardware. A null list means that the
+    // codec accepts any format, whereas a non-null empty list means that it accepts none.
+    const AVPixelFormat* supported_pixel_formats = nullptr;
+    int num_supported_pixel_formats = 0;
+#ifdef HAS_AVCODEC_GET_SUPPORTED_CONFIG
+    res = wrap_avcodec_get_supported_config(m_video_codec_context, vcodec, AV_CODEC_CONFIG_PIX_FORMAT, 0,
+                                            reinterpret_cast<const void**>(&supported_pixel_formats),
+                                            &num_supported_pixel_formats);
+    if (res < 0)
+    {
+      SetAVError(error, "avcodec_get_supported_config() failed: ", res);
+      return false;
+    }
+#else
+    supported_pixel_formats = vcodec->pix_fmts;
+    if (supported_pixel_formats)
+    {
+      while (supported_pixel_formats[num_supported_pixel_formats] != AV_PIX_FMT_NONE)
+        num_supported_pixel_formats++;
+    }
+#endif
+
+    const bool has_pixel_format_list = (supported_pixel_formats != nullptr);
+
+    // An explicitly selected hardware encoder must not silently turn into software encoding. The
+    // capability flag is authoritative where available, while the hardware configuration check
+    // also covers encoders which expose hardware frames without setting the flag consistently.
+    bool has_hardware_frames_config = false;
+    for (int hw_index = 0;; hw_index++)
+    {
+      const AVCodecHWConfig* config = wrap_avcodec_get_hw_config(vcodec, hw_index);
+      if (!config)
+        break;
+      if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) != 0)
+      {
+        has_hardware_frames_config = true;
+        break;
+      }
+    }
+    const bool requested_hardware_encoder =
+      !video_codec.empty() &&
+      ((vcodec->capabilities & (AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_HYBRID)) != 0 || has_hardware_frames_config);
+
+    // Select the requested format when possible, then use formats that are broadly supported by
+    // software and hardware encoders. NV12 is intentionally preferred for hardware frames: some
+    // APIs (notably D3D11) give YUV420P special opaque-surface semantics which cannot be copied to.
     AVPixelFormat request_pix_fmt = AV_PIX_FMT_YUV420P;
+    bool has_pixel_format_override = false;
     if (const AVDictionaryEntry* de = wrap_av_dict_get(m_video_codec_arguments, "pixel_format", nullptr, 0))
     {
+      has_pixel_format_override = true;
       const AVPixelFormat de_fmt = wrap_av_get_pix_fmt(de->value);
-      request_pix_fmt = (de_fmt != AV_PIX_FMT_NONE) ? de_fmt : request_pix_fmt;
-      if (de_fmt == AV_PIX_FMT_NONE)
+      if (de_fmt != AV_PIX_FMT_NONE)
+        request_pix_fmt = de_fmt;
+      else
         WARNING_LOG("Invalid pixel format override: {}", de->value);
     }
 
-    // Can we use hardware encoding?
-    const AVCodecHWConfig* hwconfig = wrap_avcodec_get_hw_config(vcodec, 0);
-    AVPixelFormat sw_pix_fmt = request_pix_fmt;
-    if (hwconfig)
+    static constexpr std::array<AVPixelFormat, 7> software_pixel_format_preference = {
+      AV_PIX_FMT_YUV420P,     AV_PIX_FMT_NV12, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P,
+      AV_PIX_FMT_YUV420P10LE, AV_PIX_FMT_P010, AV_PIX_FMT_BGRA,
+    };
+    static constexpr std::array<AVPixelFormat, 7> hardware_pixel_format_preference = {
+      AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P, AV_PIX_FMT_P010,    AV_PIX_FMT_YUV420P10LE,
+      AV_PIX_FMT_BGRA, AV_PIX_FMT_RGBA,    AV_PIX_FMT_YUYV422,
+    };
+
+    const auto contains_pixel_format = [](const AVPixelFormat* formats, int count, AVPixelFormat format) {
+      return (!formats || std::find(formats, formats + count, format) != formats + count);
+    };
+    const auto select_pixel_format = [&](const AVPixelFormat* formats, int count, AVPixelFormat requested,
+                                         const auto& preference, const auto& is_allowed) {
+      if (requested != AV_PIX_FMT_NONE && contains_pixel_format(formats, count, requested) && is_allowed(requested))
+        return requested;
+
+      for (const AVPixelFormat format : preference)
+      {
+        if (contains_pixel_format(formats, count, format) && is_allowed(format))
+          return format;
+      }
+
+      if (formats)
+      {
+        for (int i = 0; i < count; i++)
+        {
+          if (is_allowed(formats[i]))
+            return formats[i];
+        }
+      }
+
+      return AV_PIX_FMT_NONE;
+    };
+
+    AVPixelFormat sw_pix_fmt = AV_PIX_FMT_NONE;
+    const AVCodecHWConfig* hwconfig = nullptr;
+    bool hardware_dimensions_too_large = false;
+    u32 hardware_max_width = 0;
+    u32 hardware_max_height = 0;
+
+    // Try each frames-context hardware configuration. The first configuration is not necessarily
+    // usable for an encoder, and a codec may expose several devices with different transfer formats.
+    for (int hw_index = 0; (hwconfig = wrap_avcodec_get_hw_config(vcodec, hw_index)) != nullptr; hw_index++)
     {
-      // Can't do this test for hardware codecs, because they don't list the software formats as inputs.
+      if (!(hwconfig->methods & AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX) || hwconfig->pix_fmt == AV_PIX_FMT_NONE)
+        continue;
+
       Error hw_error;
+      AVBufferRef* hw_context = nullptr;
+      AVBufferRef* hw_frames = nullptr;
       INFO_LOG("Trying to use {} hardware device for video encoding.",
                wrap_av_hwdevice_get_type_name(hwconfig->device_type));
-      res = wrap_av_hwdevice_ctx_create(&m_video_hw_context, hwconfig->device_type, nullptr, nullptr, 0);
+
+      res = wrap_av_hwdevice_ctx_create(&hw_context, hwconfig->device_type, nullptr, nullptr, 0);
       if (res < 0)
       {
         SetAVError(&hw_error, "av_hwdevice_ctx_create() failed: ", res);
         ERROR_LOG(hw_error.GetDescription());
+        continue;
       }
-      else
+
+      AVHWFramesConstraints* constraints = wrap_av_hwdevice_get_hwframe_constraints(hw_context, nullptr);
+      if (!constraints)
+      {
+        ERROR_LOG("av_hwdevice_get_hwframe_constraints() failed");
+        wrap_av_buffer_unref(&hw_context);
+        continue;
+      }
+
+      if (constraints->min_width > m_video_codec_context->width ||
+          constraints->min_height > m_video_codec_context->height ||
+          constraints->max_width < m_video_codec_context->width ||
+          constraints->max_height < m_video_codec_context->height)
       {
-        m_video_hw_frames = wrap_av_hwframe_ctx_alloc(m_video_hw_context);
-        if (!m_video_hw_frames)
+        if (constraints->max_width < m_video_codec_context->width ||
+            constraints->max_height < m_video_codec_context->height)
         {
-          ERROR_LOG("s_video_hw_frames() failed");
-          wrap_av_buffer_unref(&m_video_hw_context);
+          hardware_dimensions_too_large = true;
+          hardware_max_width = std::max(hardware_max_width, static_cast<u32>(std::max(constraints->max_width, 0)));
+          hardware_max_height = std::max(hardware_max_height, static_cast<u32>(std::max(constraints->max_height, 0)));
         }
-        else
+        ERROR_LOG("Video dimensions are outside the hardware frame constraints.");
+        wrap_av_hwframe_constraints_free(&constraints);
+        wrap_av_buffer_unref(&hw_context);
+        continue;
+      }
+
+      int num_valid_sw_formats = 0;
+      if (constraints->valid_sw_formats)
+      {
+        while (constraints->valid_sw_formats[num_valid_sw_formats] != AV_PIX_FMT_NONE)
+          num_valid_sw_formats++;
+      }
+
+      bool codec_has_valid_sw_format = false;
+      if (has_pixel_format_list && constraints->valid_sw_formats)
+      {
+        for (int i = 0; i < num_valid_sw_formats; i++)
         {
-          AVHWFramesContext* frames_ctx = reinterpret_cast<AVHWFramesContext*>(m_video_hw_frames->data);
-          frames_ctx->format = (hwconfig->pix_fmt != AV_PIX_FMT_NONE) ? hwconfig->pix_fmt : sw_pix_fmt;
-          frames_ctx->sw_format = sw_pix_fmt;
-          frames_ctx->width = m_video_codec_context->width;
-          frames_ctx->height = m_video_codec_context->height;
-          res = wrap_av_hwframe_ctx_init(m_video_hw_frames);
-          if (res < 0)
-          {
-            SetAVError(&hw_error, "av_hwframe_ctx_init() failed: ", res);
-            ERROR_LOG(hw_error.GetDescription());
-            wrap_av_buffer_unref(&m_video_hw_frames);
-            wrap_av_buffer_unref(&m_video_hw_context);
-          }
-          else
+          if (contains_pixel_format(supported_pixel_formats, num_supported_pixel_formats,
+                                    constraints->valid_sw_formats[i]))
           {
-            m_video_codec_context->hw_frames_ctx = wrap_av_buffer_ref(m_video_hw_frames);
-            if (hwconfig->pix_fmt != AV_PIX_FMT_NONE)
-              m_video_codec_context->pix_fmt = hwconfig->pix_fmt;
+            codec_has_valid_sw_format = true;
+            break;
           }
         }
       }
+      const auto is_allowed_sw_format = [&](AVPixelFormat format) {
+        return !codec_has_valid_sw_format ||
+               contains_pixel_format(supported_pixel_formats, num_supported_pixel_formats, format);
+      };
+
+      // valid_sw_formats describes formats that can actually be used to initialize this device.
+      // Unlike av_hwframe_transfer_get_formats(), it lists alternatives rather than just echoing
+      // the sw_format already selected in the frames context.
+      const AVPixelFormat requested_hw_format = has_pixel_format_override ? request_pix_fmt : AV_PIX_FMT_NONE;
+      sw_pix_fmt = select_pixel_format(constraints->valid_sw_formats, num_valid_sw_formats, requested_hw_format,
+                                       hardware_pixel_format_preference, is_allowed_sw_format);
+      if (sw_pix_fmt == AV_PIX_FMT_NONE)
+      {
+        ERROR_LOG("Hardware encoder does not support a usable software pixel format.");
+        wrap_av_hwframe_constraints_free(&constraints);
+        wrap_av_buffer_unref(&hw_context);
+        continue;
+      }
+      wrap_av_hwframe_constraints_free(&constraints);
+
+      const auto create_hw_frames = [&](AVPixelFormat sw_format) {
+        AVBufferRef* frames = wrap_av_hwframe_ctx_alloc(hw_context);
+        if (!frames)
+        {
+          ERROR_LOG("av_hwframe_ctx_alloc() failed");
+          return static_cast<AVBufferRef*>(nullptr);
+        }
+
+        AVHWFramesContext* frames_ctx = reinterpret_cast<AVHWFramesContext*>(frames->data);
+        frames_ctx->format = hwconfig->pix_fmt;
+        frames_ctx->sw_format = sw_format;
+        frames_ctx->width = m_video_codec_context->width;
+        frames_ctx->height = m_video_codec_context->height;
+        const int init_res = wrap_av_hwframe_ctx_init(frames);
+        if (init_res < 0)
+        {
+          SetAVError(&hw_error, "av_hwframe_ctx_init() failed: ", init_res);
+          ERROR_LOG(hw_error.GetDescription());
+          wrap_av_buffer_unref(&frames);
+          return static_cast<AVBufferRef*>(nullptr);
+        }
+
+        return frames;
+      };
+
+      hw_frames = create_hw_frames(sw_pix_fmt);
+      if (!hw_frames)
+      {
+        wrap_av_buffer_unref(&hw_context);
+        continue;
+      }
+
+      // Validate that the selected software format can actually be transferred to the hardware
+      // frame. Some implementations return only the selected sw_format from this query.
+      AVPixelFormat* transfer_formats = nullptr;
+      res = wrap_av_hwframe_transfer_get_formats(hw_frames, AV_HWFRAME_TRANSFER_DIRECTION_TO, &transfer_formats, 0);
+      if (res < 0)
+      {
+        SetAVError(&hw_error, "av_hwframe_transfer_get_formats() failed: ", res);
+        ERROR_LOG(hw_error.GetDescription());
+        wrap_av_buffer_unref(&hw_frames);
+        wrap_av_buffer_unref(&hw_context);
+        continue;
+      }
 
-      if (!m_video_hw_context)
+      int num_transfer_formats = 0;
+      while (transfer_formats[num_transfer_formats] != AV_PIX_FMT_NONE)
+        num_transfer_formats++;
+      if (!contains_pixel_format(transfer_formats, num_transfer_formats, sw_pix_fmt))
       {
-        ERROR_LOG("Failed to create hardware encoder, using software encoding.");
-        hwconfig = nullptr;
+        ERROR_LOG("Hardware encoder does not support a transferable software pixel format.");
+        wrap_av_free(transfer_formats);
+        wrap_av_buffer_unref(&hw_frames);
+        wrap_av_buffer_unref(&hw_context);
+        continue;
       }
+      wrap_av_free(transfer_formats);
+
+      m_video_hw_context = hw_context;
+      m_video_hw_frames = hw_frames;
+      m_video_codec_context->hw_frames_ctx = wrap_av_buffer_ref(m_video_hw_frames);
+      if (!m_video_codec_context->hw_frames_ctx)
+      {
+        ERROR_LOG("av_buffer_ref() failed for hardware frames context.");
+        wrap_av_buffer_unref(&m_video_hw_frames);
+        wrap_av_buffer_unref(&m_video_hw_context);
+        continue;
+      }
+      m_video_codec_context->pix_fmt = hwconfig->pix_fmt;
+      break;
     }
 
-    if (!hwconfig)
+    if (requested_hardware_encoder && !m_video_hw_context)
     {
-      // Default to YUV 4:2:0 if the codec doesn't specify a pixel format.
-      const AVPixelFormat* supported_pixel_formats = nullptr;
-      int num_supported_pixel_formats = 0;
-#ifdef HAS_AVCODEC_GET_SUPPORTED_CONFIG
-      res = wrap_avcodec_get_supported_config(m_video_codec_context, vcodec, AV_CODEC_CONFIG_PIX_FORMAT, 0,
-                                              reinterpret_cast<const void**>(&supported_pixel_formats),
-                                              &num_supported_pixel_formats);
-      if (res < 0)
+      if (hardware_dimensions_too_large)
       {
-        SetAVError(error, "avcodec_get_supported_config() failed: ", res);
-        return false;
+        const u32 maximum_width = (hardware_max_width != 0) ? hardware_max_width : m_video_width;
+        const u32 maximum_height = (hardware_max_height != 0) ? hardware_max_height : m_video_height;
+        Error::SetStringFmt(
+          error, "Video dimensions {}x{} exceed the maximum supported dimensions {}x{} for hardware encoder '{}'.",
+          m_video_width, m_video_height, maximum_width, maximum_height, video_codec);
       }
-#else
-      supported_pixel_formats = vcodec->pix_fmts;
-      if (supported_pixel_formats)
+      else
       {
-        while (supported_pixel_formats[num_supported_pixel_formats] != AV_PIX_FMT_NONE)
-          num_supported_pixel_formats++;
+        Error::SetStringFmt(error, "Failed to initialize requested hardware encoder '{}'.", video_codec);
       }
-#endif
+      return false;
+    }
 
-      if (!supported_pixel_formats || num_supported_pixel_formats == 0)
+    if (!hwconfig || !m_video_hw_context)
+    {
+      // For software encoders a null list means that all pixel formats are accepted.
+      const auto is_allowed_software_format = [](AVPixelFormat) { return true; };
+      sw_pix_fmt = select_pixel_format(supported_pixel_formats, num_supported_pixel_formats, request_pix_fmt,
+                                       software_pixel_format_preference, is_allowed_software_format);
+      if (sw_pix_fmt == AV_PIX_FMT_NONE)
       {
-        Error::SetStringView(error, "Video codec supports no formats.");
+        Error::SetStringView(error, "Video codec supports no usable pixel formats.");
         return false;
       }
 
-      // Prefer YUV420 given the choice, but otherwise fall back to whatever it supports.
-      sw_pix_fmt = supported_pixel_formats[0];
-      for (int i = 0; i < num_supported_pixel_formats; i++)
+      m_video_codec_context->pix_fmt = sw_pix_fmt;
+      if (has_pixel_format_override && request_pix_fmt != sw_pix_fmt)
       {
-        if (supported_pixel_formats[i] == request_pix_fmt)
-        {
-          sw_pix_fmt = supported_pixel_formats[i];
-          break;
-        }
+        WARNING_LOG("Requested video pixel format was not supported; using {} instead.",
+                    wrap_av_get_pix_fmt_name(sw_pix_fmt));
+      }
+      if (m_video_hw_context)
+      {
+        ERROR_LOG("Failed to create a compatible hardware encoder, using software encoding.");
+        wrap_av_buffer_unref(&m_video_hw_frames);
+        wrap_av_buffer_unref(&m_video_hw_context);
+        wrap_av_buffer_unref(&m_video_codec_context->hw_frames_ctx);
       }
-
-      m_video_codec_context->pix_fmt = sw_pix_fmt;
     }
 
     if (output_format->flags & AVFMT_GLOBALHEADER)
@@ -2389,6 +2592,16 @@ bool MediaCaptureFFmpeg::InternalBeginCapture(float fps, float aspect, u32 sampl
       return false;
     }
 
+    // Codec options may adjust the final software format during open. Use the format reported by
+    // the opened software encoder so swscale and avcodec receive the same format.
+    if (!IsUsingHardwareVideoEncoding())
+      sw_pix_fmt = m_video_codec_context->pix_fmt;
+    if (sw_pix_fmt == AV_PIX_FMT_NONE)
+    {
+      Error::SetStringView(error, "Video encoder did not select a pixel format.");
+      return false;
+    }
+
     m_converted_video_frame = wrap_av_frame_alloc();
     m_hw_video_frame = IsUsingHardwareVideoEncoding() ? wrap_av_frame_alloc() : nullptr;
     if (!m_converted_video_frame || (IsUsingHardwareVideoEncoding() && !m_hw_video_frame))
@@ -2407,19 +2620,6 @@ bool MediaCaptureFFmpeg::InternalBeginCapture(float fps, float aspect, u32 sampl
       return false;
     }
 
-    if (IsUsingHardwareVideoEncoding())
-    {
-      m_hw_video_frame->format = m_video_codec_context->pix_fmt;
-      m_hw_video_frame->width = m_video_codec_context->width;
-      m_hw_video_frame->height = m_video_codec_context->height;
-      res = wrap_av_hwframe_get_buffer(m_video_hw_frames, m_hw_video_frame, 0);
-      if (res < 0)
-      {
-        SetAVError(error, "av_frame_get_buffer() for HW frame failed: ", res);
-        return false;
-      }
-    }
-
     m_video_stream = wrap_avformat_new_stream(m_format_context, vcodec);
     if (!m_video_stream)
     {
@@ -2453,7 +2653,7 @@ bool MediaCaptureFFmpeg::InternalBeginCapture(float fps, float aspect, u32 sampl
       acodec = wrap_avcodec_find_encoder_by_name(TinyString(audio_codec).c_str());
       if (!acodec)
       {
-        Error::SetStringFmt(error, "Audio codec {} not found.", video_codec);
+        Error::SetStringFmt(error, "Audio codec {} not found.", audio_codec);
         return false;
       }
     }
@@ -2483,7 +2683,7 @@ bool MediaCaptureFFmpeg::InternalBeginCapture(float fps, float aspect, u32 sampl
     const AVSampleFormat* supported_sample_formats = nullptr;
     int num_supported_sample_formats = 0;
 #ifdef HAS_AVCODEC_GET_SUPPORTED_CONFIG
-    res = wrap_avcodec_get_supported_config(m_video_codec_context, acodec, AV_CODEC_CONFIG_SAMPLE_FORMAT, 0,
+    res = wrap_avcodec_get_supported_config(m_audio_codec_context, acodec, AV_CODEC_CONFIG_SAMPLE_FORMAT, 0,
                                             reinterpret_cast<const void**>(&supported_sample_formats),
                                             &num_supported_sample_formats);
     if (res < 0)
@@ -2494,7 +2694,7 @@ bool MediaCaptureFFmpeg::InternalBeginCapture(float fps, float aspect, u32 sampl
 #else
     if (!acodec->sample_fmts)
     {
-      Error::SetStringView(error, "Video codec supports no formats.");
+      Error::SetStringView(error, "Audio codec supports no sample formats.");
       return false;
     }
 
@@ -2511,9 +2711,17 @@ bool MediaCaptureFFmpeg::InternalBeginCapture(float fps, float aspect, u32 sampl
         break;
       }
     }
+    if (!supported_sample_formats)
+      supports_format = true;
+
     if (!supports_format)
     {
       WARNING_LOG("Audio codec '{}' does not support S16 samples, using default.", acodec->name);
+      if (num_supported_sample_formats == 0)
+      {
+        Error::SetStringView(error, "Audio codec supports no sample formats.");
+        return false;
+      }
       m_audio_codec_context->sample_fmt = supported_sample_formats[0];
       m_swr_context = wrap_swr_alloc();
       if (!m_swr_context)
@@ -2778,7 +2986,12 @@ bool MediaCaptureFFmpeg::SendFrame(const PendingFrame& pf, Error* error)
   }
 
   // In case a previous frame is still using the frame.
-  wrap_av_frame_make_writable(m_converted_video_frame);
+  int res = wrap_av_frame_make_writable(m_converted_video_frame);
+  if (res < 0)
+  {
+    SetAVError(error, "av_frame_make_writable() failed: ", res);
+    return false;
+  }
 
   m_sws_context = wrap_sws_getCachedContext(m_sws_context, source_width, source_height, m_video_pixel_format,
                                             m_converted_video_frame->width, m_converted_video_frame->height,
@@ -2790,14 +3003,34 @@ bool MediaCaptureFFmpeg::SendFrame(const PendingFrame& pf, Error* error)
     return false;
   }
 
-  wrap_sws_scale(m_sws_context, reinterpret_cast<const u8**>(&source_ptr), &source_pitch, 0, source_height,
-                 m_converted_video_frame->data, m_converted_video_frame->linesize);
+  const int scaled_height =
+    wrap_sws_scale(m_sws_context, reinterpret_cast<const u8**>(&source_ptr), &source_pitch, 0, source_height,
+                   m_converted_video_frame->data, m_converted_video_frame->linesize);
+  if (scaled_height != m_converted_video_frame->height)
+  {
+    Error::SetStringFmt(error, "sws_scale() converted {} of {} rows", scaled_height, m_converted_video_frame->height);
+    return false;
+  }
 
   AVFrame* frame_to_send = m_converted_video_frame;
   if (IsUsingHardwareVideoEncoding())
   {
+    // Allocate a new hardware surface for every frame. The encoder may retain a reference to the
+    // submitted surface after avcodec_send_frame() returns, so reusing one surface corrupts frames
+    // and can also make hardware implementations fail while creating their staging texture.
+    wrap_av_frame_unref(m_hw_video_frame);
+    m_hw_video_frame->format = m_video_codec_context->pix_fmt;
+    m_hw_video_frame->width = m_video_codec_context->width;
+    m_hw_video_frame->height = m_video_codec_context->height;
+    res = wrap_av_hwframe_get_buffer(m_video_hw_frames, m_hw_video_frame, 0);
+    if (res < 0)
+    {
+      SetAVError(error, "av_hwframe_get_buffer() failed: ", res);
+      return false;
+    }
+
     // Need to transfer the frame to hardware.
-    const int res = wrap_av_hwframe_transfer_data(m_hw_video_frame, m_converted_video_frame, 0);
+    res = wrap_av_hwframe_transfer_data(m_hw_video_frame, m_converted_video_frame, 0);
     if (res < 0) [[unlikely]]
     {
       SetAVError(error, "av_hwframe_transfer_data() failed: ", res);
@@ -2810,7 +3043,7 @@ bool MediaCaptureFFmpeg::SendFrame(const PendingFrame& pf, Error* error)
   // Set the correct PTS before handing it off.
   frame_to_send->pts = pf.pts;
 
-  const int res = wrap_avcodec_send_frame(m_video_codec_context, frame_to_send);
+  res = wrap_avcodec_send_frame(m_video_codec_context, frame_to_send);
   if (res < 0) [[unlikely]]
   {
     SetAVError(error, "avcodec_send_frame() failed: ", res);