Fix playback and conversion of audio with unknown channel layout

DV/DVCPRO .mov files (e.g. captured by Final Cut / iMovie) carry a 'chan'
atom that labels every audio channel as "unused", so ffmpeg describes the
stream as "4 channels (UNSD+UNSD+UNSD+UNSD)".

swresample only accepts native or fully specified custom layouts, so as
soon as anything needs to resample or downmix such a stream it fails with:

  [SWR] Input channel layout '4 channels (UNSD+UNSD+UNSD+UNSD)' is not supported
  [af#0:1] Error reinitializing filters!
  Nothing was written into output file

This broke both preview playback and "convert to supported format" for
these files.

Prepend a `channelmap` filter that re-labels the channels, which turns the
layout into a plain "N channels" (unspecified) layout that swresample does
support. It only re-labels, the samples are untouched, and the stream then
behaves exactly as if it had carried no channel layout information at all.

Only applied to streams that actually have an unsupported layout, so files
with a proper layout keep their correct downmix.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YCf7BBiPqNLUjVoN7rTCF6

src/common/ffprobe.ts

@@ -50,7 +50,7 @@
  * @property {string} [sample_fmt] The audio sample format (not present if codec_type is not "audio")
  * @property {string} [sample_rate] A string representation of an integer showing the audio sample rate (not present if codec_type is not "audio")
  * @property {number} [channels] The audio track's channel count (not present if codec_type is not "audio")
- * @property {'stereo'|'mono'} [channel_layout] The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio")
+ * @property {string} [channel_layout] The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio")
  * @property {number} [bits_per_sample] Bits per audio sample (might not be accurate, may just be 0) (not present if codec_type is not "audio")
  * @property {number} [width] The video stream width (also available for images) (not present if codec_type is not "video")
  * @property {number} [height] The stream height (also available for images) (not present if codec_type is not "video")
@@ -346,7 +346,7 @@ export interface FFprobeStream {
   /**
    * The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio")
    */
-  channel_layout?: 'stereo' | 'mono',
+  channel_layout?: string,
 
   /**
    * Bits per audio sample (might not be accurate, may just be 0) (not present if codec_type is not "audio")

src/common/util.test.ts

@@ -0,0 +1,40 @@
+// eslint-disable-next-line import/no-extraneous-dependencies
+import { describe, expect, test } from 'vitest';
+
+import { getFixChannelLayoutFilter, hasUnsupportedChannelLayout } from './util.js';
+
+describe('hasUnsupportedChannelLayout', () => {
+  test('detects unused/unknown channels', () => {
+    // e.g. DV/DVCPRO .mov captured by Final Cut / iMovie
+    expect(hasUnsupportedChannelLayout('4 channels (UNSD+UNSD+UNSD+UNSD)')).toBe(true);
+    expect(hasUnsupportedChannelLayout('3 channels (FL+FR+UNK)')).toBe(true);
+  });
+
+  test('accepts layouts that swresample supports', () => {
+    expect(hasUnsupportedChannelLayout('stereo')).toBe(false);
+    expect(hasUnsupportedChannelLayout('mono')).toBe(false);
+    expect(hasUnsupportedChannelLayout('5.1(side)')).toBe(false);
+    expect(hasUnsupportedChannelLayout('2.1')).toBe(false);
+    // unspecified layouts are fine, ffmpeg just uses a default downmix
+    expect(hasUnsupportedChannelLayout('4 channels')).toBe(false);
+    expect(hasUnsupportedChannelLayout(undefined)).toBe(false);
+  });
+});
+
+describe('getFixChannelLayoutFilter', () => {
+  test('relabels all channels of an unsupported layout', () => {
+    expect(getFixChannelLayoutFilter({ channels: 4, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBe('channelmap=0|1|2|3');
+    expect(getFixChannelLayoutFilter({ channels: 3, channelLayout: '3 channels (FL+FR+UNK)' })).toBe('channelmap=0|1|2');
+    expect(getFixChannelLayoutFilter({ channels: 1, channelLayout: '1 channels (UNSD)' })).toBe('channelmap=0');
+  });
+
+  test('leaves supported layouts alone', () => {
+    expect(getFixChannelLayoutFilter({ channels: 2, channelLayout: 'stereo' })).toBeUndefined();
+    expect(getFixChannelLayoutFilter({ channels: 6, channelLayout: '5.1' })).toBeUndefined();
+  });
+
+  test('needs a channel count to build the map', () => {
+    expect(getFixChannelLayoutFilter({ channels: undefined, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBeUndefined();
+    expect(getFixChannelLayoutFilter({ channels: 0, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBeUndefined();
+  });
+});

src/common/util.ts

@@ -19,3 +19,24 @@ export function parseRatio(str: string, char = '/') {
   if (den <= 0) return undefined;
   return num / den;
 }
+
+// ffmpeg's swresample cannot handle channel layouts that contain unknown ("UNK") or unused ("UNSD")
+// channels, which ffprobe describes like "4 channels (UNSD+UNSD+UNSD+UNSD)". Such layouts occur in
+// e.g. DV/DVCPRO .mov files, whose 'chan' atom labels every channel as "unused". Any operation that
+// needs to resample or downmix such a stream then fails with:
+//   [SWR] Input channel layout '4 channels (UNSD+UNSD+UNSD+UNSD)' is not supported
+// (swresample only accepts native or fully-specified custom layouts, see swr_init in libswresample)
+export const hasUnsupportedChannelLayout = (channelLayout: string | undefined) => (
+  channelLayout != null && /\b(?:UNK|UNSD)\b/.test(channelLayout)
+);
+
+// The `channelmap` filter re-labels the channels without touching the samples, which turns the
+// layout into a plain "N channels" (unspecified) layout that swresample does support. The stream
+// then behaves exactly as if it had carried no channel layout information at all.
+export function getFixChannelLayoutFilter({ channels, channelLayout }: {
+  channels?: number | undefined,
+  channelLayout?: string | undefined,
+}) {
+  if (channels == null || channels <= 0 || !hasUnsupportedChannelLayout(channelLayout)) return undefined;
+  return `channelmap=${Array.from({ length: channels }, (_, i) => i).join('|')}`;
+}

src/main/compatPlayer.ts

@@ -11,14 +11,14 @@ export function createMediaSourceStream(params: Parameters<typeof createMediaSou
   const abort = () => abortController.abort();
 
   async function attemptCreateProcess({ forceColorspace }: { forceColorspace?: boolean } = {}) {
-    const { videoStreamIndex, audioStreamIndexes, seekTo } = params;
+    const { videoStreamIndex, audioStreams, seekTo } = params;
 
-    logger.info('Starting preview process', { videoStreamIndex, audioStreamIndexes, seekTo });
+    logger.info('Starting preview process', { videoStreamIndex, audioStreams, seekTo });
     const process = createMediaSourceProcess({ ...params, forceColorspace });
 
     // eslint-disable-next-line unicorn/prefer-add-event-listener
     abortController.signal.onabort = () => {
-      logger.info('Aborting preview process', { videoStreamIndex, audioStreamIndexes, seekTo });
+      logger.info('Aborting preview process', { videoStreamIndex, audioStreams, seekTo });
       process.kill('SIGKILL');
     };
 

src/main/ffmpeg.ts

@@ -15,7 +15,7 @@ import type { FFprobeFormat } from '../common/ffprobe.js';
 import isDev from './isDev.js';
 import logger from './logger.js';
 import { parseFfmpegProgressLine } from './progress.js';
-import { formatFfmpegNumber, getHwaccelArgs, parseFfprobeDuration } from '../common/util.js';
+import { formatFfmpegNumber, getFixChannelLayoutFilter, getHwaccelArgs, parseFfprobeDuration } from '../common/util.js';
 import { getFfmpegJpegQuality } from './ffmpegUtil.js';
 import { throwIfDisabledNetworking } from './networking.js';
 
@@ -591,10 +591,10 @@ export async function getDuration(filePath: string) {
 const enableLog = false;
 const encode = true;
 
-export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIndexes, seekTo, size, fps, rotate, forceColorspace, ffmpegHwaccel }: {
+export function createMediaSourceProcess({ path, videoStreamIndex, audioStreams, seekTo, size, fps, rotate, forceColorspace, ffmpegHwaccel }: {
   path: string,
   videoStreamIndex?: number | undefined,
-  audioStreamIndexes: number[],
+  audioStreams: { index: number, channels?: number | undefined, channelLayout?: string | undefined }[],
   seekTo: number,
   size?: number | undefined,
   fps?: number | undefined,
@@ -664,18 +664,24 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn
       graph.push(`[0:${videoStreamIndex}]${videoFiltersStr}[video]`);
     }
 
-    if (audioStreamIndexes.length > 0) {
-      if (audioStreamIndexes.length > 1) {
-        const resampledStr = audioStreamIndexes.map((i) => `[resampled${i}]`).join('');
-        const weightsStr = audioStreamIndexes.map(() => '1').join(' ');
+    if (audioStreams.length > 0) {
+      // some streams have a channel layout that ffmpeg cannot resample or downmix, so relabel it first
+      const getAudioFilters = (stream: typeof audioStreams[number], rest: string[]) => {
+        const filters = [getFixChannelLayoutFilter(stream), ...rest].filter((filter) => filter != null);
+        return filters.length > 0 ? filters.join(',') : 'anull';
+      };
+
+      if (audioStreams.length > 1) {
+        const resampledStr = audioStreams.map(({ index }) => `[resampled${index}]`).join('');
+        const weightsStr = audioStreams.map(() => '1').join(' ');
         graph.push(
           // First resample because else we get the lowest sample rate
-          ...audioStreamIndexes.map((i) => `[0:${i}]aresample=44100[resampled${i}]`),
+          ...audioStreams.map((stream) => `[0:${stream.index}]${getAudioFilters(stream, ['aresample=44100'])}[resampled${stream.index}]`),
           // now mix all audio channels together
-          `${resampledStr}amix=inputs=${audioStreamIndexes.length}:duration=longest:weights=${weightsStr}:normalize=0:dropout_transition=2[audio]`,
+          `${resampledStr}amix=inputs=${audioStreams.length}:duration=longest:weights=${weightsStr}:normalize=0:dropout_transition=2[audio]`,
         );
       } else {
-        graph.push(`[0:${audioStreamIndexes[0]}]anull[audio]`);
+        graph.push(`[0:${audioStreams[0]!.index}]${getAudioFilters(audioStreams[0]!, [])}[audio]`);
       }
     }
 
@@ -727,7 +733,7 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn
         '-g', '1', // reduces latency and buffering
       ] : ['-vn']),
 
-      ...(audioStreamIndexes.length > 0 ? [
+      ...(audioStreams.length > 0 ? [
         '-map', '[audio]',
         '-ac', '2', '-c:a', 'aac', '-b:a', '128k',
       ] : ['-an']),

src/renderer/src/MediaSourcePlayer.tsx

@@ -13,12 +13,12 @@ import type { FfmpegHwAccel } from '../../common/types';
 const { compatPlayer: { createMediaSourceStream } } = window.require('@electron/remote').require('./index.js');
 
 
-async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, audioStreamIndexes, seekTo, signal, size, fps, rotate, onCanPlay, onResetNeeded, onWaiting, ffmpegHwaccel }: {
+async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, audioStreams, seekTo, signal, size, fps, rotate, onCanPlay, onResetNeeded, onWaiting, ffmpegHwaccel }: {
   path: string,
   slaveVideo: ChromiumHTMLVideoElement,
   masterVideo: ChromiumHTMLVideoElement,
   videoStreamIndex?: number | undefined,
-  audioStreamIndexes: number[],
+  audioStreams: { index: number, channels?: number | undefined, channelLayout?: string | undefined }[],
   seekTo: number,
   signal: AbortSignal,
   size?: number | undefined,
@@ -73,7 +73,7 @@ async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex,
 
   const codecs: string[] = [];
   if (videoStreamIndex != null) codecs.push('avc1.42C01F');
-  if (audioStreamIndexes.length > 0) codecs.push('mp4a.40.2');
+  if (audioStreams.length > 0) codecs.push('mp4a.40.2');
   const codecTag = codecs.join(', ');
 
   const mimeCodec = `video/mp4; codecs="${codecTag}"`;
@@ -90,7 +90,7 @@ async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex,
     throw new Error(`Unsupported MIME type or codec: ${mimeCodec}`);
   }
 
-  mediaSourceProcess = createMediaSourceStream({ path, videoStreamIndex, audioStreamIndexes, seekTo, size, fps, rotate, ffmpegHwaccel });
+  mediaSourceProcess = createMediaSourceStream({ path, videoStreamIndex, audioStreams, seekTo, size, fps, rotate, ffmpegHwaccel });
   console.log('Waiting for media source process to emit first data...');
   const readChunk = await mediaSourceProcess.promise;
   if (readChunk == null) {
@@ -325,7 +325,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master
     console.error('video error', error);
   }, []);
 
-  const audioStreamIndexes = useMemo(() => audioStreams.map((s) => s.index), [audioStreams]);
+  const audioStreamsForPreview = useMemo(() => audioStreams.map(({ index, channels, channel_layout: channelLayout }) => ({ index, channels, channelLayout })), [audioStreams]);
 
   useEffect(() => {
     const video = videoRef.current;
@@ -369,7 +369,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master
           slaveVideo: video,
           masterVideo,
           videoStreamIndex: videoStream?.index,
-          audioStreamIndexes,
+          audioStreams: audioStreamsForPreview,
           seekTo,
           size,
           fps,
@@ -398,7 +398,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master
 
     return () => abortController.abort();
     // Important that we also have eventId in the deps, so that we can restart the preview when the eventId changes
-  }, [audioStreamIndexes, ffmpegHwaccel, filePath, masterVideoRef, mediaSourceQuality, rotate, videoStream]);
+  }, [audioStreamsForPreview, ffmpegHwaccel, filePath, masterVideoRef, mediaSourceQuality, rotate, videoStream]);
 
   const onFocus = useCallback<FocusEventHandler<HTMLVideoElement>>((e) => {
     // prevent video element from stealing focus in fullscreen mode https://github.com/mifi/lossless-cut/issues/543#issuecomment-1868167775

src/renderer/src/hooks/useFfmpegOperations.ts

@@ -15,7 +15,7 @@ import { deleteDispositionValue, type AllFilesMeta, type Chapter, type CopyfileS
 import type { LossyMode } from '../../../main';
 import { UserFacingError } from '../../errors';
 import mainApi from '../mainApi';
-import { formatFfmpegNumber, getHwaccelArgs } from '../../../common/util';
+import { formatFfmpegNumber, getFixChannelLayoutFilter, getHwaccelArgs, hasUnsupportedChannelLayout } from '../../../common/util';
 
 const { join, resolve, dirname } = window.require('node:path');
 const { writeFile, mkdir, access, constants: { W_OK } } = window.require('node:fs/promises');
@@ -890,6 +890,26 @@ function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, trea
       }
     }
 
+    // Some files (e.g. DV/DVCPRO .mov) have an audio channel layout that ffmpeg cannot resample or
+    // downmix, which makes the encode fail. Relabel the channels first so that it can.
+    // We don't pass -map, so ffmpeg picks one audio stream by itself. Therefore only apply the filter
+    // when every audio stream has the same channel count, or the filter might not match the picked stream.
+    let audioFilterArgs: string[] = [];
+    if (audio != null && audio !== 'copy') {
+      try {
+        const audioStreams = (await readFileFfprobeMeta(filePathArg)).streams.filter((s) => s.codec_type === 'audio');
+        const unsupportedStream = audioStreams.find((s) => hasUnsupportedChannelLayout(s.channel_layout));
+        const sameChannelCount = new Set(audioStreams.map((s) => s.channels)).size === 1;
+        if (unsupportedStream != null && sameChannelCount) {
+          const filter = getFixChannelLayoutFilter({ channels: unsupportedStream.channels, channelLayout: unsupportedStream.channel_layout });
+          if (filter != null) audioFilterArgs = ['-af', filter];
+        }
+      } catch (err) {
+        // don't fail the conversion just because we couldn't probe it
+        console.warn('Failed to probe audio channel layout', err);
+      }
+    }
+
     const ffmpegArgs = [
       '-hide_banner',
       ...((video === 'lq' || video === 'hq') ? getHwaccelArgs(ffmpegHwaccel) : []),
@@ -897,6 +917,7 @@ function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, trea
       '-i', filePathArg,
       ...videoArgs,
       ...audioArgs,
+      ...audioFilterArgs,
       '-sn',
       '-y', outPath,
     ];

tsconfig.common.json

@@ -16,4 +16,9 @@
   "include": [
     "src/common/**/*",
   ],
+  // tests are run by vitest from the sources; don't emit them into the build output,
+  // or vitest would discover and run the compiled copies too
+  "exclude": [
+    "src/common/**/*.test.ts",
+  ],
 }