From ca6813f9a41c00c74e023bfe9c341d2800cb2f09 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 1 Sep 2026 10:10:43 +0000 Subject: [PATCH] Fix playback and conversion of audio with unknown channel layout DV/DVCPRO .mov files (e.g. captured by Final Cut / iMovie) carry a 'chan' atom that labels every audio channel as "unused", so ffmpeg describes the stream as "4 channels (UNSD+UNSD+UNSD+UNSD)". swresample only accepts native or fully specified custom layouts, so as soon as anything needs to resample or downmix such a stream it fails with: [SWR] Input channel layout '4 channels (UNSD+UNSD+UNSD+UNSD)' is not supported [af#0:1] Error reinitializing filters! Nothing was written into output file This broke both preview playback and "convert to supported format" for these files. Prepend a `channelmap` filter that re-labels the channels, which turns the layout into a plain "N channels" (unspecified) layout that swresample does support. It only re-labels, the samples are untouched, and the stream then behaves exactly as if it had carried no channel layout information at all. Only applied to streams that actually have an unsupported layout, so files with a proper layout keep their correct downmix. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YCf7BBiPqNLUjVoN7rTCF6 --- src/common/ffprobe.ts | 4 +- src/common/util.test.ts | 40 +++++++++++++++++++ src/common/util.ts | 21 ++++++++++ src/main/compatPlayer.ts | 6 +-- src/main/ffmpeg.ts | 28 ++++++++----- src/renderer/src/MediaSourcePlayer.tsx | 14 +++---- src/renderer/src/hooks/useFfmpegOperations.ts | 23 ++++++++++- tsconfig.common.json | 5 +++ 8 files changed, 117 insertions(+), 24 deletions(-) create mode 100644 src/common/util.test.ts diff --git a/src/common/ffprobe.ts b/src/common/ffprobe.ts index 13cd2a8d..44ee68fa 100644 --- a/src/common/ffprobe.ts +++ b/src/common/ffprobe.ts @@ -50,7 +50,7 @@ * @property {string} [sample_fmt] The audio sample format (not present if codec_type is not "audio") * @property {string} [sample_rate] A string representation of an integer showing the audio sample rate (not present if codec_type is not "audio") * @property {number} [channels] The audio track's channel count (not present if codec_type is not "audio") - * @property {'stereo'|'mono'} [channel_layout] The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio") + * @property {string} [channel_layout] The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio") * @property {number} [bits_per_sample] Bits per audio sample (might not be accurate, may just be 0) (not present if codec_type is not "audio") * @property {number} [width] The video stream width (also available for images) (not present if codec_type is not "video") * @property {number} [height] The stream height (also available for images) (not present if codec_type is not "video") @@ -346,7 +346,7 @@ export interface FFprobeStream { /** * The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio") */ - channel_layout?: 'stereo' | 'mono', + channel_layout?: string, /** * Bits per audio sample (might not be accurate, may just be 0) (not present if codec_type is not "audio") diff --git a/src/common/util.test.ts b/src/common/util.test.ts new file mode 100644 index 00000000..9c38f3a3 --- /dev/null +++ b/src/common/util.test.ts @@ -0,0 +1,40 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +import { describe, expect, test } from 'vitest'; + +import { getFixChannelLayoutFilter, hasUnsupportedChannelLayout } from './util.js'; + +describe('hasUnsupportedChannelLayout', () => { + test('detects unused/unknown channels', () => { + // e.g. DV/DVCPRO .mov captured by Final Cut / iMovie + expect(hasUnsupportedChannelLayout('4 channels (UNSD+UNSD+UNSD+UNSD)')).toBe(true); + expect(hasUnsupportedChannelLayout('3 channels (FL+FR+UNK)')).toBe(true); + }); + + test('accepts layouts that swresample supports', () => { + expect(hasUnsupportedChannelLayout('stereo')).toBe(false); + expect(hasUnsupportedChannelLayout('mono')).toBe(false); + expect(hasUnsupportedChannelLayout('5.1(side)')).toBe(false); + expect(hasUnsupportedChannelLayout('2.1')).toBe(false); + // unspecified layouts are fine, ffmpeg just uses a default downmix + expect(hasUnsupportedChannelLayout('4 channels')).toBe(false); + expect(hasUnsupportedChannelLayout(undefined)).toBe(false); + }); +}); + +describe('getFixChannelLayoutFilter', () => { + test('relabels all channels of an unsupported layout', () => { + expect(getFixChannelLayoutFilter({ channels: 4, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBe('channelmap=0|1|2|3'); + expect(getFixChannelLayoutFilter({ channels: 3, channelLayout: '3 channels (FL+FR+UNK)' })).toBe('channelmap=0|1|2'); + expect(getFixChannelLayoutFilter({ channels: 1, channelLayout: '1 channels (UNSD)' })).toBe('channelmap=0'); + }); + + test('leaves supported layouts alone', () => { + expect(getFixChannelLayoutFilter({ channels: 2, channelLayout: 'stereo' })).toBeUndefined(); + expect(getFixChannelLayoutFilter({ channels: 6, channelLayout: '5.1' })).toBeUndefined(); + }); + + test('needs a channel count to build the map', () => { + expect(getFixChannelLayoutFilter({ channels: undefined, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBeUndefined(); + expect(getFixChannelLayoutFilter({ channels: 0, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBeUndefined(); + }); +}); diff --git a/src/common/util.ts b/src/common/util.ts index 1e0a8865..d0bba7d6 100644 --- a/src/common/util.ts +++ b/src/common/util.ts @@ -19,3 +19,24 @@ export function parseRatio(str: string, char = '/') { if (den <= 0) return undefined; return num / den; } + +// ffmpeg's swresample cannot handle channel layouts that contain unknown ("UNK") or unused ("UNSD") +// channels, which ffprobe describes like "4 channels (UNSD+UNSD+UNSD+UNSD)". Such layouts occur in +// e.g. DV/DVCPRO .mov files, whose 'chan' atom labels every channel as "unused". Any operation that +// needs to resample or downmix such a stream then fails with: +// [SWR] Input channel layout '4 channels (UNSD+UNSD+UNSD+UNSD)' is not supported +// (swresample only accepts native or fully-specified custom layouts, see swr_init in libswresample) +export const hasUnsupportedChannelLayout = (channelLayout: string | undefined) => ( + channelLayout != null && /\b(?:UNK|UNSD)\b/.test(channelLayout) +); + +// The `channelmap` filter re-labels the channels without touching the samples, which turns the +// layout into a plain "N channels" (unspecified) layout that swresample does support. The stream +// then behaves exactly as if it had carried no channel layout information at all. +export function getFixChannelLayoutFilter({ channels, channelLayout }: { + channels?: number | undefined, + channelLayout?: string | undefined, +}) { + if (channels == null || channels <= 0 || !hasUnsupportedChannelLayout(channelLayout)) return undefined; + return `channelmap=${Array.from({ length: channels }, (_, i) => i).join('|')}`; +} diff --git a/src/main/compatPlayer.ts b/src/main/compatPlayer.ts index 556faa4c..766d062e 100644 --- a/src/main/compatPlayer.ts +++ b/src/main/compatPlayer.ts @@ -11,14 +11,14 @@ export function createMediaSourceStream(params: Parameters abortController.abort(); async function attemptCreateProcess({ forceColorspace }: { forceColorspace?: boolean } = {}) { - const { videoStreamIndex, audioStreamIndexes, seekTo } = params; + const { videoStreamIndex, audioStreams, seekTo } = params; - logger.info('Starting preview process', { videoStreamIndex, audioStreamIndexes, seekTo }); + logger.info('Starting preview process', { videoStreamIndex, audioStreams, seekTo }); const process = createMediaSourceProcess({ ...params, forceColorspace }); // eslint-disable-next-line unicorn/prefer-add-event-listener abortController.signal.onabort = () => { - logger.info('Aborting preview process', { videoStreamIndex, audioStreamIndexes, seekTo }); + logger.info('Aborting preview process', { videoStreamIndex, audioStreams, seekTo }); process.kill('SIGKILL'); }; diff --git a/src/main/ffmpeg.ts b/src/main/ffmpeg.ts index b86be90f..22ee0d04 100644 --- a/src/main/ffmpeg.ts +++ b/src/main/ffmpeg.ts @@ -15,7 +15,7 @@ import type { FFprobeFormat } from '../common/ffprobe.js'; import isDev from './isDev.js'; import logger from './logger.js'; import { parseFfmpegProgressLine } from './progress.js'; -import { formatFfmpegNumber, getHwaccelArgs, parseFfprobeDuration } from '../common/util.js'; +import { formatFfmpegNumber, getFixChannelLayoutFilter, getHwaccelArgs, parseFfprobeDuration } from '../common/util.js'; import { getFfmpegJpegQuality } from './ffmpegUtil.js'; import { throwIfDisabledNetworking } from './networking.js'; @@ -591,10 +591,10 @@ export async function getDuration(filePath: string) { const enableLog = false; const encode = true; -export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIndexes, seekTo, size, fps, rotate, forceColorspace, ffmpegHwaccel }: { +export function createMediaSourceProcess({ path, videoStreamIndex, audioStreams, seekTo, size, fps, rotate, forceColorspace, ffmpegHwaccel }: { path: string, videoStreamIndex?: number | undefined, - audioStreamIndexes: number[], + audioStreams: { index: number, channels?: number | undefined, channelLayout?: string | undefined }[], seekTo: number, size?: number | undefined, fps?: number | undefined, @@ -664,18 +664,24 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn graph.push(`[0:${videoStreamIndex}]${videoFiltersStr}[video]`); } - if (audioStreamIndexes.length > 0) { - if (audioStreamIndexes.length > 1) { - const resampledStr = audioStreamIndexes.map((i) => `[resampled${i}]`).join(''); - const weightsStr = audioStreamIndexes.map(() => '1').join(' '); + if (audioStreams.length > 0) { + // some streams have a channel layout that ffmpeg cannot resample or downmix, so relabel it first + const getAudioFilters = (stream: typeof audioStreams[number], rest: string[]) => { + const filters = [getFixChannelLayoutFilter(stream), ...rest].filter((filter) => filter != null); + return filters.length > 0 ? filters.join(',') : 'anull'; + }; + + if (audioStreams.length > 1) { + const resampledStr = audioStreams.map(({ index }) => `[resampled${index}]`).join(''); + const weightsStr = audioStreams.map(() => '1').join(' '); graph.push( // First resample because else we get the lowest sample rate - ...audioStreamIndexes.map((i) => `[0:${i}]aresample=44100[resampled${i}]`), + ...audioStreams.map((stream) => `[0:${stream.index}]${getAudioFilters(stream, ['aresample=44100'])}[resampled${stream.index}]`), // now mix all audio channels together - `${resampledStr}amix=inputs=${audioStreamIndexes.length}:duration=longest:weights=${weightsStr}:normalize=0:dropout_transition=2[audio]`, + `${resampledStr}amix=inputs=${audioStreams.length}:duration=longest:weights=${weightsStr}:normalize=0:dropout_transition=2[audio]`, ); } else { - graph.push(`[0:${audioStreamIndexes[0]}]anull[audio]`); + graph.push(`[0:${audioStreams[0]!.index}]${getAudioFilters(audioStreams[0]!, [])}[audio]`); } } @@ -727,7 +733,7 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn '-g', '1', // reduces latency and buffering ] : ['-vn']), - ...(audioStreamIndexes.length > 0 ? [ + ...(audioStreams.length > 0 ? [ '-map', '[audio]', '-ac', '2', '-c:a', 'aac', '-b:a', '128k', ] : ['-an']), diff --git a/src/renderer/src/MediaSourcePlayer.tsx b/src/renderer/src/MediaSourcePlayer.tsx index af57036a..979c0cc9 100644 --- a/src/renderer/src/MediaSourcePlayer.tsx +++ b/src/renderer/src/MediaSourcePlayer.tsx @@ -13,12 +13,12 @@ import type { FfmpegHwAccel } from '../../common/types'; const { compatPlayer: { createMediaSourceStream } } = window.require('@electron/remote').require('./index.js'); -async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, audioStreamIndexes, seekTo, signal, size, fps, rotate, onCanPlay, onResetNeeded, onWaiting, ffmpegHwaccel }: { +async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, audioStreams, seekTo, signal, size, fps, rotate, onCanPlay, onResetNeeded, onWaiting, ffmpegHwaccel }: { path: string, slaveVideo: ChromiumHTMLVideoElement, masterVideo: ChromiumHTMLVideoElement, videoStreamIndex?: number | undefined, - audioStreamIndexes: number[], + audioStreams: { index: number, channels?: number | undefined, channelLayout?: string | undefined }[], seekTo: number, signal: AbortSignal, size?: number | undefined, @@ -73,7 +73,7 @@ async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, const codecs: string[] = []; if (videoStreamIndex != null) codecs.push('avc1.42C01F'); - if (audioStreamIndexes.length > 0) codecs.push('mp4a.40.2'); + if (audioStreams.length > 0) codecs.push('mp4a.40.2'); const codecTag = codecs.join(', '); const mimeCodec = `video/mp4; codecs="${codecTag}"`; @@ -90,7 +90,7 @@ async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, throw new Error(`Unsupported MIME type or codec: ${mimeCodec}`); } - mediaSourceProcess = createMediaSourceStream({ path, videoStreamIndex, audioStreamIndexes, seekTo, size, fps, rotate, ffmpegHwaccel }); + mediaSourceProcess = createMediaSourceStream({ path, videoStreamIndex, audioStreams, seekTo, size, fps, rotate, ffmpegHwaccel }); console.log('Waiting for media source process to emit first data...'); const readChunk = await mediaSourceProcess.promise; if (readChunk == null) { @@ -325,7 +325,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master console.error('video error', error); }, []); - const audioStreamIndexes = useMemo(() => audioStreams.map((s) => s.index), [audioStreams]); + const audioStreamsForPreview = useMemo(() => audioStreams.map(({ index, channels, channel_layout: channelLayout }) => ({ index, channels, channelLayout })), [audioStreams]); useEffect(() => { const video = videoRef.current; @@ -369,7 +369,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master slaveVideo: video, masterVideo, videoStreamIndex: videoStream?.index, - audioStreamIndexes, + audioStreams: audioStreamsForPreview, seekTo, size, fps, @@ -398,7 +398,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master return () => abortController.abort(); // Important that we also have eventId in the deps, so that we can restart the preview when the eventId changes - }, [audioStreamIndexes, ffmpegHwaccel, filePath, masterVideoRef, mediaSourceQuality, rotate, videoStream]); + }, [audioStreamsForPreview, ffmpegHwaccel, filePath, masterVideoRef, mediaSourceQuality, rotate, videoStream]); const onFocus = useCallback>((e) => { // prevent video element from stealing focus in fullscreen mode https://github.com/mifi/lossless-cut/issues/543#issuecomment-1868167775 diff --git a/src/renderer/src/hooks/useFfmpegOperations.ts b/src/renderer/src/hooks/useFfmpegOperations.ts index ea075026..50963837 100644 --- a/src/renderer/src/hooks/useFfmpegOperations.ts +++ b/src/renderer/src/hooks/useFfmpegOperations.ts @@ -15,7 +15,7 @@ import { deleteDispositionValue, type AllFilesMeta, type Chapter, type CopyfileS import type { LossyMode } from '../../../main'; import { UserFacingError } from '../../errors'; import mainApi from '../mainApi'; -import { formatFfmpegNumber, getHwaccelArgs } from '../../../common/util'; +import { formatFfmpegNumber, getFixChannelLayoutFilter, getHwaccelArgs, hasUnsupportedChannelLayout } from '../../../common/util'; const { join, resolve, dirname } = window.require('node:path'); const { writeFile, mkdir, access, constants: { W_OK } } = window.require('node:fs/promises'); @@ -890,6 +890,26 @@ function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, trea } } + // Some files (e.g. DV/DVCPRO .mov) have an audio channel layout that ffmpeg cannot resample or + // downmix, which makes the encode fail. Relabel the channels first so that it can. + // We don't pass -map, so ffmpeg picks one audio stream by itself. Therefore only apply the filter + // when every audio stream has the same channel count, or the filter might not match the picked stream. + let audioFilterArgs: string[] = []; + if (audio != null && audio !== 'copy') { + try { + const audioStreams = (await readFileFfprobeMeta(filePathArg)).streams.filter((s) => s.codec_type === 'audio'); + const unsupportedStream = audioStreams.find((s) => hasUnsupportedChannelLayout(s.channel_layout)); + const sameChannelCount = new Set(audioStreams.map((s) => s.channels)).size === 1; + if (unsupportedStream != null && sameChannelCount) { + const filter = getFixChannelLayoutFilter({ channels: unsupportedStream.channels, channelLayout: unsupportedStream.channel_layout }); + if (filter != null) audioFilterArgs = ['-af', filter]; + } + } catch (err) { + // don't fail the conversion just because we couldn't probe it + console.warn('Failed to probe audio channel layout', err); + } + } + const ffmpegArgs = [ '-hide_banner', ...((video === 'lq' || video === 'hq') ? getHwaccelArgs(ffmpegHwaccel) : []), @@ -897,6 +917,7 @@ function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, trea '-i', filePathArg, ...videoArgs, ...audioArgs, + ...audioFilterArgs, '-sn', '-y', outPath, ]; diff --git a/tsconfig.common.json b/tsconfig.common.json index eda60050..886ae832 100644 --- a/tsconfig.common.json +++ b/tsconfig.common.json @@ -16,4 +16,9 @@ "include": [ "src/common/**/*", ], + // tests are run by vitest from the sources; don't emit them into the build output, + // or vitest would discover and run the compiled copies too + "exclude": [ + "src/common/**/*.test.ts", + ], } \ No newline at end of file