diff --git a/src/common/ffprobe.ts b/src/common/ffprobe.ts index 13cd2a8d..44ee68fa 100644 --- a/src/common/ffprobe.ts +++ b/src/common/ffprobe.ts @@ -50,7 +50,7 @@ * @property {string} [sample_fmt] The audio sample format (not present if codec_type is not "audio") * @property {string} [sample_rate] A string representation of an integer showing the audio sample rate (not present if codec_type is not "audio") * @property {number} [channels] The audio track's channel count (not present if codec_type is not "audio") - * @property {'stereo'|'mono'} [channel_layout] The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio") + * @property {string} [channel_layout] The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio") * @property {number} [bits_per_sample] Bits per audio sample (might not be accurate, may just be 0) (not present if codec_type is not "audio") * @property {number} [width] The video stream width (also available for images) (not present if codec_type is not "video") * @property {number} [height] The stream height (also available for images) (not present if codec_type is not "video") @@ -346,7 +346,7 @@ export interface FFprobeStream { /** * The audio track's channel layout (e.g. "stereo") (not present if codec_type is not "audio") */ - channel_layout?: 'stereo' | 'mono', + channel_layout?: string, /** * Bits per audio sample (might not be accurate, may just be 0) (not present if codec_type is not "audio") diff --git a/src/common/util.test.ts b/src/common/util.test.ts new file mode 100644 index 00000000..9c38f3a3 --- /dev/null +++ b/src/common/util.test.ts @@ -0,0 +1,40 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +import { describe, expect, test } from 'vitest'; + +import { getFixChannelLayoutFilter, hasUnsupportedChannelLayout } from './util.js'; + +describe('hasUnsupportedChannelLayout', () => { + test('detects unused/unknown channels', () => { + // e.g. DV/DVCPRO .mov captured by Final Cut / iMovie + expect(hasUnsupportedChannelLayout('4 channels (UNSD+UNSD+UNSD+UNSD)')).toBe(true); + expect(hasUnsupportedChannelLayout('3 channels (FL+FR+UNK)')).toBe(true); + }); + + test('accepts layouts that swresample supports', () => { + expect(hasUnsupportedChannelLayout('stereo')).toBe(false); + expect(hasUnsupportedChannelLayout('mono')).toBe(false); + expect(hasUnsupportedChannelLayout('5.1(side)')).toBe(false); + expect(hasUnsupportedChannelLayout('2.1')).toBe(false); + // unspecified layouts are fine, ffmpeg just uses a default downmix + expect(hasUnsupportedChannelLayout('4 channels')).toBe(false); + expect(hasUnsupportedChannelLayout(undefined)).toBe(false); + }); +}); + +describe('getFixChannelLayoutFilter', () => { + test('relabels all channels of an unsupported layout', () => { + expect(getFixChannelLayoutFilter({ channels: 4, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBe('channelmap=0|1|2|3'); + expect(getFixChannelLayoutFilter({ channels: 3, channelLayout: '3 channels (FL+FR+UNK)' })).toBe('channelmap=0|1|2'); + expect(getFixChannelLayoutFilter({ channels: 1, channelLayout: '1 channels (UNSD)' })).toBe('channelmap=0'); + }); + + test('leaves supported layouts alone', () => { + expect(getFixChannelLayoutFilter({ channels: 2, channelLayout: 'stereo' })).toBeUndefined(); + expect(getFixChannelLayoutFilter({ channels: 6, channelLayout: '5.1' })).toBeUndefined(); + }); + + test('needs a channel count to build the map', () => { + expect(getFixChannelLayoutFilter({ channels: undefined, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBeUndefined(); + expect(getFixChannelLayoutFilter({ channels: 0, channelLayout: '4 channels (UNSD+UNSD+UNSD+UNSD)' })).toBeUndefined(); + }); +}); diff --git a/src/common/util.ts b/src/common/util.ts index 1e0a8865..d0bba7d6 100644 --- a/src/common/util.ts +++ b/src/common/util.ts @@ -19,3 +19,24 @@ export function parseRatio(str: string, char = '/') { if (den <= 0) return undefined; return num / den; } + +// ffmpeg's swresample cannot handle channel layouts that contain unknown ("UNK") or unused ("UNSD") +// channels, which ffprobe describes like "4 channels (UNSD+UNSD+UNSD+UNSD)". Such layouts occur in +// e.g. DV/DVCPRO .mov files, whose 'chan' atom labels every channel as "unused". Any operation that +// needs to resample or downmix such a stream then fails with: +// [SWR] Input channel layout '4 channels (UNSD+UNSD+UNSD+UNSD)' is not supported +// (swresample only accepts native or fully-specified custom layouts, see swr_init in libswresample) +export const hasUnsupportedChannelLayout = (channelLayout: string | undefined) => ( + channelLayout != null && /\b(?:UNK|UNSD)\b/.test(channelLayout) +); + +// The `channelmap` filter re-labels the channels without touching the samples, which turns the +// layout into a plain "N channels" (unspecified) layout that swresample does support. The stream +// then behaves exactly as if it had carried no channel layout information at all. +export function getFixChannelLayoutFilter({ channels, channelLayout }: { + channels?: number | undefined, + channelLayout?: string | undefined, +}) { + if (channels == null || channels <= 0 || !hasUnsupportedChannelLayout(channelLayout)) return undefined; + return `channelmap=${Array.from({ length: channels }, (_, i) => i).join('|')}`; +} diff --git a/src/main/compatPlayer.ts b/src/main/compatPlayer.ts index 556faa4c..766d062e 100644 --- a/src/main/compatPlayer.ts +++ b/src/main/compatPlayer.ts @@ -11,14 +11,14 @@ export function createMediaSourceStream(params: Parameters abortController.abort(); async function attemptCreateProcess({ forceColorspace }: { forceColorspace?: boolean } = {}) { - const { videoStreamIndex, audioStreamIndexes, seekTo } = params; + const { videoStreamIndex, audioStreams, seekTo } = params; - logger.info('Starting preview process', { videoStreamIndex, audioStreamIndexes, seekTo }); + logger.info('Starting preview process', { videoStreamIndex, audioStreams, seekTo }); const process = createMediaSourceProcess({ ...params, forceColorspace }); // eslint-disable-next-line unicorn/prefer-add-event-listener abortController.signal.onabort = () => { - logger.info('Aborting preview process', { videoStreamIndex, audioStreamIndexes, seekTo }); + logger.info('Aborting preview process', { videoStreamIndex, audioStreams, seekTo }); process.kill('SIGKILL'); }; diff --git a/src/main/ffmpeg.ts b/src/main/ffmpeg.ts index b86be90f..22ee0d04 100644 --- a/src/main/ffmpeg.ts +++ b/src/main/ffmpeg.ts @@ -15,7 +15,7 @@ import type { FFprobeFormat } from '../common/ffprobe.js'; import isDev from './isDev.js'; import logger from './logger.js'; import { parseFfmpegProgressLine } from './progress.js'; -import { formatFfmpegNumber, getHwaccelArgs, parseFfprobeDuration } from '../common/util.js'; +import { formatFfmpegNumber, getFixChannelLayoutFilter, getHwaccelArgs, parseFfprobeDuration } from '../common/util.js'; import { getFfmpegJpegQuality } from './ffmpegUtil.js'; import { throwIfDisabledNetworking } from './networking.js'; @@ -591,10 +591,10 @@ export async function getDuration(filePath: string) { const enableLog = false; const encode = true; -export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIndexes, seekTo, size, fps, rotate, forceColorspace, ffmpegHwaccel }: { +export function createMediaSourceProcess({ path, videoStreamIndex, audioStreams, seekTo, size, fps, rotate, forceColorspace, ffmpegHwaccel }: { path: string, videoStreamIndex?: number | undefined, - audioStreamIndexes: number[], + audioStreams: { index: number, channels?: number | undefined, channelLayout?: string | undefined }[], seekTo: number, size?: number | undefined, fps?: number | undefined, @@ -664,18 +664,24 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn graph.push(`[0:${videoStreamIndex}]${videoFiltersStr}[video]`); } - if (audioStreamIndexes.length > 0) { - if (audioStreamIndexes.length > 1) { - const resampledStr = audioStreamIndexes.map((i) => `[resampled${i}]`).join(''); - const weightsStr = audioStreamIndexes.map(() => '1').join(' '); + if (audioStreams.length > 0) { + // some streams have a channel layout that ffmpeg cannot resample or downmix, so relabel it first + const getAudioFilters = (stream: typeof audioStreams[number], rest: string[]) => { + const filters = [getFixChannelLayoutFilter(stream), ...rest].filter((filter) => filter != null); + return filters.length > 0 ? filters.join(',') : 'anull'; + }; + + if (audioStreams.length > 1) { + const resampledStr = audioStreams.map(({ index }) => `[resampled${index}]`).join(''); + const weightsStr = audioStreams.map(() => '1').join(' '); graph.push( // First resample because else we get the lowest sample rate - ...audioStreamIndexes.map((i) => `[0:${i}]aresample=44100[resampled${i}]`), + ...audioStreams.map((stream) => `[0:${stream.index}]${getAudioFilters(stream, ['aresample=44100'])}[resampled${stream.index}]`), // now mix all audio channels together - `${resampledStr}amix=inputs=${audioStreamIndexes.length}:duration=longest:weights=${weightsStr}:normalize=0:dropout_transition=2[audio]`, + `${resampledStr}amix=inputs=${audioStreams.length}:duration=longest:weights=${weightsStr}:normalize=0:dropout_transition=2[audio]`, ); } else { - graph.push(`[0:${audioStreamIndexes[0]}]anull[audio]`); + graph.push(`[0:${audioStreams[0]!.index}]${getAudioFilters(audioStreams[0]!, [])}[audio]`); } } @@ -727,7 +733,7 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn '-g', '1', // reduces latency and buffering ] : ['-vn']), - ...(audioStreamIndexes.length > 0 ? [ + ...(audioStreams.length > 0 ? [ '-map', '[audio]', '-ac', '2', '-c:a', 'aac', '-b:a', '128k', ] : ['-an']), diff --git a/src/renderer/src/MediaSourcePlayer.tsx b/src/renderer/src/MediaSourcePlayer.tsx index af57036a..979c0cc9 100644 --- a/src/renderer/src/MediaSourcePlayer.tsx +++ b/src/renderer/src/MediaSourcePlayer.tsx @@ -13,12 +13,12 @@ import type { FfmpegHwAccel } from '../../common/types'; const { compatPlayer: { createMediaSourceStream } } = window.require('@electron/remote').require('./index.js'); -async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, audioStreamIndexes, seekTo, signal, size, fps, rotate, onCanPlay, onResetNeeded, onWaiting, ffmpegHwaccel }: { +async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, audioStreams, seekTo, signal, size, fps, rotate, onCanPlay, onResetNeeded, onWaiting, ffmpegHwaccel }: { path: string, slaveVideo: ChromiumHTMLVideoElement, masterVideo: ChromiumHTMLVideoElement, videoStreamIndex?: number | undefined, - audioStreamIndexes: number[], + audioStreams: { index: number, channels?: number | undefined, channelLayout?: string | undefined }[], seekTo: number, signal: AbortSignal, size?: number | undefined, @@ -73,7 +73,7 @@ async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, const codecs: string[] = []; if (videoStreamIndex != null) codecs.push('avc1.42C01F'); - if (audioStreamIndexes.length > 0) codecs.push('mp4a.40.2'); + if (audioStreams.length > 0) codecs.push('mp4a.40.2'); const codecTag = codecs.join(', '); const mimeCodec = `video/mp4; codecs="${codecTag}"`; @@ -90,7 +90,7 @@ async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, throw new Error(`Unsupported MIME type or codec: ${mimeCodec}`); } - mediaSourceProcess = createMediaSourceStream({ path, videoStreamIndex, audioStreamIndexes, seekTo, size, fps, rotate, ffmpegHwaccel }); + mediaSourceProcess = createMediaSourceStream({ path, videoStreamIndex, audioStreams, seekTo, size, fps, rotate, ffmpegHwaccel }); console.log('Waiting for media source process to emit first data...'); const readChunk = await mediaSourceProcess.promise; if (readChunk == null) { @@ -325,7 +325,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master console.error('video error', error); }, []); - const audioStreamIndexes = useMemo(() => audioStreams.map((s) => s.index), [audioStreams]); + const audioStreamsForPreview = useMemo(() => audioStreams.map(({ index, channels, channel_layout: channelLayout }) => ({ index, channels, channelLayout })), [audioStreams]); useEffect(() => { const video = videoRef.current; @@ -369,7 +369,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master slaveVideo: video, masterVideo, videoStreamIndex: videoStream?.index, - audioStreamIndexes, + audioStreams: audioStreamsForPreview, seekTo, size, fps, @@ -398,7 +398,7 @@ function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, master return () => abortController.abort(); // Important that we also have eventId in the deps, so that we can restart the preview when the eventId changes - }, [audioStreamIndexes, ffmpegHwaccel, filePath, masterVideoRef, mediaSourceQuality, rotate, videoStream]); + }, [audioStreamsForPreview, ffmpegHwaccel, filePath, masterVideoRef, mediaSourceQuality, rotate, videoStream]); const onFocus = useCallback>((e) => { // prevent video element from stealing focus in fullscreen mode https://github.com/mifi/lossless-cut/issues/543#issuecomment-1868167775 diff --git a/src/renderer/src/hooks/useFfmpegOperations.ts b/src/renderer/src/hooks/useFfmpegOperations.ts index ea075026..50963837 100644 --- a/src/renderer/src/hooks/useFfmpegOperations.ts +++ b/src/renderer/src/hooks/useFfmpegOperations.ts @@ -15,7 +15,7 @@ import { deleteDispositionValue, type AllFilesMeta, type Chapter, type CopyfileS import type { LossyMode } from '../../../main'; import { UserFacingError } from '../../errors'; import mainApi from '../mainApi'; -import { formatFfmpegNumber, getHwaccelArgs } from '../../../common/util'; +import { formatFfmpegNumber, getFixChannelLayoutFilter, getHwaccelArgs, hasUnsupportedChannelLayout } from '../../../common/util'; const { join, resolve, dirname } = window.require('node:path'); const { writeFile, mkdir, access, constants: { W_OK } } = window.require('node:fs/promises'); @@ -890,6 +890,26 @@ function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, trea } } + // Some files (e.g. DV/DVCPRO .mov) have an audio channel layout that ffmpeg cannot resample or + // downmix, which makes the encode fail. Relabel the channels first so that it can. + // We don't pass -map, so ffmpeg picks one audio stream by itself. Therefore only apply the filter + // when every audio stream has the same channel count, or the filter might not match the picked stream. + let audioFilterArgs: string[] = []; + if (audio != null && audio !== 'copy') { + try { + const audioStreams = (await readFileFfprobeMeta(filePathArg)).streams.filter((s) => s.codec_type === 'audio'); + const unsupportedStream = audioStreams.find((s) => hasUnsupportedChannelLayout(s.channel_layout)); + const sameChannelCount = new Set(audioStreams.map((s) => s.channels)).size === 1; + if (unsupportedStream != null && sameChannelCount) { + const filter = getFixChannelLayoutFilter({ channels: unsupportedStream.channels, channelLayout: unsupportedStream.channel_layout }); + if (filter != null) audioFilterArgs = ['-af', filter]; + } + } catch (err) { + // don't fail the conversion just because we couldn't probe it + console.warn('Failed to probe audio channel layout', err); + } + } + const ffmpegArgs = [ '-hide_banner', ...((video === 'lq' || video === 'hq') ? getHwaccelArgs(ffmpegHwaccel) : []), @@ -897,6 +917,7 @@ function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, trea '-i', filePathArg, ...videoArgs, ...audioArgs, + ...audioFilterArgs, '-sn', '-y', outPath, ]; diff --git a/tsconfig.common.json b/tsconfig.common.json index eda60050..886ae832 100644 --- a/tsconfig.common.json +++ b/tsconfig.common.json @@ -16,4 +16,9 @@ "include": [ "src/common/**/*", ], + // tests are run by vitest from the sources; don't emit them into the build output, + // or vitest would discover and run the compiled copies too + "exclude": [ + "src/common/**/*.test.ts", + ], } \ No newline at end of file