diff --git a/src/main/ffmpeg.ts b/src/main/ffmpeg.ts index 1726113f..18919fba 100644 --- a/src/main/ffmpeg.ts +++ b/src/main/ffmpeg.ts @@ -173,21 +173,32 @@ export async function runFfprobe(args: readonly string[], { timeout = isDev ? 10 } } -export async function renderWaveformPng({ filePath, start, duration, color, streamIndex }: { +export async function renderWaveformPng({ filePath, start, duration, resample, color, streamIndex, timeout }: { filePath: string, - start: number, - duration: number, + start?: number, + duration?: number, + resample?: number, color: string, streamIndex: number, + timeout?: number, }): Promise { const args1 = [ '-hide_banner', '-i', filePath, - '-ss', String(start), - '-t', String(duration), - '-c', 'copy', '-vn', '-map', `0:${streamIndex}`, + ...(start != null ? ['-ss', String(start)] : []), + ...(duration != null ? ['-t', String(duration)] : []), + ...(resample != null ? [ + // the operation is faster if we resample + // the higher the resample rate, the faster the resample + // but the slower the showwavespic operation will be... + // https://github.com/mifi/lossless-cut/issues/260#issuecomment-605603456 + '-c:a', 'pcm_s32le', + '-ar', String(resample), + ] : [ + '-c', 'copy', + ]), '-f', 'matroska', // mpegts doesn't support vorbis etc '-', ]; @@ -202,14 +213,13 @@ export async function renderWaveformPng({ filePath, start, duration, color, stre '-', ]; - logger.info(`${getFfCommandLine('ffmpeg1', args1)} | \n${getFfCommandLine('ffmpeg2', args2)}`); + logger.info(`${getFfCommandLine('ffmpeg', args1)} | \n${getFfCommandLine('ffmpeg', args2)}`); let ps1: ResultPromise<{ encoding: 'buffer' }> | undefined; let ps2: ResultPromise<{ encoding: 'buffer' }> | undefined; try { - const timeout = 10000; - ps1 = runFfmpegProcess(args1, { buffer: false, timeout }, { logCli: false }); - ps2 = runFfmpegProcess(args2, { timeout }, { logCli: false }); + ps1 = runFfmpegProcess(args1, { buffer: false, ...(timeout != null && { timeout }) }, { logCli: false }); + ps2 = runFfmpegProcess(args2, timeout != null ? { timeout } : undefined, { logCli: false }); assert(ps1.stdout != null); assert(ps2.stdin != null); ps1.stdout.pipe(ps2.stdin); diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index ea68c47b..10547cff 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -259,7 +259,10 @@ function App() { const toggleSegmentsList = useCallback(() => setShowRightBar((v) => !v), []); const toggleWaveformMode = useCallback(() => { + // eslint-disable-next-line unicorn/prefer-switch if (waveformMode === 'waveform') { + setWaveformMode('waveform-tall'); + } else if (waveformMode === 'waveform-tall') { setWaveformMode('big-waveform'); } else if (waveformMode === 'big-waveform') { setWaveformMode(undefined); @@ -532,17 +535,27 @@ function App() { const hasAudio = !!activeAudioStream; const hasVideo = !!activeVideoStream; - const waveformEnabled = hasAudio && waveformMode != null && ['waveform', 'big-waveform'].includes(waveformMode); + const waveformEnabled = hasAudio && waveformMode != null; const bigWaveformEnabled = waveformEnabled && waveformMode === 'big-waveform'; const showThumbnails = thumbnailsEnabled && hasVideo; const { thumbnailsSorted, setThumbnails } = useThumbnails({ filePath, zoomedDuration, zoomWindowStartTime, showThumbnails }); - const shouldShowKeyframes = keyframesEnabled && hasVideo && calcShouldShowKeyframes(zoomedDuration); - const shouldShowWaveform = calcShouldShowWaveform(zoomedDuration); - const { neighbouringKeyFrames, findNearestKeyFrameTime } = useKeyframes({ keyframesEnabled, filePath, commandedTime, videoStream: activeVideoStream, detectedFps, ffmpegExtractWindow }); - const { waveforms } = useWaveform({ filePath, relevantTime, waveformEnabled, audioStream: activeAudioStream, ffmpegExtractWindow, fileDuration }); + const { waveforms, overviewWaveform, renderOverviewWaveform } = useWaveform({ filePath, relevantTime, waveformEnabled, audioStream: activeAudioStream, ffmpegExtractWindow, fileDuration }); + + const onGenerateOverviewWaveformClick = useCallback(async () => { + if (working) return; + try { + setWorking({ text: t('Generating full overview waveform, this may take a few minutes.') }); + await renderOverviewWaveform(); + } finally { + setWorking(); + } + }, [renderOverviewWaveform, setWorking, t, working]); + + const shouldShowKeyframes = keyframesEnabled && hasVideo && calcShouldShowKeyframes(zoomedDuration); + const shouldShowWaveform = calcShouldShowWaveform(zoomedDuration) || overviewWaveform != null; const resetState = useCallback(() => { console.log('State reset'); @@ -2556,7 +2569,9 @@ function App() { toggleWaveformMode()} diff --git a/src/renderer/src/Timeline.tsx b/src/renderer/src/Timeline.tsx index 3800409b..eecb9389 100644 --- a/src/renderer/src/Timeline.tsx +++ b/src/renderer/src/Timeline.tsx @@ -15,7 +15,9 @@ import styles from './Timeline.module.css'; import { timelineBackground, darkModeTransition } from './colors'; import { Frame } from './ffmpeg'; -import { FormatTimecode, InverseCutSegment, RenderableWaveform, StateSegment, Thumbnail } from './types'; +import { FormatTimecode, InverseCutSegment, OverviewWaveform, RenderableWaveform, WaveformSlice, StateSegment, Thumbnail } from './types'; +import Button from './components/Button'; +import { WaveformMode } from '../../../types'; type CalculateTimelinePercent = (time: number) => string | undefined; @@ -29,13 +31,13 @@ const Waveform = memo(({ waveform, calculateTimelinePercent, fileDurationNonZero fileDurationNonZero: number, darkMode: boolean, }) => { - const leftPos = calculateTimelinePercent(waveform.from); + const leftPos = 'from' in waveform ? calculateTimelinePercent(waveform.from) : '0%'; - const toTruncated = Math.min(waveform.to, fileDurationNonZero); + const width = 'to' in waveform ? ((Math.min(waveform.to, fileDurationNonZero) - waveform.from) / fileDurationNonZero) * 100 : 100; const style = useMemo(() => ({ - position: 'absolute', height: '100%', left: leftPos, width: `${((toTruncated - waveform.from) / fileDurationNonZero) * 100}%`, filter: darkMode ? undefined : 'invert(1)', - }), [darkMode, fileDurationNonZero, leftPos, toTruncated, waveform.from]); + pointerEvents: 'none', position: 'absolute', height: '100%', left: leftPos, width: `${width}%`, filter: darkMode ? undefined : 'invert(1)', imageRendering: 'pixelated', + }), [darkMode, leftPos, width]); if (waveform.url == null) { return
; @@ -47,16 +49,19 @@ const Waveform = memo(({ waveform, calculateTimelinePercent, fileDurationNonZero }); // eslint-disable-next-line react/display-name -const Waveforms = memo(({ calculateTimelinePercent, fileDurationNonZero, waveforms, zoom, height, darkMode }: { +const Waveforms = memo(({ calculateTimelinePercent, fileDurationNonZero, waveforms, overviewWaveform, zoom, darkMode, waveformMode }: { calculateTimelinePercent: CalculateTimelinePercent, fileDurationNonZero: number, - waveforms: RenderableWaveform[], + waveforms: WaveformSlice[], + overviewWaveform: OverviewWaveform | undefined, zoom: number, - height: number, darkMode: boolean, + waveformMode: WaveformMode | undefined, }) => ( -
- {waveforms.map((waveform) => ( +
+ {zoom === 1 && overviewWaveform != null ? ( + + ) : waveforms.map((waveform) => ( ))}
@@ -95,12 +100,15 @@ function Timeline({ formatTimecode, formatTimeAndFrames, waveforms, + overviewWaveform, shouldShowWaveform, shouldShowKeyframes, thumbnails, zoomWindowStartTime, zoomWindowEndTime, onZoomWindowStartTimeChange, + onGenerateOverviewWaveformClick, + waveformMode, waveformEnabled, showThumbnails, playing, @@ -125,13 +133,16 @@ function Timeline({ inverseCutSegments: InverseCutSegment[], formatTimecode: FormatTimecode, formatTimeAndFrames: (a: number) => string, - waveforms: RenderableWaveform[], + waveforms: WaveformSlice[], + overviewWaveform: OverviewWaveform | undefined, shouldShowWaveform: boolean, shouldShowKeyframes: boolean, thumbnails: Thumbnail[], zoomWindowStartTime: number, zoomWindowEndTime: number | undefined, onZoomWindowStartTimeChange: (a: number) => void, + onGenerateOverviewWaveformClick: () => void, + waveformMode: WaveformMode | undefined, waveformEnabled: boolean, showThumbnails: boolean, playing: boolean, @@ -328,6 +339,11 @@ function Timeline({ useContextMenu(timelineScrollerRef, contextMenuTemplate); + const onGenerateOverviewWaveformClick2 = useCallback>((e) => { + e.preventDefault(); // todo this doesn't work. dunno why + onGenerateOverviewWaveformClick(); + }, [onGenerateOverviewWaveformClick]); + return ( // eslint-disable-next-line jsx-a11y/no-static-element-interactions,jsx-a11y/mouse-events-have-key-events
{(waveformEnabled && !shouldShowWaveform) && ( -
+
{t('Zoom in more to view waveform')} +
)} @@ -350,14 +367,15 @@ function Timeline({ onScroll={onTimelineScroll} ref={timelineScrollerRef} > - {waveformEnabled && shouldShowWaveform && waveforms.length > 0 && ( + {waveformEnabled && shouldShowWaveform && (waveforms.length > 0 || overviewWaveform != null) && ( )} diff --git a/src/renderer/src/components/BigWaveform.tsx b/src/renderer/src/components/BigWaveform.tsx index e83f33ad..175f5c4b 100644 --- a/src/renderer/src/components/BigWaveform.tsx +++ b/src/renderer/src/components/BigWaveform.tsx @@ -2,11 +2,11 @@ import { memo, useEffect, useState, useCallback, useRef, CSSProperties, MouseEve import { Spinner } from 'evergreen-ui'; import { ffmpegExtractWindow } from '../util/constants'; -import { RenderableWaveform } from '../types'; +import { WaveformSlice } from '../types'; function BigWaveform({ waveforms, relevantTime, playing, fileDurationNonZero, zoom, seekRel, darkMode }: { - waveforms: RenderableWaveform[], + waveforms: WaveformSlice[], relevantTime: number, playing: boolean, fileDurationNonZero: number, diff --git a/src/renderer/src/components/Working.tsx b/src/renderer/src/components/Working.tsx index ee7d1026..bda94934 100644 --- a/src/renderer/src/components/Working.tsx +++ b/src/renderer/src/components/Working.tsx @@ -9,7 +9,9 @@ import loadingLottie from '../7077-magic-flow.json'; function Working({ text, progress, onAbortClick }: { - text: string, progress?: number | undefined, onAbortClick: () => void + text: string, + progress?: number | undefined, + onAbortClick: () => void }) { return (
diff --git a/src/renderer/src/hooks/useLoading.ts b/src/renderer/src/hooks/useLoading.ts index 1be02066..c99e1ceb 100644 --- a/src/renderer/src/hooks/useLoading.ts +++ b/src/renderer/src/hooks/useLoading.ts @@ -10,10 +10,11 @@ export default () => { // Store "working" in a ref so we can avoid race conditions const workingRef = useRef(!!working); + const setWorking = useCallback((valOrBool?: { text: string, abortController?: AbortController } | true | undefined) => { workingRef.current = !!valOrBool; const val = valOrBool === true ? { text: t('Loading') } : valOrBool; - setWorkingState(val ? { text: val.text, abortController: val.abortController } : undefined); + setWorkingState(val); }, [t]); const abortWorking = useCallback(() => { diff --git a/src/renderer/src/hooks/useWaveform.ts b/src/renderer/src/hooks/useWaveform.ts index 1620bf62..26bce0e1 100644 --- a/src/renderer/src/hooks/useWaveform.ts +++ b/src/renderer/src/hooks/useWaveform.ts @@ -1,14 +1,17 @@ -import { useState, useRef, useEffect } from 'react'; +import { useState, useRef, useEffect, useCallback } from 'react'; import sortBy from 'lodash/sortBy'; +import invariant from 'tiny-invariant'; import { renderWaveformPng, safeCreateBlob } from '../ffmpeg'; -import { RenderableWaveform } from '../types'; +import { OverviewWaveform, WaveformSlice } from '../types'; import { FFprobeStream } from '../../../../ffprobe'; const maxWaveforms = 100; // const maxWaveforms = 3; // testing +const color = '#ffffff'; + export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioStream, ffmpegExtractWindow }: { filePath: string | undefined, relevantTime: number, @@ -17,8 +20,9 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt audioStream: FFprobeStream | undefined, ffmpegExtractWindow: number, }) => { - const [waveforms, setWaveforms] = useState([]); - const waveformsRef = useRef(); + const [waveforms, setWaveforms] = useState([]); + const [overviewWaveform, setOverviewWaveform] = useState(); + const waveformsRef = useRef(); useEffect(() => { waveformsRef.current = waveforms; @@ -27,6 +31,7 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt useEffect(() => { waveformsRef.current = []; setWaveforms([]); + setOverviewWaveform(undefined); }, [filePath, audioStream, setWaveforms]); const waveformStartTime = Math.floor(relevantTime / ffmpegExtractWindow) * ffmpegExtractWindow; @@ -57,7 +62,7 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt const alreadyHaveWaveformAtTime = (waveformsRef.current ?? []).some((waveform) => waveform.from === time); if (!alreadyHaveWaveformAtTime) { try { - const promise = renderWaveformPng({ filePath, start: time, duration: safeExtractDuration, color: '##000000', streamIndex: audioStream.index }); + const promise = renderWaveformPng({ filePath, start: time, duration: safeExtractDuration, color, streamIndex: audioStream.index, timeout: 10000 }); setWaveforms((currentWaveforms) => { const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt'); @@ -108,7 +113,7 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt }; }, [audioStream, ffmpegExtractWindow, filePath, safeExtractDuration, waveformEnabled]); - const lastWaveformsRef = useRef([]); + const lastWaveformsRef = useRef([]); useEffect(() => { const removedWaveforms = lastWaveformsRef.current.filter((wf) => !waveforms.includes(wf)); // Cleanup old @@ -122,7 +127,29 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt lastWaveformsRef.current = waveforms; }, [waveforms]); + const renderOverviewWaveform = useCallback(async () => { + invariant(filePath != null); + invariant(audioStream != null); + + // todo allow actual abort + const promise = renderWaveformPng({ filePath, color, streamIndex: audioStream.index, resample: 10000 }); + + const { buffer } = await promise; + + setOverviewWaveform({ + createdAt: new Date(), + url: URL.createObjectURL(safeCreateBlob(buffer, { type: 'image/png' })), + }); + }, [audioStream, filePath]); + + useEffect(() => () => { + if (overviewWaveform?.url != null) { + console.log('Cleanup overview waveform'); + URL.revokeObjectURL(overviewWaveform.url); + } + }, [overviewWaveform]); + useEffect(() => () => setWaveforms([]), [setWaveforms]); - return { waveforms }; + return { overviewWaveform, waveforms, renderOverviewWaveform }; }; diff --git a/src/renderer/src/types.ts b/src/renderer/src/types.ts index c4eebb44..62d91879 100644 --- a/src/renderer/src/types.ts +++ b/src/renderer/src/types.ts @@ -87,14 +87,23 @@ export type EdlExportType = 'csv' | 'tsv-human' | 'csv-human' | 'csv-frames' | ' export type TunerType = 'wheelSensitivity' | 'keyboardNormalSeekSpeed' | 'keyboardSeekSpeed2' | 'keyboardSeekSpeed3' | 'keyboardSeekAccFactor'; -export interface RenderableWaveform { +export interface WaveformBase { createdAt: Date, +} + +export interface WaveformSlice extends WaveformBase { from: number, to: number, duration: number, - url?: string, + url?: string, // undefined while rendering } +export interface OverviewWaveform extends WaveformBase { + url: string, +} + +export type RenderableWaveform = WaveformSlice | OverviewWaveform; + export type FfmpegCommandLog = { command: string, time: Date }[]; export interface Thumbnail { diff --git a/types.ts b/types.ts index dd22adf1..79565437 100644 --- a/types.ts +++ b/types.ts @@ -127,7 +127,7 @@ export interface ApiActionRequest { export type Html5ifyMode = 'fastest' | 'fast-audio-remux' | 'fast-audio' | 'fast' | 'slow' | 'slow-audio' | 'slowest'; -export type WaveformMode = 'big-waveform' | 'waveform'; +export type WaveformMode = 'big-waveform' | 'waveform' | 'waveform-tall'; // This is the contract with the user, see https://github.com/mifi/lossless-cut/blob/master/expressions.md export interface ScopeSegment {