improve waveform #260

- allow loading full waveform overvoew
- allow taller bottom waveform (10% of window height)

src/main/ffmpeg.ts

@@ -173,21 +173,32 @@ export async function runFfprobe(args: readonly string[], { timeout = isDev ? 10
   }
 }
 
-export async function renderWaveformPng({ filePath, start, duration, color, streamIndex }: {
+export async function renderWaveformPng({ filePath, start, duration, resample, color, streamIndex, timeout }: {
   filePath: string,
-  start: number,
-  duration: number,
+  start?: number,
+  duration?: number,
+  resample?: number,
   color: string,
   streamIndex: number,
+  timeout?: number,
 }): Promise<Waveform> {
   const args1 = [
     '-hide_banner',
     '-i', filePath,
-    '-ss', String(start),
-    '-t', String(duration),
-    '-c', 'copy',
     '-vn',
     '-map', `0:${streamIndex}`,
+    ...(start != null ? ['-ss', String(start)] : []),
+    ...(duration != null ? ['-t', String(duration)] : []),
+    ...(resample != null ? [
+      // the operation is faster if we resample
+      // the higher the resample rate, the faster the resample
+      // but the slower the showwavespic operation will be...
+      // https://github.com/mifi/lossless-cut/issues/260#issuecomment-605603456
+      '-c:a', 'pcm_s32le',
+      '-ar', String(resample),
+    ] : [
+      '-c', 'copy',
+    ]),
     '-f', 'matroska', // mpegts doesn't support vorbis etc
     '-',
   ];
@@ -202,14 +213,13 @@ export async function renderWaveformPng({ filePath, start, duration, color, stre
     '-',
   ];
 
-  logger.info(`${getFfCommandLine('ffmpeg1', args1)} | \n${getFfCommandLine('ffmpeg2', args2)}`);
+  logger.info(`${getFfCommandLine('ffmpeg', args1)} | \n${getFfCommandLine('ffmpeg', args2)}`);
 
   let ps1: ResultPromise<{ encoding: 'buffer' }> | undefined;
   let ps2: ResultPromise<{ encoding: 'buffer' }> | undefined;
   try {
-    const timeout = 10000;
-    ps1 = runFfmpegProcess(args1, { buffer: false, timeout }, { logCli: false });
-    ps2 = runFfmpegProcess(args2, { timeout }, { logCli: false });
+    ps1 = runFfmpegProcess(args1, { buffer: false, ...(timeout != null && { timeout }) }, { logCli: false });
+    ps2 = runFfmpegProcess(args2, timeout != null ? { timeout } : undefined, { logCli: false });
     assert(ps1.stdout != null);
     assert(ps2.stdin != null);
     ps1.stdout.pipe(ps2.stdin);

src/renderer/src/App.tsx

@@ -259,7 +259,10 @@ function App() {
   const toggleSegmentsList = useCallback(() => setShowRightBar((v) => !v), []);
 
   const toggleWaveformMode = useCallback(() => {
+    // eslint-disable-next-line unicorn/prefer-switch
     if (waveformMode === 'waveform') {
+      setWaveformMode('waveform-tall');
+    } else if (waveformMode === 'waveform-tall') {
       setWaveformMode('big-waveform');
     } else if (waveformMode === 'big-waveform') {
       setWaveformMode(undefined);
@@ -532,17 +535,27 @@ function App() {
   const hasAudio = !!activeAudioStream;
   const hasVideo = !!activeVideoStream;
 
-  const waveformEnabled = hasAudio && waveformMode != null && ['waveform', 'big-waveform'].includes(waveformMode);
+  const waveformEnabled = hasAudio && waveformMode != null;
   const bigWaveformEnabled = waveformEnabled && waveformMode === 'big-waveform';
   const showThumbnails = thumbnailsEnabled && hasVideo;
 
   const { thumbnailsSorted, setThumbnails } = useThumbnails({ filePath, zoomedDuration, zoomWindowStartTime, showThumbnails });
 
-  const shouldShowKeyframes = keyframesEnabled && hasVideo && calcShouldShowKeyframes(zoomedDuration);
-  const shouldShowWaveform = calcShouldShowWaveform(zoomedDuration);
-
   const { neighbouringKeyFrames, findNearestKeyFrameTime } = useKeyframes({ keyframesEnabled, filePath, commandedTime, videoStream: activeVideoStream, detectedFps, ffmpegExtractWindow });
-  const { waveforms } = useWaveform({ filePath, relevantTime, waveformEnabled, audioStream: activeAudioStream, ffmpegExtractWindow, fileDuration });
+  const { waveforms, overviewWaveform, renderOverviewWaveform } = useWaveform({ filePath, relevantTime, waveformEnabled, audioStream: activeAudioStream, ffmpegExtractWindow, fileDuration });
+
+  const onGenerateOverviewWaveformClick = useCallback(async () => {
+    if (working) return;
+    try {
+      setWorking({ text: t('Generating full overview waveform, this may take a few minutes.') });
+      await renderOverviewWaveform();
+    } finally {
+      setWorking();
+    }
+  }, [renderOverviewWaveform, setWorking, t, working]);
+
+  const shouldShowKeyframes = keyframesEnabled && hasVideo && calcShouldShowKeyframes(zoomedDuration);
+  const shouldShowWaveform = calcShouldShowWaveform(zoomedDuration) || overviewWaveform != null;
 
   const resetState = useCallback(() => {
     console.log('State reset');
@@ -2556,7 +2569,9 @@ function App() {
                 <Timeline
                   shouldShowKeyframes={shouldShowKeyframes}
                   waveforms={waveforms}
+                  overviewWaveform={overviewWaveform}
                   shouldShowWaveform={shouldShowWaveform}
+                  waveformMode={waveformMode}
                   waveformEnabled={waveformEnabled}
                   showThumbnails={showThumbnails}
                   neighbouringKeyFrames={neighbouringKeyFrames}
@@ -2578,6 +2593,7 @@ function App() {
                   zoomWindowStartTime={zoomWindowStartTime}
                   zoomWindowEndTime={zoomWindowEndTime}
                   onZoomWindowStartTimeChange={setZoomWindowStartTime}
+                  onGenerateOverviewWaveformClick={onGenerateOverviewWaveformClick}
                   playing={playing}
                   isFileOpened={isFileOpened}
                   onWheel={onTimelineWheel}

src/renderer/src/BottomBar.tsx

@@ -350,7 +350,7 @@ function BottomBar({
               {hasAudio && (
                 <GiSoundWaves
                   size={24}
-                  style={{ padding: '0 .1em', color: waveformMode != null && ['big-waveform', 'waveform'].includes(waveformMode) ? primaryTextColor : undefined }}
+                  style={{ padding: '0 .1em', color: waveformMode != null ? primaryTextColor : undefined }}
                   role="button"
                   title={t('Show waveform')}
                   onClick={() => toggleWaveformMode()}

src/renderer/src/Timeline.tsx

@@ -15,7 +15,9 @@ import styles from './Timeline.module.css';
 
 import { timelineBackground, darkModeTransition } from './colors';
 import { Frame } from './ffmpeg';
-import { FormatTimecode, InverseCutSegment, RenderableWaveform, StateSegment, Thumbnail } from './types';
+import { FormatTimecode, InverseCutSegment, OverviewWaveform, RenderableWaveform, WaveformSlice, StateSegment, Thumbnail } from './types';
+import Button from './components/Button';
+import { WaveformMode } from '../../../types';
 
 
 type CalculateTimelinePercent = (time: number) => string | undefined;
@@ -29,13 +31,13 @@ const Waveform = memo(({ waveform, calculateTimelinePercent, fileDurationNonZero
   fileDurationNonZero: number,
   darkMode: boolean,
 }) => {
-  const leftPos = calculateTimelinePercent(waveform.from);
+  const leftPos = 'from' in waveform ? calculateTimelinePercent(waveform.from) : '0%';
 
-  const toTruncated = Math.min(waveform.to, fileDurationNonZero);
+  const width = 'to' in waveform ? ((Math.min(waveform.to, fileDurationNonZero) - waveform.from) / fileDurationNonZero) * 100 : 100;
 
   const style = useMemo<CSSProperties>(() => ({
-    position: 'absolute', height: '100%', left: leftPos, width: `${((toTruncated - waveform.from) / fileDurationNonZero) * 100}%`, filter: darkMode ? undefined : 'invert(1)',
-  }), [darkMode, fileDurationNonZero, leftPos, toTruncated, waveform.from]);
+    pointerEvents: 'none', position: 'absolute', height: '100%', left: leftPos, width: `${width}%`, filter: darkMode ? undefined : 'invert(1)', imageRendering: 'pixelated',
+  }), [darkMode, leftPos, width]);
 
   if (waveform.url == null) {
     return <div style={{ ...style }} className={styles['loading-bg']} />;
@@ -47,16 +49,19 @@ const Waveform = memo(({ waveform, calculateTimelinePercent, fileDurationNonZero
 });
 
 // eslint-disable-next-line react/display-name
-const Waveforms = memo(({ calculateTimelinePercent, fileDurationNonZero, waveforms, zoom, height, darkMode }: {
+const Waveforms = memo(({ calculateTimelinePercent, fileDurationNonZero, waveforms, overviewWaveform, zoom, darkMode, waveformMode }: {
   calculateTimelinePercent: CalculateTimelinePercent,
   fileDurationNonZero: number,
-  waveforms: RenderableWaveform[],
+  waveforms: WaveformSlice[],
+  overviewWaveform: OverviewWaveform | undefined,
   zoom: number,
-  height: number,
   darkMode: boolean,
+  waveformMode: WaveformMode | undefined,
 }) => (
-  <div style={{ height, width: `${zoom * 100}%`, position: 'relative' }}>
-    {waveforms.map((waveform) => (
+  <div style={{ height: waveformMode === 'waveform-tall' ? '15vh' : 40, width: `${zoom * 100}%`, position: 'relative' }}>
+    {zoom === 1 && overviewWaveform != null ? (
+      <Waveform waveform={overviewWaveform} calculateTimelinePercent={calculateTimelinePercent} fileDurationNonZero={fileDurationNonZero} darkMode={darkMode} />
+    ) : waveforms.map((waveform) => (
       <Waveform key={`${waveform.from}-${waveform.to}`} waveform={waveform} calculateTimelinePercent={calculateTimelinePercent} fileDurationNonZero={fileDurationNonZero} darkMode={darkMode} />
     ))}
   </div>
@@ -95,12 +100,15 @@ function Timeline({
   formatTimecode,
   formatTimeAndFrames,
   waveforms,
+  overviewWaveform,
   shouldShowWaveform,
   shouldShowKeyframes,
   thumbnails,
   zoomWindowStartTime,
   zoomWindowEndTime,
   onZoomWindowStartTimeChange,
+  onGenerateOverviewWaveformClick,
+  waveformMode,
   waveformEnabled,
   showThumbnails,
   playing,
@@ -125,13 +133,16 @@ function Timeline({
   inverseCutSegments: InverseCutSegment[],
   formatTimecode: FormatTimecode,
   formatTimeAndFrames: (a: number) => string,
-  waveforms: RenderableWaveform[],
+  waveforms: WaveformSlice[],
+  overviewWaveform: OverviewWaveform | undefined,
   shouldShowWaveform: boolean,
   shouldShowKeyframes: boolean,
   thumbnails: Thumbnail[],
   zoomWindowStartTime: number,
   zoomWindowEndTime: number | undefined,
   onZoomWindowStartTimeChange: (a: number) => void,
+  onGenerateOverviewWaveformClick: () => void,
+  waveformMode: WaveformMode | undefined,
   waveformEnabled: boolean,
   showThumbnails: boolean,
   playing: boolean,
@@ -328,6 +339,11 @@ function Timeline({
 
   useContextMenu(timelineScrollerRef, contextMenuTemplate);
 
+  const onGenerateOverviewWaveformClick2 = useCallback<MouseEventHandler<HTMLButtonElement>>((e) => {
+    e.preventDefault(); // todo this doesn't work. dunno why
+    onGenerateOverviewWaveformClick();
+  }, [onGenerateOverviewWaveformClick]);
+
   return (
     // eslint-disable-next-line jsx-a11y/no-static-element-interactions,jsx-a11y/mouse-events-have-key-events
     <div
@@ -338,8 +354,9 @@ function Timeline({
       onMouseOut={onMouseOut}
     >
       {(waveformEnabled && !shouldShowWaveform) && (
-        <div style={{ pointerEvents: 'none', display: 'flex', alignItems: 'center', justifyContent: 'center', height: timelineHeight, bottom: timelineHeight, left: 0, right: 0, color: 'var(--gray11)' }}>
+        <div style={{ display: 'flex', alignItems: 'center', justifyContent: 'center', height: timelineHeight, bottom: timelineHeight, left: 0, right: 0, color: 'var(--gray11)' }}>
           {t('Zoom in more to view waveform')}
+          <Button onClick={onGenerateOverviewWaveformClick2} style={{ marginLeft: '.5em' }}>{t('Load overview')}</Button>
         </div>
       )}
 
@@ -350,14 +367,15 @@ function Timeline({
         onScroll={onTimelineScroll}
         ref={timelineScrollerRef}
       >
-        {waveformEnabled && shouldShowWaveform && waveforms.length > 0 && (
+        {waveformEnabled && shouldShowWaveform && (waveforms.length > 0 || overviewWaveform != null) && (
           <Waveforms
             calculateTimelinePercent={calculateTimelinePercent}
             fileDurationNonZero={fileDurationNonZero}
             waveforms={waveforms}
+            overviewWaveform={overviewWaveform}
             zoom={zoom}
-            height={40}
             darkMode={darkMode}
+            waveformMode={waveformMode}
           />
         )}
 

src/renderer/src/components/BigWaveform.tsx

@@ -2,11 +2,11 @@ import { memo, useEffect, useState, useCallback, useRef, CSSProperties, MouseEve
 import { Spinner } from 'evergreen-ui';
 
 import { ffmpegExtractWindow } from '../util/constants';
-import { RenderableWaveform } from '../types';
+import { WaveformSlice } from '../types';
 
 
 function BigWaveform({ waveforms, relevantTime, playing, fileDurationNonZero, zoom, seekRel, darkMode }: {
-  waveforms: RenderableWaveform[],
+  waveforms: WaveformSlice[],
   relevantTime: number,
   playing: boolean,
   fileDurationNonZero: number,

src/renderer/src/components/Working.tsx

@@ -9,7 +9,9 @@ import loadingLottie from '../7077-magic-flow.json';
 
 
 function Working({ text, progress, onAbortClick }: {
-  text: string, progress?: number | undefined, onAbortClick: () => void
+  text: string,
+  progress?: number | undefined,
+  onAbortClick: () => void
 }) {
   return (
     <div style={{ position: 'absolute', bottom: 0, top: 0, left: 0, right: 0, display: 'flex', justifyContent: 'center', alignItems: 'center' }}>

src/renderer/src/hooks/useLoading.ts

@@ -10,10 +10,11 @@ export default () => {
 
   // Store "working" in a ref so we can avoid race conditions
   const workingRef = useRef(!!working);
+
   const setWorking = useCallback((valOrBool?: { text: string, abortController?: AbortController } | true | undefined) => {
     workingRef.current = !!valOrBool;
     const val = valOrBool === true ? { text: t('Loading') } : valOrBool;
-    setWorkingState(val ? { text: val.text, abortController: val.abortController } : undefined);
+    setWorkingState(val);
   }, [t]);
 
   const abortWorking = useCallback(() => {

src/renderer/src/hooks/useWaveform.ts

@@ -1,14 +1,17 @@
-import { useState, useRef, useEffect } from 'react';
+import { useState, useRef, useEffect, useCallback } from 'react';
 import sortBy from 'lodash/sortBy';
+import invariant from 'tiny-invariant';
 
 import { renderWaveformPng, safeCreateBlob } from '../ffmpeg';
-import { RenderableWaveform } from '../types';
+import { OverviewWaveform, WaveformSlice } from '../types';
 import { FFprobeStream } from '../../../../ffprobe';
 
 
 const maxWaveforms = 100;
 // const maxWaveforms = 3; // testing
 
+const color = '#ffffff';
+
 export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioStream, ffmpegExtractWindow }: {
   filePath: string | undefined,
   relevantTime: number,
@@ -17,8 +20,9 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
   audioStream: FFprobeStream | undefined,
   ffmpegExtractWindow: number,
 }) => {
-  const [waveforms, setWaveforms] = useState<RenderableWaveform[]>([]);
-  const waveformsRef = useRef<RenderableWaveform[]>();
+  const [waveforms, setWaveforms] = useState<WaveformSlice[]>([]);
+  const [overviewWaveform, setOverviewWaveform] = useState<OverviewWaveform>();
+  const waveformsRef = useRef<WaveformSlice[]>();
 
   useEffect(() => {
     waveformsRef.current = waveforms;
@@ -27,6 +31,7 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
   useEffect(() => {
     waveformsRef.current = [];
     setWaveforms([]);
+    setOverviewWaveform(undefined);
   }, [filePath, audioStream, setWaveforms]);
 
   const waveformStartTime = Math.floor(relevantTime / ffmpegExtractWindow) * ffmpegExtractWindow;
@@ -57,7 +62,7 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
           const alreadyHaveWaveformAtTime = (waveformsRef.current ?? []).some((waveform) => waveform.from === time);
           if (!alreadyHaveWaveformAtTime) {
             try {
-              const promise = renderWaveformPng({ filePath, start: time, duration: safeExtractDuration, color: '##000000', streamIndex: audioStream.index });
+              const promise = renderWaveformPng({ filePath, start: time, duration: safeExtractDuration, color, streamIndex: audioStream.index, timeout: 10000 });
 
               setWaveforms((currentWaveforms) => {
                 const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt');
@@ -108,7 +113,7 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
     };
   }, [audioStream, ffmpegExtractWindow, filePath, safeExtractDuration, waveformEnabled]);
 
-  const lastWaveformsRef = useRef<RenderableWaveform[]>([]);
+  const lastWaveformsRef = useRef<WaveformSlice[]>([]);
   useEffect(() => {
     const removedWaveforms = lastWaveformsRef.current.filter((wf) => !waveforms.includes(wf));
     // Cleanup old
@@ -122,7 +127,29 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
     lastWaveformsRef.current = waveforms;
   }, [waveforms]);
 
+  const renderOverviewWaveform = useCallback(async () => {
+    invariant(filePath != null);
+    invariant(audioStream != null);
+
+    // todo allow actual abort
+    const promise = renderWaveformPng({ filePath, color, streamIndex: audioStream.index, resample: 10000 });
+
+    const { buffer } = await promise;
+
+    setOverviewWaveform({
+      createdAt: new Date(),
+      url: URL.createObjectURL(safeCreateBlob(buffer, { type: 'image/png' })),
+    });
+  }, [audioStream, filePath]);
+
+  useEffect(() => () => {
+    if (overviewWaveform?.url != null) {
+      console.log('Cleanup overview waveform');
+      URL.revokeObjectURL(overviewWaveform.url);
+    }
+  }, [overviewWaveform]);
+
   useEffect(() => () => setWaveforms([]), [setWaveforms]);
 
-  return { waveforms };
+  return { overviewWaveform, waveforms, renderOverviewWaveform };
 };

src/renderer/src/types.ts

@@ -87,14 +87,23 @@ export type EdlExportType = 'csv' | 'tsv-human' | 'csv-human' | 'csv-frames' | '
 
 export type TunerType = 'wheelSensitivity' | 'keyboardNormalSeekSpeed' | 'keyboardSeekSpeed2' | 'keyboardSeekSpeed3' | 'keyboardSeekAccFactor';
 
-export interface RenderableWaveform {
+export interface WaveformBase {
   createdAt: Date,
+}
+
+export interface WaveformSlice extends WaveformBase {
   from: number,
   to: number,
   duration: number,
-  url?: string,
+  url?: string, // undefined while rendering
 }
 
+export interface OverviewWaveform extends WaveformBase {
+  url: string,
+}
+
+export type RenderableWaveform = WaveformSlice | OverviewWaveform;
+
 export type FfmpegCommandLog = { command: string, time: Date }[];
 
 export interface Thumbnail {

types.ts

@@ -127,7 +127,7 @@ export interface ApiActionRequest {
 
 export type Html5ifyMode = 'fastest' | 'fast-audio-remux' | 'fast-audio' | 'fast' | 'slow' | 'slow-audio' | 'slowest';
 
-export type WaveformMode = 'big-waveform' | 'waveform';
+export type WaveformMode = 'big-waveform' | 'waveform' | 'waveform-tall';
 
 // This is the contract with the user, see https://github.com/mifi/lossless-cut/blob/master/expressions.md
 export interface ScopeSegment {