improve waveform rendering #260

make it faster
also render around current time

src/main/ffmpeg.ts

@@ -174,7 +174,11 @@ export async function runFfprobe(args: readonly string[], { timeout = isDev ? 10
 }
 
 export async function renderWaveformPng({ filePath, start, duration, color, streamIndex }: {
-  filePath: string, start: number, duration: number, color: string, streamIndex: number,
+  filePath: string,
+  start: number,
+  duration: number,
+  color: string,
+  streamIndex: number,
 }): Promise<Waveform> {
   const args1 = [
     '-hide_banner',
@@ -203,31 +207,21 @@ export async function renderWaveformPng({ filePath, start, duration, color, stre
   let ps1: ResultPromise<{ encoding: 'buffer' }> | undefined;
   let ps2: ResultPromise<{ encoding: 'buffer' }> | undefined;
   try {
-    ps1 = runFfmpegProcess(args1, { buffer: false }, { logCli: false });
-    ps2 = runFfmpegProcess(args2, undefined, { logCli: false });
+    const timeout = 10000;
+    ps1 = runFfmpegProcess(args1, { buffer: false, timeout }, { logCli: false });
+    ps2 = runFfmpegProcess(args2, { timeout }, { logCli: false });
     assert(ps1.stdout != null);
     assert(ps2.stdin != null);
     ps1.stdout.pipe(ps2.stdin);
 
-    const timer = setTimeout(() => {
-      ps1?.kill();
-      ps2?.kill();
-      logger.warn('ffmpeg timed out');
-    }, 10000);
-
-    let stdout;
-    try {
-      ({ stdout } = await ps2);
-    } finally {
-      clearTimeout(timer);
-    }
+    const { stdout } = await ps2;
 
     return {
       buffer: Buffer.from(stdout),
     };
   } catch (err) {
-    if (ps1) ps1.kill();
-    if (ps2) ps2.kill();
+    ps1?.kill();
+    ps2?.kill();
     throw err;
   }
 }

src/renderer/src/hooks/useWaveform.ts

@@ -1,6 +1,5 @@
 import { useState, useRef, useEffect } from 'react';
 import sortBy from 'lodash/sortBy';
-import { useThrottle } from '@uidotdev/usehooks';
 
 import { renderWaveformPng, safeCreateBlob } from '../ffmpeg';
 import { RenderableWaveform } from '../types';
@@ -18,7 +17,6 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
   audioStream: FFprobeStream | undefined,
   ffmpegExtractWindow: number,
 }) => {
-  const creatingWaveformPromise = useRef<Promise<unknown>>();
   const [waveforms, setWaveforms] = useState<RenderableWaveform[]>([]);
   const waveformsRef = useRef<RenderableWaveform[]>();
 
@@ -34,29 +32,42 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
   const waveformStartTime = Math.floor(relevantTime / ffmpegExtractWindow) * ffmpegExtractWindow;
   const safeExtractDuration = fileDuration != null ? Math.min(waveformStartTime + ffmpegExtractWindow, fileDuration) - waveformStartTime : undefined;
 
-  const waveformStartTimeThrottled = useThrottle(waveformStartTime, 1000);
+  const waveformStartTimeRef = useRef(waveformStartTime);
+
+  useEffect(() => {
+    waveformStartTimeRef.current = waveformStartTime;
+  }, [waveformStartTime]);
 
   useEffect(() => {
     let aborted = false;
 
     (async () => {
-      const alreadyHaveWaveformAtTime = (waveformsRef.current ?? []).some((waveform) => waveform.from === waveformStartTimeThrottled);
-      const shouldRun = !!filePath && safeExtractDuration != null && audioStream && waveformEnabled && !alreadyHaveWaveformAtTime && !creatingWaveformPromise.current;
-      if (!shouldRun) return;
+      if (!filePath || safeExtractDuration == null || !audioStream || !waveformEnabled) {
+        return;
+      }
+
+      while (!aborted) {
+        const times = [
+          waveformStartTimeRef.current,
+          waveformStartTimeRef.current + ffmpegExtractWindow,
+          waveformStartTimeRef.current - ffmpegExtractWindow,
+        ];
 
+        for (const time of times) {
+          const alreadyHaveWaveformAtTime = (waveformsRef.current ?? []).some((waveform) => waveform.from === time);
+          if (!alreadyHaveWaveformAtTime) {
             try {
-        const promise = renderWaveformPng({ filePath, start: waveformStartTimeThrottled, duration: safeExtractDuration, color: '##000000', streamIndex: audioStream.index });
-        creatingWaveformPromise.current = promise;
+              const promise = renderWaveformPng({ filePath, start: time, duration: safeExtractDuration, color: '##000000', streamIndex: audioStream.index });
 
               setWaveforms((currentWaveforms) => {
                 const waveformsByCreatedAt = sortBy(currentWaveforms, 'createdAt');
                 return [
-            // cleanup old
+                  // If too many waveforms, cleanup old
                   ...(currentWaveforms.length >= maxWaveforms ? waveformsByCreatedAt.slice(1) : waveformsByCreatedAt),
-            // add new
+                  // Add new waveform
                   {
-              from: waveformStartTimeThrottled,
-              to: waveformStartTimeThrottled + safeExtractDuration,
+                    from: time,
+                    to: time + safeExtractDuration,
                     duration: safeExtractDuration,
                     createdAt: new Date(),
                   },
@@ -66,12 +77,13 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
               const { buffer } = await promise;
 
               if (aborted) {
-          setWaveforms((currentWaveforms) => currentWaveforms.filter((w) => w.from !== waveformStartTimeThrottled));
+                // remove unfinished waveform
+                setWaveforms((currentWaveforms) => currentWaveforms.filter((w) => w.from !== time));
                 return;
               }
 
               setWaveforms((currentWaveforms) => currentWaveforms.map((w) => {
-          if (w.from !== waveformStartTimeThrottled) {
+                if (w.from !== time) {
                   return w;
                 }
 
@@ -82,15 +94,19 @@ export default ({ filePath, relevantTime, fileDuration, waveformEnabled, audioSt
               }));
             } catch (err) {
               console.error('Failed to render waveform', err);
-      } finally {
-        creatingWaveformPromise.current = undefined;
+            }
+          }
+        }
+
+        // could be problematic if we spawn ffmpeg processes too often, so throttle it
+        await new Promise((r) => setTimeout(r, 100));
       }
     })();
 
     return () => {
       aborted = true;
     };
-  }, [audioStream, filePath, safeExtractDuration, waveformEnabled, waveformStartTimeThrottled]);
+  }, [audioStream, ffmpegExtractWindow, filePath, safeExtractDuration, waveformEnabled]);
 
   const lastWaveformsRef = useRef<RenderableWaveform[]>([]);
   useEffect(() => {