implement align segments to keyframes

public/menu.js

@@ -246,6 +246,12 @@ module.exports = (app, mainWindow, newVersion) => {
                 mainWindow.webContents.send('shiftAllSegmentTimes');
               },
             },
+            {
+              label: i18n.t('Align segment times to keyframes'),
+              click() {
+                mainWindow.webContents.send('alignSegmentTimesToKeyframes');
+              },
+            },
           ],
         },
         {

src/App.jsx

@@ -9,6 +9,7 @@ import { useDebounce } from 'use-debounce';
 import i18n from 'i18next';
 import { useTranslation } from 'react-i18next';
 import JSON5 from 'json5';
+import pMap from 'p-map';
 
 import fromPairs from 'lodash/fromPairs';
 import sortBy from 'lodash/sortBy';
@@ -56,7 +57,7 @@ import {
   extractStreams, setCustomFfPath as ffmpegSetCustomFfPath,
   isIphoneHevc, isProblematicAvc1, tryMapChaptersToEdl, blackDetect, silenceDetect, detectSceneChanges as ffmpegDetectSceneChanges,
   getDuration, getTimecodeFromStreams, createChaptersFromSegments, extractSubtitleTrack,
-  RefuseOverwriteError, readFrames, mapTimesToSegments, abortFfmpegs,
+  RefuseOverwriteError, readFrames, mapTimesToSegments, abortFfmpegs, findKeyframeNearTime,
 } from './ffmpeg';
 import { shouldCopyStreamByDefault, getAudioStreams, getRealVideoStreams, isAudioDefinitelyNotSupported, willPlayerProperlyHandleVideo, doesPlayerSupportHevcPlayback, isStreamThumbnail } from './util/streams';
 import { exportEdlFile, readEdlFile, saveLlcProject, loadLlcProject, askForEdlImport } from './edlStore';
@@ -73,7 +74,7 @@ import { adjustRate } from './util/rate-calculator';
 import { showParametersDialog } from './dialogs/parameters';
 import { askExtractFramesAsImages } from './dialogs/extractFrames';
 import { askForHtml5ifySpeed } from './dialogs/html5ify';
-import { askForOutDir, askForInputDir, askForImportChapters, createNumSegments as createNumSegmentsDialog, createFixedDurationSegments as createFixedDurationSegmentsDialog, createRandomSegments as createRandomSegmentsDialog, promptTimeOffset, askForFileOpenAction, confirmExtractAllStreamsDialog, showCleanupFilesDialog, showDiskFull, showExportFailedDialog, showConcatFailedDialog, labelSegmentDialog, openYouTubeChaptersDialog, openAbout, showEditableJsonDialog, askForShiftSegments, selectSegmentsByLabelDialog, showRefuseToOverwrite, openDirToast, openCutFinishedToast, openConcatFinishedToast } from './dialogs';
+import { askForOutDir, askForInputDir, askForImportChapters, createNumSegments as createNumSegmentsDialog, createFixedDurationSegments as createFixedDurationSegmentsDialog, createRandomSegments as createRandomSegmentsDialog, promptTimeOffset, askForFileOpenAction, confirmExtractAllStreamsDialog, showCleanupFilesDialog, showDiskFull, showExportFailedDialog, showConcatFailedDialog, labelSegmentDialog, openYouTubeChaptersDialog, openAbout, showEditableJsonDialog, askForShiftSegments, selectSegmentsByLabelDialog, showRefuseToOverwrite, openDirToast, openCutFinishedToast, openConcatFinishedToast, askForAlignSegments } from './dialogs';
 import { openSendReportDialog } from './reporting';
 import { fallbackLng } from './i18n';
 import { createSegment, getCleanCutSegments, getSegApparentStart, getSegApparentEnd as getSegApparentEnd2, findSegmentsAtCursor, sortSegments, invertSegments, getSegmentTags, convertSegmentsToChapters, hasAnySegmentOverlap, combineOverlappingSegments as combineOverlappingSegments2, isDurationValid } from './segments';
@@ -442,22 +443,61 @@ const App = memo(() => {
 
   const isSegmentSelected = useCallback(({ segId }) => !deselectedSegmentIds[segId], [deselectedSegmentIds]);
 
+  const modifySelectedSegmentTimes = useCallback(async (transformSegment, concurrency = 5) => {
+    const clampValue = (val) => Math.min(Math.max(val, 0), duration);
+
+    let newSegments = await pMap(apparentCutSegments, async (segment) => {
+      if (!isSegmentSelected(segment)) return segment; // pass thru non-selected segments
+      const newSegment = await transformSegment(segment);
+      newSegment.start = clampValue(newSegment.start);
+      newSegment.end = clampValue(newSegment.end);
+      return newSegment;
+    }, { concurrency });
+    newSegments = newSegments.filter((segment) => segment.end > segment.start);
+    if (newSegments.length < 1) setCutSegments(createInitialCutSegments());
+    else setCutSegments(newSegments);
+  }, [apparentCutSegments, createInitialCutSegments, duration, isSegmentSelected, setCutSegments]);
+
   const shiftAllSegmentTimes = useCallback(async () => {
     const shift = await askForShiftSegments();
     if (shift == null) return;
-    const { shiftAmount, shiftValues } = shift;
-    const clampValue = (val) => Math.min(Math.max(val, 0), duration);
-    const newSegments = apparentCutSegments.map((segment) => {
-      if (!isSegmentSelected(segment)) return segment;
+
+    const { shiftAmount, shiftKeys } = shift;
+    await modifySelectedSegmentTimes((segment) => {
       const newSegment = { ...segment };
-      shiftValues.forEach((key) => {
-        newSegment[key] = clampValue(segment[key] + shiftAmount);
+      shiftKeys.forEach((key) => {
+        newSegment[key] += shiftAmount;
       });
       return newSegment;
-    }).filter((segment) => segment.end > segment.start);
-    if (newSegments.length < 1) setCutSegments(createInitialCutSegments());
-    else setCutSegments(newSegments);
-  }, [apparentCutSegments, createInitialCutSegments, duration, isSegmentSelected, setCutSegments]);
+    });
+  }, [modifySelectedSegmentTimes]);
+
+  const alignSegmentTimesToKeyframes = useCallback(async () => {
+    if (!mainVideoStream || workingRef.current) return;
+    try {
+      const response = await askForAlignSegments();
+      if (response == null) return;
+      setWorking(i18n.t('Aligning segments to keyframes'));
+      const { mode, startOrEnd } = response;
+      await modifySelectedSegmentTimes(async (segment) => {
+        const newSegment = { ...segment };
+
+        async function align(key) {
+          const time = newSegment[key];
+          const keyframe = await findKeyframeNearTime({ filePath, streamIndex: mainVideoStream.index, time, mode });
+          if (!keyframe == null) throw new Error(`Cannot find any keyframe within 60 seconds of frame ${time}`);
+          newSegment[key] = keyframe;
+        }
+        if (startOrEnd.includes('start')) await align('start');
+        if (startOrEnd.includes('end')) await align('end');
+        return newSegment;
+      });
+    } catch (err) {
+      handleError(err);
+    } finally {
+      setWorking();
+    }
+  }, [filePath, mainVideoStream, modifySelectedSegmentTimes, setWorking]);
 
   const maxLabelLength = safeOutputFileName ? 100 : 500;
 
@@ -2395,6 +2435,7 @@ const App = memo(() => {
       detectSceneChanges,
       createSegmentsFromKeyframes,
       shiftAllSegmentTimes,
+      alignSegmentTimesToKeyframes,
     };
 
     const actionsWithCatch = Object.entries(actions).map(([key, action]) => [
@@ -2410,7 +2451,7 @@ const App = memo(() => {
 
     actionsWithCatch.forEach(([key, action]) => electron.ipcRenderer.on(key, action));
     return () => actionsWithCatch.forEach(([key, action]) => electron.ipcRenderer.removeListener(key, action));
-  }, [apparentCutSegments, askSetStartTimeOffset, checkFileOpened, clearSegments, closeBatch, closeFileWithConfirm, combineOverlappingSegments, concatCurrentBatch, createFixedDurationSegments, createNumSegments, createRandomSegments, createSegmentsFromKeyframes, customOutDir, cutSegments, detectBlackScenes, detectSceneChanges, detectSilentScenes, detectedFps, extractAllStreams, fileFormat, filePath, fillSegmentsGaps, getFrameCount, invertAllSegments, loadCutSegments, loadMedia, openFilesDialog, openSendReportDialogWithState, reorderSegsByStartTime, setWorking, shiftAllSegmentTimes, shuffleSegments, toggleKeyboardShortcuts, toggleLastCommands, toggleSettings, tryFixInvalidDuration, userHtml5ifyCurrentFile, userOpenFiles]);
+  }, [alignSegmentTimesToKeyframes, apparentCutSegments, askSetStartTimeOffset, checkFileOpened, clearSegments, closeBatch, closeFileWithConfirm, combineOverlappingSegments, concatCurrentBatch, createFixedDurationSegments, createNumSegments, createRandomSegments, createSegmentsFromKeyframes, customOutDir, cutSegments, detectBlackScenes, detectSceneChanges, detectSilentScenes, detectedFps, extractAllStreams, fileFormat, filePath, fillSegmentsGaps, getFrameCount, invertAllSegments, loadCutSegments, loadMedia, openFilesDialog, openSendReportDialogWithState, reorderSegsByStartTime, setWorking, shiftAllSegmentTimes, shuffleSegments, toggleKeyboardShortcuts, toggleLastCommands, toggleSettings, tryFixInvalidDuration, userHtml5ifyCurrentFile, userOpenFiles]);
 
   const showAddStreamSourceDialog = useCallback(async () => {
     try {

src/dialogs/index.jsx

@@ -223,7 +223,7 @@ async function askForSegmentsRandomDurationRange() {
   return parse(value);
 }
 
-async function askForShiftSegmentsVariant(time) {
+async function askForSegmentsStartOrEnd(text) {
   const { value } = await Swal.fire({
     input: 'radio',
     showCancelButton: true,
@@ -233,9 +233,11 @@ async function askForShiftSegmentsVariant(time) {
       both: i18n.t('Both'),
     },
     inputValue: 'both',
-    text: i18n.t('Do you want to shift the start or end timestamp by {{time}}?', { time: formatDuration({ seconds: time, shorten: true }) }),
+    text,
   });
-  return value;
+  if (!value) return undefined;
+
+  return value === 'both' ? ['start', 'end'] : [value];
 }
 
 export async function askForShiftSegments() {
@@ -268,12 +270,37 @@ export async function askForShiftSegments() {
   if (value == null) return undefined;
   const parsed = parseValue(value);
 
-  const shiftVariant = await askForShiftSegmentsVariant(parsed);
-  if (shiftVariant == null) return undefined;
+  const startOrEnd = await askForSegmentsStartOrEnd(i18n.t('Do you want to shift the start or end timestamp by {{time}}?', { time: formatDuration({ seconds: parsed, shorten: true }) }));
+  if (startOrEnd == null) return undefined;
 
   return {
     shiftAmount: parsed,
-    shiftValues: shiftVariant === 'both' ? ['start', 'end'] : [shiftVariant],
+    shiftKeys: startOrEnd,
+  };
+}
+
+
+export async function askForAlignSegments() {
+  const startOrEnd = await askForSegmentsStartOrEnd(i18n.t('Do you want to align the segment start or end timestamps to keyframes?'));
+  if (startOrEnd == null) return undefined;
+
+  const { value: mode } = await Swal.fire({
+    input: 'radio',
+    showCancelButton: true,
+    inputOptions: {
+      nearest: i18n.t('Nearest keyframe'),
+      before: i18n.t('Previous keyframe'),
+      after: i18n.t('Next keyframe'),
+    },
+    inputValue: 'before',
+    text: i18n.t('Do you want to align segment times to the nearest, previous or next keyframe?'),
+  });
+
+  if (mode == null) return undefined;
+
+  return {
+    mode,
+    startOrEnd,
   };
 }
 

src/ffmpeg.js

@@ -3,6 +3,7 @@ import sortBy from 'lodash/sortBy';
 import moment from 'moment';
 import i18n from 'i18next';
 import Timecode from 'smpte-timecode';
+import minBy from 'lodash/minBy';
 
 import { pcmAudioCodecs, getMapStreamsArgs, isMov } from './util/streams';
 import { getSuffixedOutPath, isWindows, isMac, platform, arch, isExecaFailure } from './util';
@@ -153,6 +154,39 @@ export async function readFramesAroundTime({ filePath, streamIndex, aroundTime,
   return readFrames({ filePath, from, to, streamIndex });
 }
 
+export async function readKeyframesAroundTime({ filePath, streamIndex, aroundTime, window }) {
+  const frames = await readFramesAroundTime({ filePath, aroundTime, streamIndex, window });
+  return frames.filter((frame) => frame.keyframe);
+}
+
+export const findKeyframeAtExactTime = (keyframes, time) => keyframes.find((keyframe) => Math.abs(keyframe.time - time) < 0.000001);
+export const findNextKeyframe = (keyframes, time) => keyframes.find((keyframe) => keyframe.time >= time); // (assume they are already sorted)
+const findPreviousKeyframe = (keyframes, time) => keyframes.findLast((keyframe) => keyframe.time <= time);
+const findNearestKeyframe = (keyframes, time) => minBy(keyframes, (keyframe) => Math.abs(keyframe.time - time));
+
+function findKeyframe(keyframes, time, mode) {
+  switch (mode) {
+    case 'nearest': return findNearestKeyframe(keyframes, time);
+    case 'before': return findPreviousKeyframe(keyframes, time);
+    case 'after': return findNextKeyframe(keyframes, time);
+    default: return undefined;
+  }
+}
+
+export async function findKeyframeNearTime({ filePath, streamIndex, time, mode }) {
+  let keyframes = await readKeyframesAroundTime({ filePath, streamIndex, aroundTime: time, window: 10 });
+  let nearByKeyframe = findKeyframe(keyframes, time, mode);
+
+  if (!nearByKeyframe) {
+    keyframes = await readKeyframesAroundTime({ filePath, streamIndex, aroundTime: time, window: 60 });
+    nearByKeyframe = findKeyframe(keyframes, time, mode);
+  }
+
+  if (!nearByKeyframe) return undefined;
+  return nearByKeyframe.time;
+}
+
+// todo this is not in use
 // https://stackoverflow.com/questions/14005110/how-to-split-a-video-using-ffmpeg-so-that-each-chunk-starts-with-a-key-frame
 // http://kicherer.org/joomla/index.php/de/blog/42-avcut-frame-accurate-video-cutting-with-only-small-quality-loss
 export function getSafeCutTime(frames, cutTime, nextMode) {

src/smartcut.js

@@ -1,6 +1,6 @@
 import { getRealVideoStreams, getVideoTimebase } from './util/streams';
 
-import { readFramesAroundTime } from './ffmpeg';
+import { readKeyframesAroundTime, findNextKeyframe, findKeyframeAtExactTime } from './ffmpeg';
 
 const { stat } = window.require('fs-extra');
 
@@ -12,14 +12,11 @@ export async function getSmartCutParams({ path, videoDuration, desiredCutFrom, s
 
   const videoStream = videoStreams[0];
 
-  async function readKeyframes(window) {
-    const frames = await readFramesAroundTime({ filePath: path, aroundTime: desiredCutFrom, streamIndex: videoStream.index, window });
-    return frames.filter((frame) => frame.keyframe);
-  }
+  const readKeyframes = async (window) => readKeyframesAroundTime({ filePath: path, streamIndex: videoStream.index, aroundTime: desiredCutFrom, window });
 
   let keyframes = await readKeyframes(10);
 
-  const keyframeAtExactTime = keyframes.find((keyframe) => Math.abs(keyframe.time - desiredCutFrom) < 0.000001);
+  const keyframeAtExactTime = findKeyframeAtExactTime(keyframes, desiredCutFrom);
   if (keyframeAtExactTime) {
     console.log('Start cut is already on exact keyframe', keyframeAtExactTime.time);
 
@@ -30,15 +27,14 @@ export async function getSmartCutParams({ path, videoDuration, desiredCutFrom, s
     };
   }
 
-  const findNextKeyframe = () => keyframes.find((keyframe) => keyframe.time > desiredCutFrom); // (they are already sorted)
-  let nextKeyframe = findNextKeyframe();
-  if (!nextKeyframe) {
-    console.log('Cannot find any keyframe after desired start cut point, trying with larger window');
+  let nextKeyframe = findNextKeyframe(keyframes, desiredCutFrom);
+
+  if (nextKeyframe == null) {
+    // try again with a larger window
     keyframes = await readKeyframes(60);
-    nextKeyframe = findNextKeyframe();
+    nextKeyframe = findNextKeyframe(keyframes, desiredCutFrom);
   }
-
-  if (!nextKeyframe) throw new Error('Cannot find any keyframe after desired start cut point');
+  if (nextKeyframe == null) throw new Error('Cannot find any keyframe after the desired start cut point');
 
   console.log('Smart cut from keyframe', { keyframe: nextKeyframe.time, desiredCutFrom });