diff --git a/public/menu.js b/public/menu.js index 37ef2f42..30ea5b7b 100644 --- a/public/menu.js +++ b/public/menu.js @@ -246,6 +246,12 @@ module.exports = (app, mainWindow, newVersion) => { mainWindow.webContents.send('shiftAllSegmentTimes'); }, }, + { + label: i18n.t('Align segment times to keyframes'), + click() { + mainWindow.webContents.send('alignSegmentTimesToKeyframes'); + }, + }, ], }, { diff --git a/src/App.jsx b/src/App.jsx index a4d8387d..bbfcbc34 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -9,6 +9,7 @@ import { useDebounce } from 'use-debounce'; import i18n from 'i18next'; import { useTranslation } from 'react-i18next'; import JSON5 from 'json5'; +import pMap from 'p-map'; import fromPairs from 'lodash/fromPairs'; import sortBy from 'lodash/sortBy'; @@ -56,7 +57,7 @@ import { extractStreams, setCustomFfPath as ffmpegSetCustomFfPath, isIphoneHevc, isProblematicAvc1, tryMapChaptersToEdl, blackDetect, silenceDetect, detectSceneChanges as ffmpegDetectSceneChanges, getDuration, getTimecodeFromStreams, createChaptersFromSegments, extractSubtitleTrack, - RefuseOverwriteError, readFrames, mapTimesToSegments, abortFfmpegs, + RefuseOverwriteError, readFrames, mapTimesToSegments, abortFfmpegs, findKeyframeNearTime, } from './ffmpeg'; import { shouldCopyStreamByDefault, getAudioStreams, getRealVideoStreams, isAudioDefinitelyNotSupported, willPlayerProperlyHandleVideo, doesPlayerSupportHevcPlayback, isStreamThumbnail } from './util/streams'; import { exportEdlFile, readEdlFile, saveLlcProject, loadLlcProject, askForEdlImport } from './edlStore'; @@ -73,7 +74,7 @@ import { adjustRate } from './util/rate-calculator'; import { showParametersDialog } from './dialogs/parameters'; import { askExtractFramesAsImages } from './dialogs/extractFrames'; import { askForHtml5ifySpeed } from './dialogs/html5ify'; -import { askForOutDir, askForInputDir, askForImportChapters, createNumSegments as createNumSegmentsDialog, createFixedDurationSegments as createFixedDurationSegmentsDialog, createRandomSegments as createRandomSegmentsDialog, promptTimeOffset, askForFileOpenAction, confirmExtractAllStreamsDialog, showCleanupFilesDialog, showDiskFull, showExportFailedDialog, showConcatFailedDialog, labelSegmentDialog, openYouTubeChaptersDialog, openAbout, showEditableJsonDialog, askForShiftSegments, selectSegmentsByLabelDialog, showRefuseToOverwrite, openDirToast, openCutFinishedToast, openConcatFinishedToast } from './dialogs'; +import { askForOutDir, askForInputDir, askForImportChapters, createNumSegments as createNumSegmentsDialog, createFixedDurationSegments as createFixedDurationSegmentsDialog, createRandomSegments as createRandomSegmentsDialog, promptTimeOffset, askForFileOpenAction, confirmExtractAllStreamsDialog, showCleanupFilesDialog, showDiskFull, showExportFailedDialog, showConcatFailedDialog, labelSegmentDialog, openYouTubeChaptersDialog, openAbout, showEditableJsonDialog, askForShiftSegments, selectSegmentsByLabelDialog, showRefuseToOverwrite, openDirToast, openCutFinishedToast, openConcatFinishedToast, askForAlignSegments } from './dialogs'; import { openSendReportDialog } from './reporting'; import { fallbackLng } from './i18n'; import { createSegment, getCleanCutSegments, getSegApparentStart, getSegApparentEnd as getSegApparentEnd2, findSegmentsAtCursor, sortSegments, invertSegments, getSegmentTags, convertSegmentsToChapters, hasAnySegmentOverlap, combineOverlappingSegments as combineOverlappingSegments2, isDurationValid } from './segments'; @@ -442,22 +443,61 @@ const App = memo(() => { const isSegmentSelected = useCallback(({ segId }) => !deselectedSegmentIds[segId], [deselectedSegmentIds]); + const modifySelectedSegmentTimes = useCallback(async (transformSegment, concurrency = 5) => { + const clampValue = (val) => Math.min(Math.max(val, 0), duration); + + let newSegments = await pMap(apparentCutSegments, async (segment) => { + if (!isSegmentSelected(segment)) return segment; // pass thru non-selected segments + const newSegment = await transformSegment(segment); + newSegment.start = clampValue(newSegment.start); + newSegment.end = clampValue(newSegment.end); + return newSegment; + }, { concurrency }); + newSegments = newSegments.filter((segment) => segment.end > segment.start); + if (newSegments.length < 1) setCutSegments(createInitialCutSegments()); + else setCutSegments(newSegments); + }, [apparentCutSegments, createInitialCutSegments, duration, isSegmentSelected, setCutSegments]); + const shiftAllSegmentTimes = useCallback(async () => { const shift = await askForShiftSegments(); if (shift == null) return; - const { shiftAmount, shiftValues } = shift; - const clampValue = (val) => Math.min(Math.max(val, 0), duration); - const newSegments = apparentCutSegments.map((segment) => { - if (!isSegmentSelected(segment)) return segment; + + const { shiftAmount, shiftKeys } = shift; + await modifySelectedSegmentTimes((segment) => { const newSegment = { ...segment }; - shiftValues.forEach((key) => { - newSegment[key] = clampValue(segment[key] + shiftAmount); + shiftKeys.forEach((key) => { + newSegment[key] += shiftAmount; }); return newSegment; - }).filter((segment) => segment.end > segment.start); - if (newSegments.length < 1) setCutSegments(createInitialCutSegments()); - else setCutSegments(newSegments); - }, [apparentCutSegments, createInitialCutSegments, duration, isSegmentSelected, setCutSegments]); + }); + }, [modifySelectedSegmentTimes]); + + const alignSegmentTimesToKeyframes = useCallback(async () => { + if (!mainVideoStream || workingRef.current) return; + try { + const response = await askForAlignSegments(); + if (response == null) return; + setWorking(i18n.t('Aligning segments to keyframes')); + const { mode, startOrEnd } = response; + await modifySelectedSegmentTimes(async (segment) => { + const newSegment = { ...segment }; + + async function align(key) { + const time = newSegment[key]; + const keyframe = await findKeyframeNearTime({ filePath, streamIndex: mainVideoStream.index, time, mode }); + if (!keyframe == null) throw new Error(`Cannot find any keyframe within 60 seconds of frame ${time}`); + newSegment[key] = keyframe; + } + if (startOrEnd.includes('start')) await align('start'); + if (startOrEnd.includes('end')) await align('end'); + return newSegment; + }); + } catch (err) { + handleError(err); + } finally { + setWorking(); + } + }, [filePath, mainVideoStream, modifySelectedSegmentTimes, setWorking]); const maxLabelLength = safeOutputFileName ? 100 : 500; @@ -2395,6 +2435,7 @@ const App = memo(() => { detectSceneChanges, createSegmentsFromKeyframes, shiftAllSegmentTimes, + alignSegmentTimesToKeyframes, }; const actionsWithCatch = Object.entries(actions).map(([key, action]) => [ @@ -2410,7 +2451,7 @@ const App = memo(() => { actionsWithCatch.forEach(([key, action]) => electron.ipcRenderer.on(key, action)); return () => actionsWithCatch.forEach(([key, action]) => electron.ipcRenderer.removeListener(key, action)); - }, [apparentCutSegments, askSetStartTimeOffset, checkFileOpened, clearSegments, closeBatch, closeFileWithConfirm, combineOverlappingSegments, concatCurrentBatch, createFixedDurationSegments, createNumSegments, createRandomSegments, createSegmentsFromKeyframes, customOutDir, cutSegments, detectBlackScenes, detectSceneChanges, detectSilentScenes, detectedFps, extractAllStreams, fileFormat, filePath, fillSegmentsGaps, getFrameCount, invertAllSegments, loadCutSegments, loadMedia, openFilesDialog, openSendReportDialogWithState, reorderSegsByStartTime, setWorking, shiftAllSegmentTimes, shuffleSegments, toggleKeyboardShortcuts, toggleLastCommands, toggleSettings, tryFixInvalidDuration, userHtml5ifyCurrentFile, userOpenFiles]); + }, [alignSegmentTimesToKeyframes, apparentCutSegments, askSetStartTimeOffset, checkFileOpened, clearSegments, closeBatch, closeFileWithConfirm, combineOverlappingSegments, concatCurrentBatch, createFixedDurationSegments, createNumSegments, createRandomSegments, createSegmentsFromKeyframes, customOutDir, cutSegments, detectBlackScenes, detectSceneChanges, detectSilentScenes, detectedFps, extractAllStreams, fileFormat, filePath, fillSegmentsGaps, getFrameCount, invertAllSegments, loadCutSegments, loadMedia, openFilesDialog, openSendReportDialogWithState, reorderSegsByStartTime, setWorking, shiftAllSegmentTimes, shuffleSegments, toggleKeyboardShortcuts, toggleLastCommands, toggleSettings, tryFixInvalidDuration, userHtml5ifyCurrentFile, userOpenFiles]); const showAddStreamSourceDialog = useCallback(async () => { try { diff --git a/src/dialogs/index.jsx b/src/dialogs/index.jsx index 7c1f50f7..40038214 100644 --- a/src/dialogs/index.jsx +++ b/src/dialogs/index.jsx @@ -223,7 +223,7 @@ async function askForSegmentsRandomDurationRange() { return parse(value); } -async function askForShiftSegmentsVariant(time) { +async function askForSegmentsStartOrEnd(text) { const { value } = await Swal.fire({ input: 'radio', showCancelButton: true, @@ -233,9 +233,11 @@ async function askForShiftSegmentsVariant(time) { both: i18n.t('Both'), }, inputValue: 'both', - text: i18n.t('Do you want to shift the start or end timestamp by {{time}}?', { time: formatDuration({ seconds: time, shorten: true }) }), + text, }); - return value; + if (!value) return undefined; + + return value === 'both' ? ['start', 'end'] : [value]; } export async function askForShiftSegments() { @@ -268,12 +270,37 @@ export async function askForShiftSegments() { if (value == null) return undefined; const parsed = parseValue(value); - const shiftVariant = await askForShiftSegmentsVariant(parsed); - if (shiftVariant == null) return undefined; + const startOrEnd = await askForSegmentsStartOrEnd(i18n.t('Do you want to shift the start or end timestamp by {{time}}?', { time: formatDuration({ seconds: parsed, shorten: true }) })); + if (startOrEnd == null) return undefined; return { shiftAmount: parsed, - shiftValues: shiftVariant === 'both' ? ['start', 'end'] : [shiftVariant], + shiftKeys: startOrEnd, + }; +} + + +export async function askForAlignSegments() { + const startOrEnd = await askForSegmentsStartOrEnd(i18n.t('Do you want to align the segment start or end timestamps to keyframes?')); + if (startOrEnd == null) return undefined; + + const { value: mode } = await Swal.fire({ + input: 'radio', + showCancelButton: true, + inputOptions: { + nearest: i18n.t('Nearest keyframe'), + before: i18n.t('Previous keyframe'), + after: i18n.t('Next keyframe'), + }, + inputValue: 'before', + text: i18n.t('Do you want to align segment times to the nearest, previous or next keyframe?'), + }); + + if (mode == null) return undefined; + + return { + mode, + startOrEnd, }; } diff --git a/src/ffmpeg.js b/src/ffmpeg.js index a0ea22ca..bf863c59 100644 --- a/src/ffmpeg.js +++ b/src/ffmpeg.js @@ -3,6 +3,7 @@ import sortBy from 'lodash/sortBy'; import moment from 'moment'; import i18n from 'i18next'; import Timecode from 'smpte-timecode'; +import minBy from 'lodash/minBy'; import { pcmAudioCodecs, getMapStreamsArgs, isMov } from './util/streams'; import { getSuffixedOutPath, isWindows, isMac, platform, arch, isExecaFailure } from './util'; @@ -153,6 +154,39 @@ export async function readFramesAroundTime({ filePath, streamIndex, aroundTime, return readFrames({ filePath, from, to, streamIndex }); } +export async function readKeyframesAroundTime({ filePath, streamIndex, aroundTime, window }) { + const frames = await readFramesAroundTime({ filePath, aroundTime, streamIndex, window }); + return frames.filter((frame) => frame.keyframe); +} + +export const findKeyframeAtExactTime = (keyframes, time) => keyframes.find((keyframe) => Math.abs(keyframe.time - time) < 0.000001); +export const findNextKeyframe = (keyframes, time) => keyframes.find((keyframe) => keyframe.time >= time); // (assume they are already sorted) +const findPreviousKeyframe = (keyframes, time) => keyframes.findLast((keyframe) => keyframe.time <= time); +const findNearestKeyframe = (keyframes, time) => minBy(keyframes, (keyframe) => Math.abs(keyframe.time - time)); + +function findKeyframe(keyframes, time, mode) { + switch (mode) { + case 'nearest': return findNearestKeyframe(keyframes, time); + case 'before': return findPreviousKeyframe(keyframes, time); + case 'after': return findNextKeyframe(keyframes, time); + default: return undefined; + } +} + +export async function findKeyframeNearTime({ filePath, streamIndex, time, mode }) { + let keyframes = await readKeyframesAroundTime({ filePath, streamIndex, aroundTime: time, window: 10 }); + let nearByKeyframe = findKeyframe(keyframes, time, mode); + + if (!nearByKeyframe) { + keyframes = await readKeyframesAroundTime({ filePath, streamIndex, aroundTime: time, window: 60 }); + nearByKeyframe = findKeyframe(keyframes, time, mode); + } + + if (!nearByKeyframe) return undefined; + return nearByKeyframe.time; +} + +// todo this is not in use // https://stackoverflow.com/questions/14005110/how-to-split-a-video-using-ffmpeg-so-that-each-chunk-starts-with-a-key-frame // http://kicherer.org/joomla/index.php/de/blog/42-avcut-frame-accurate-video-cutting-with-only-small-quality-loss export function getSafeCutTime(frames, cutTime, nextMode) { diff --git a/src/smartcut.js b/src/smartcut.js index 7fa304be..54510705 100644 --- a/src/smartcut.js +++ b/src/smartcut.js @@ -1,6 +1,6 @@ import { getRealVideoStreams, getVideoTimebase } from './util/streams'; -import { readFramesAroundTime } from './ffmpeg'; +import { readKeyframesAroundTime, findNextKeyframe, findKeyframeAtExactTime } from './ffmpeg'; const { stat } = window.require('fs-extra'); @@ -12,14 +12,11 @@ export async function getSmartCutParams({ path, videoDuration, desiredCutFrom, s const videoStream = videoStreams[0]; - async function readKeyframes(window) { - const frames = await readFramesAroundTime({ filePath: path, aroundTime: desiredCutFrom, streamIndex: videoStream.index, window }); - return frames.filter((frame) => frame.keyframe); - } + const readKeyframes = async (window) => readKeyframesAroundTime({ filePath: path, streamIndex: videoStream.index, aroundTime: desiredCutFrom, window }); let keyframes = await readKeyframes(10); - const keyframeAtExactTime = keyframes.find((keyframe) => Math.abs(keyframe.time - desiredCutFrom) < 0.000001); + const keyframeAtExactTime = findKeyframeAtExactTime(keyframes, desiredCutFrom); if (keyframeAtExactTime) { console.log('Start cut is already on exact keyframe', keyframeAtExactTime.time); @@ -30,15 +27,14 @@ export async function getSmartCutParams({ path, videoDuration, desiredCutFrom, s }; } - const findNextKeyframe = () => keyframes.find((keyframe) => keyframe.time > desiredCutFrom); // (they are already sorted) - let nextKeyframe = findNextKeyframe(); - if (!nextKeyframe) { - console.log('Cannot find any keyframe after desired start cut point, trying with larger window'); + let nextKeyframe = findNextKeyframe(keyframes, desiredCutFrom); + + if (nextKeyframe == null) { + // try again with a larger window keyframes = await readKeyframes(60); - nextKeyframe = findNextKeyframe(); + nextKeyframe = findNextKeyframe(keyframes, desiredCutFrom); } - - if (!nextKeyframe) throw new Error('Cannot find any keyframe after desired start cut point'); + if (nextKeyframe == null) throw new Error('Cannot find any keyframe after the desired start cut point'); console.log('Smart cut from keyframe', { keyframe: nextKeyframe.time, desiredCutFrom });