From d91e9b1c4571ed7111aae1d5386b13c1d467020f Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Sat, 22 Feb 2020 23:44:21 +0800 Subject: [PATCH] improvements: - show keyframes on timeline - add help to help menu - improve buttons - round all seeking to frame time - bugfixes --- src/HelpSheet.jsx | 6 +- src/SegmentList.jsx | 22 ++--- src/StreamsSelector.jsx | 2 +- src/ffmpeg.js | 94 ++++++++++++++---- src/menu.js | 9 +- src/renderer.jsx | 206 ++++++++++++++++++++++++++-------------- 6 files changed, 233 insertions(+), 106 deletions(-) diff --git a/src/HelpSheet.jsx b/src/HelpSheet.jsx index 5a6d658f..f5aeb47a 100644 --- a/src/HelpSheet.jsx +++ b/src/HelpSheet.jsx @@ -57,9 +57,9 @@ const HelpSheet = memo(({

Last ffmpeg commands

- {ffmpegCommandLog.reverse().map((log) => ( -
- { clipboard.writeText(log); toast.fire({ timer: 2000, icon: 'success', title: 'Copied to clipboard' }); }} /> {log} + {ffmpegCommandLog.reverse().map(({ command, time }) => ( +
+ { clipboard.writeText(command); toast.fire({ timer: 2000, icon: 'success', title: 'Copied to clipboard' }); }} /> {command}
))}
diff --git a/src/SegmentList.jsx b/src/SegmentList.jsx index 8302b321..739cfac8 100644 --- a/src/SegmentList.jsx +++ b/src/SegmentList.jsx @@ -1,6 +1,6 @@ import React, { memo, Fragment } from 'react'; import prettyMs from 'pretty-ms'; -import { FaSave, FaPlus, FaMinus, FaTag } from 'react-icons/fa'; +import { FaSave, FaPlus, FaMinus, FaTag, FaSortNumericDown } from 'react-icons/fa'; import { motion } from 'framer-motion'; import Swal from 'sweetalert2'; @@ -20,10 +20,7 @@ const SegmentList = memo(({ return
No segments to export.
; } - const { - segActiveBgColor: currentSegActiveBgColor, - segBorderColor: currentSegBorderColor, - } = getSegColors(currentCutSeg); + const { segActiveBgColor: currentSegActiveBgColor } = getSegColors(currentCutSeg); async function onLabelSegmentPress() { const { value } = await Swal.fire({ @@ -110,7 +107,7 @@ const SegmentList = memo(({
-
- {currentSegIndex + 1} -
+ /> { {!Number.isNaN(duration) && `${formatDuration({ seconds: duration })}`} {stream.nb_frames} {!Number.isNaN(bitrate) && `${(bitrate / 1e6).toFixed(1)}MBit/s`} - {stream.width && stream.height && `${stream.width}x${stream.height}`} {stream.channels && `${stream.channels}c`} {stream.channel_layout} {streamFps && `${streamFps.toFixed(1)}fps`} + {stream.width && stream.height && `${stream.width}x${stream.height}`} {stream.channels && `${stream.channels}c`} {stream.channel_layout} {streamFps && `${streamFps.toFixed(2)}fps`} onInfoClick(stream)} size={26} /> ); diff --git a/src/ffmpeg.js b/src/ffmpeg.js index f91f2fc8..303e3c62 100644 --- a/src/ffmpeg.js +++ b/src/ffmpeg.js @@ -16,6 +16,12 @@ const os = require('os'); const { formatDuration, getOutPath, transferTimestamps } = require('./util'); + +function getFfCommandLine(cmd, args) { + const mapArg = arg => (/[^0-9a-zA-Z-_]/.test(arg) ? `'${arg}'` : arg); + return `${cmd} ${args.map(mapArg).join(' ')}`; +} + function getPath(type) { const platform = os.platform(); @@ -36,11 +42,13 @@ function getPath(type) { async function runFfprobe(args) { const ffprobePath = await getPath('ffprobe'); + console.log(getFfCommandLine('ffprobe', args)); return execa(ffprobePath, args); } async function runFfmpeg(args) { const ffmpegPath = await getPath('ffmpeg'); + console.log(getFfCommandLine('ffmpeg', args)); return execa(ffmpegPath, args); } @@ -80,22 +88,79 @@ function getExtensionForFormat(format) { return ext || format; } +async function readFrames({ filePath, aroundTime, window = 30, stream }) { + const intervalsArgs = aroundTime != null ? ['-read_intervals', `${Math.max(aroundTime - window, 0)}%${aroundTime + window}`] : []; + const { stdout } = await runFfprobe(['-v', 'error', ...intervalsArgs, '-show_packets', '-select_streams', stream, '-show_entries', 'packet=pts_time,flags', '-of', 'json', filePath]); + return sortBy(JSON.parse(stdout).packets.map(p => ({ keyframe: p.flags[0] === 'K', time: parseFloat(p.pts_time, 10) })), 'time'); +} + +// https://stackoverflow.com/questions/14005110/how-to-split-a-video-using-ffmpeg-so-that-each-chunk-starts-with-a-key-frame +// http://kicherer.org/joomla/index.php/de/blog/42-avcut-frame-accurate-video-cutting-with-only-small-quality-loss +function getNextPrevKeyframe(frames, cutTime, nextMode) { + const sigma = 0.01; + const isCloseTo = (time1, time2) => Math.abs(time1 - time2) < sigma; + + let index; + + if (frames.length < 2) throw new Error('Less than 2 frames found'); + + if (nextMode) { + index = frames.findIndex(f => f.keyframe && f.time >= cutTime - sigma); + if (index === -1) throw new Error('Failed to find next keyframe'); + if (index >= frames.length - 1) throw new Error('We are on the last frame'); + const { time } = frames[index]; + if (isCloseTo(time, cutTime)) { + return undefined; // Already on keyframe, no need to modify cut time + } + return time; + } + + const findReverseIndex = (arr, cb) => { + const ret = [...arr].reverse().findIndex(cb); + if (ret === -1) return -1; + return arr.length - 1 - ret; + }; + + index = findReverseIndex(frames, f => f.time <= cutTime + sigma); + if (index === -1) throw new Error('Failed to find any prev frame'); + if (index === 0) throw new Error('We are on the first frame'); + + if (index === frames.length - 1) { + // Last frame of video, no need to modify cut time + return undefined; + } + if (frames[index + 1].keyframe) { + // Already on frame before keyframe, no need to modify cut time + return undefined; + } + + // We are not on a frame before keyframe, look for preceding keyframe instead + index = findReverseIndex(frames, f => f.keyframe && f.time <= cutTime + sigma); + if (index === -1) throw new Error('Failed to find any prev keyframe'); + if (index === 0) throw new Error('We are on the first keyframe'); + + // Use frame before the found keyframe + return frames[index - 1].time; +} + async function cut({ filePath, outFormat, cutFrom, cutTo, videoDuration, rotation, onProgress, copyStreamIds, keyframeCut, outPath, appendFfmpegCommandLog, }) { console.log('Cutting from', cutFrom, 'to', cutTo); + const ssBeforeInput = keyframeCut; + const cutDuration = cutTo - cutFrom; - // https://github.com/mifi/lossless-cut/issues/50 - const cutFromArgs = isCuttingStart(cutFrom) ? ['-ss', cutFrom] : []; - const cutToArgs = isCuttingEnd(cutTo, videoDuration) ? ['-t', cutDuration] : []; + // Don't cut if no need: https://github.com/mifi/lossless-cut/issues/50 + const cutFromArgs = isCuttingStart(cutFrom) ? ['-ss', cutFrom.toFixed(5)] : []; + const cutToArgs = isCuttingEnd(cutTo, videoDuration) ? ['-t', cutDuration.toFixed(5)] : []; const copyStreamIdsFiltered = copyStreamIds.filter(({ streamIds }) => streamIds.length > 0); const inputArgs = flatMap(copyStreamIdsFiltered, ({ path }) => ['-i', path]); - const inputCutArgs = keyframeCut ? [ + const inputCutArgs = ssBeforeInput ? [ ...cutFromArgs, ...inputArgs, ...cutToArgs, @@ -126,11 +191,10 @@ async function cut({ '-f', outFormat, '-y', outPath, ]; - const mapArg = arg => (/[^0-9a-zA-Z-_]/.test(arg) ? `'${arg}'` : arg); - const ffmpegCommand = `ffmpeg ${ffmpegArgs.map(mapArg).join(' ')}`; + const ffmpegCommandLine = getFfCommandLine('ffmpeg', ffmpegArgs); - console.log(ffmpegCommand); - appendFfmpegCommandLog(ffmpegCommand); + console.log(ffmpegCommandLine); + appendFfmpegCommandLog(ffmpegCommandLine); onProgress(0); @@ -204,17 +268,15 @@ async function html5ify(filePath, outPath, encodeVideo, encodeAudio) { '-y', outPath, ]; - console.log('ffmpeg', ffmpegArgs.join(' ')); - const { stdout } = await runFfmpeg(ffmpegArgs); console.log(stdout); await transferTimestamps(filePath, outPath); } -async function getDuration(filePpath) { +async function getDuration(filePath) { // https://superuser.com/questions/650291/how-to-get-video-duration-in-seconds - const { stdout } = await runFfprobe(['-i', filePpath, '-show_entries', 'format=duration', '-print_format', 'json']); + const { stdout } = await runFfprobe(['-i', filePath, '-show_entries', 'format=duration', '-print_format', 'json']); return parseFloat(JSON.parse(stdout).format.duration); } @@ -234,8 +296,6 @@ async function html5ifyDummy(filePath, outPath) { '-y', outPath, ]; - console.log('ffmpeg', ffmpegArgs.join(' ')); - const { stdout } = await runFfmpeg(ffmpegArgs); console.log(stdout); @@ -389,8 +449,6 @@ async function extractStreams({ filePath, customOutDir, streams }) { ...streamArgs, ]; - console.log(ffmpegArgs); - // TODO progress const { stdout } = await runFfmpeg(ffmpegArgs); console.log(stdout); @@ -418,7 +476,7 @@ async function renderFrame(timestamp, filePath, rotation) { // console.time('ffmpeg'); const ffmpegPath = await getPath('ffmpeg'); // console.timeEnd('ffmpeg'); - console.log('ffmpeg', args); + // console.log('ffmpeg', args); const { stdout } = await execa(ffmpegPath, args, { encoding: null }); const blob = new Blob([stdout], { type: 'image/jpeg' }); @@ -458,4 +516,6 @@ module.exports = { getStreamFps, isCuttingStart, isCuttingEnd, + readFrames, + getNextPrevKeyframe, }; diff --git a/src/menu.js b/src/menu.js index f12d066d..a788f4f7 100644 --- a/src/menu.js +++ b/src/menu.js @@ -123,12 +123,17 @@ module.exports = (app, mainWindow, newVersion) => { { role: 'help', submenu: [ + { + label: 'Help and Settings', + click() { + mainWindow.webContents.send('openHelp'); + }, + }, + { label: 'Learn More', click() { electron.shell.openExternal(homepage); }, }, - - ...(process.platform === 'darwin' ? [] : [{ role: 'about' }]), ], }); } diff --git a/src/renderer.jsx b/src/renderer.jsx index 938afd05..f8e594c3 100644 --- a/src/renderer.jsx +++ b/src/renderer.jsx @@ -6,8 +6,9 @@ import { FiScissors } from 'react-icons/fi'; import { AnimatePresence, motion } from 'framer-motion'; import Swal from 'sweetalert2'; import Lottie from 'react-lottie'; -import { SideSheet, Button, Position, Table, SegmentedControl, Checkbox } from 'evergreen-ui'; +import { SideSheet, Button, Position, Table, SegmentedControl, Checkbox, Select } from 'evergreen-ui'; import { useStateWithHistory } from 'react-use/lib/useStateWithHistory'; +import useDebounce from 'react-use/lib/useDebounce'; import fromPairs from 'lodash/fromPairs'; import clamp from 'lodash/clamp'; @@ -106,12 +107,15 @@ const App = memo(() => { const [filePath, setFilePath] = useState(''); const [externalStreamFiles, setExternalStreamFiles] = useState([]); const [detectedFps, setDetectedFps] = useState(); - const [mainStreams, setStreams] = useState([]); + const [mainStreams, setMainStreams] = useState([]); + const [mainVideoStream, setMainVideoStream] = useState(); const [copyStreamIdsByFile, setCopyStreamIdsByFile] = useState({}); const [streamsSelectorShown, setStreamsSelectorShown] = useState(false); const [zoom, setZoom] = useState(1); const [commandedTime, setCommandedTime] = useState(0); + const [debouncedCommandedTime, setDebouncedCommandedTime] = useState(0); const [ffmpegCommandLog, setFfmpegCommandLog] = useState([]); + const [neighbouringFrames, setNeighbouringFrames] = useState([]); // Segment related state const [currentSegIndex, setCurrentSegIndex] = useState(0); @@ -122,6 +126,10 @@ const App = memo(() => { 100, ); + const [, cancelCommandedTimeDebounce] = useDebounce(() => { + setDebouncedCommandedTime(commandedTime); + }, 300, [commandedTime]); + // Preferences const [captureFormat, setCaptureFormat] = useState(configStore.get('captureFormat')); @@ -154,10 +162,11 @@ const App = memo(() => { const timelineScrollerRef = useRef(); const timelineScrollerSkipEventRef = useRef(); const lastSavedCutSegmentsRef = useRef(); + const readingKeyframesPromise = useRef(); function appendFfmpegCommandLog(command) { - setFfmpegCommandLog(old => [...old, command]); + setFfmpegCommandLog(old => [...old, { command, time: new Date() }]); } function setCopyStreamIdsForPath(path, cb) { @@ -178,21 +187,23 @@ const App = memo(() => { }); } - function seekAbs(val) { + const seekAbs = useCallback((val) => { const video = videoRef.current; if (val == null || Number.isNaN(val)) return; + let valRounded = val; + if (detectedFps) valRounded = Math.round(detectedFps * val) / detectedFps; // Round to nearest frame - let outVal = val; + let outVal = valRounded; if (outVal < 0) outVal = 0; if (outVal > video.duration) outVal = video.duration; video.currentTime = outVal; setCommandedTime(outVal); - } + }, [detectedFps]); const seekRel = useCallback((val) => { seekAbs(videoRef.current.currentTime + val); - }, []); + }, [seekAbs]); const shortStep = useCallback((dir) => { seekRel((1 / (detectedFps || 60)) * dir); @@ -200,6 +211,8 @@ const App = memo(() => { const resetState = useCallback(() => { const video = videoRef.current; + cancelCommandedTimeDebounce(); + setDebouncedCommandedTime(0); setCommandedTime(0); video.currentTime = 0; video.playbackRate = 1; @@ -224,11 +237,12 @@ const App = memo(() => { setFilePath(''); // Setting video src="" prevents memory leak in chromium setExternalStreamFiles([]); setDetectedFps(); - setStreams([]); + setMainStreams([]); setCopyStreamIdsByFile({}); setStreamsSelectorShown(false); setZoom(1); - }, [cutSegmentsHistory, setCutSegments]); + setNeighbouringFrames([]); + }, [cutSegmentsHistory, setCutSegments, cancelCommandedTimeDebounce]); useEffect(() => () => { if (dummyVideoPath) unlink(dummyVideoPath).catch(console.error); @@ -361,39 +375,58 @@ const App = memo(() => { const getCurrentTime = useCallback(() => ( playing ? playerTime : commandedTime), [commandedTime, playerTime, playing]); - const addCutSegment = useCallback(() => { - const cutStartTime = currentCutSeg.start; - const cutEndTime = currentCutSeg.end; + // const getNextPrevKeyframe = useCallback((cutTime, next) => ffmpeg.getNextPrevKeyframe(neighbouringFrames, cutTime, next), [neighbouringFrames]); - if (cutStartTime === undefined && cutEndTime === undefined) return; - - const suggestedStart = getCurrentTime(); - const suggestedEnd = suggestedStart + 10; - - const cutSegmentsNew = [ - ...cutSegments, - createSegment({ - start: suggestedStart, - end: suggestedEnd <= duration ? suggestedEnd : undefined, - }), - ]; - - setCutSegments(cutSegmentsNew); - setCurrentSegIndex(cutSegmentsNew.length - 1); + const addCutSegment = useCallback(() => { + try { + // Cannot add if prev seg is not finished + if (currentCutSeg.start === undefined && currentCutSeg.end === undefined) return; + + const suggestedStart = getCurrentTime(); + /* if (keyframeCut) { + const keyframeAlignedStart = getNextPrevKeyframe(suggestedStart, true); + if (keyframeAlignedStart != null) suggestedStart = keyframeAlignedStart; + } */ + + let suggestedEnd = suggestedStart + 10; + if (suggestedEnd >= duration) { + suggestedEnd = undefined; + } /* else if (keyframeCut) { + const keyframeAlignedEnd = getNextPrevKeyframe(suggestedEnd, false); + if (keyframeAlignedEnd != null) suggestedEnd = keyframeAlignedEnd; + } */ + + const cutSegmentsNew = [ + ...cutSegments, + createSegment({ + start: suggestedStart, + end: suggestedEnd, + }), + ]; + + setCutSegments(cutSegmentsNew); + setCurrentSegIndex(cutSegmentsNew.length - 1); + } catch (err) { + console.error(err); + } }, [ - currentCutSeg, cutSegments, getCurrentTime, duration, setCutSegments, + currentCutSeg.start, currentCutSeg.end, cutSegments, getCurrentTime, duration, setCutSegments, ]); const setCutStart = useCallback(() => { // https://github.com/mifi/lossless-cut/issues/168 // If we are after the end of the last segment in the timeline, // add a new segment that starts at playerTime - if (currentCutSeg.end != null - && getCurrentTime() > currentCutSeg.end) { + if (currentCutSeg.end != null && getCurrentTime() > currentCutSeg.end) { addCutSegment(); } else { try { - setCutTime('start', getCurrentTime()); + const startTime = getCurrentTime(); + /* if (keyframeCut) { + const keyframeAlignedCutTo = getNextPrevKeyframe(startTime, true); + if (keyframeAlignedCutTo != null) startTime = keyframeAlignedCutTo; + } */ + setCutTime('start', startTime); } catch (err) { errorToast(err.message); } @@ -402,7 +435,13 @@ const App = memo(() => { const setCutEnd = useCallback(() => { try { - setCutTime('end', getCurrentTime()); + const endTime = getCurrentTime(); + + /* if (keyframeCut) { + const keyframeAlignedCutTo = getNextPrevKeyframe(endTime, false); + if (keyframeAlignedCutTo != null) endTime = keyframeAlignedCutTo; + } */ + setCutTime('end', endTime); } catch (err) { errorToast(err.message); } @@ -840,19 +879,17 @@ const App = memo(() => { const { streams } = await ffmpeg.getAllStreams(fp); // console.log('streams', streamsNew); - setStreams(streams); + setMainStreams(streams); setCopyStreamIdsForPath(fp, () => fromPairs(streams.map((stream) => [ stream.index, defaultProcessedCodecTypes.includes(stream.codec_type), ]))); - streams.find((stream) => { - const streamFps = getStreamFps(stream); - if (streamFps != null) { - setDetectedFps(streamFps); - return true; - } - return false; - }); + const videoStream = streams.find(stream => stream.codec_type === 'video'); + setMainVideoStream(videoStream); + if (videoStream) { + const streamFps = getStreamFps(videoStream); + if (streamFps != null) setDetectedFps(streamFps); + } setFileNameTitle(fp); setFilePath(fp); @@ -1099,6 +1136,10 @@ const App = memo(() => { await loadEdlFile(filePaths[0]); } + function openHelp() { + toggleHelp(); + } + electron.ipcRenderer.on('file-opened', fileOpened); electron.ipcRenderer.on('close-file', closeFile); electron.ipcRenderer.on('html5ify', html5ify); @@ -1109,6 +1150,7 @@ const App = memo(() => { electron.ipcRenderer.on('redo', redo); electron.ipcRenderer.on('importEdlFile', importEdlFile); electron.ipcRenderer.on('exportEdlFile', exportEdlFile); + electron.ipcRenderer.on('openHelp', openHelp); return () => { electron.ipcRenderer.removeListener('file-opened', fileOpened); @@ -1121,6 +1163,7 @@ const App = memo(() => { electron.ipcRenderer.removeListener('redo', redo); electron.ipcRenderer.removeListener('importEdlFile', importEdlFile); electron.ipcRenderer.removeListener('exportEdlFile', exportEdlFile); + electron.ipcRenderer.removeListener('openHelp', openHelp); }; }, [ load, mergeFiles, outputDir, filePath, customOutDir, startTimeOffset, getHtml5ifiedPath, @@ -1215,9 +1258,10 @@ const App = memo(() => { )); } - function renderOutFmt(style = {}) { + function renderOutFmt(props) { return ( - setFileFormat(e.target.value))} {...props}> {detectedFileFormat && ( @@ -1231,19 +1275,20 @@ const App = memo(() => { {renderFormatOptions(otherFormatsMap)} - + ); } - function renderCaptureFormatButton() { + function renderCaptureFormatButton(props) { return ( - + ); } @@ -1266,12 +1311,9 @@ const App = memo(() => { This is where working files, exported files, project files (CSV) are stored. - +
{customOutDir}
 @@ -1290,8 +1332,8 @@ const App = memo(() => { Keyframe cut mode
- Nearest keyframe: Cut at the nearest keyframe (not accurate time.)
- Normal cut: Accurate time but could leave an empty portion at the beginning of the video.
+ Nearest keyframe: Cut at the nearest keyframe (not accurate time.) Equiv to ffmpeg -ss -i ...
+ Normal cut: Accurate time but could leave an empty portion at the beginning of the video. Equiv to ffmpeg -i -ss ...
 { // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + useEffect(() => { + async function run() { + if (!filePath || debouncedCommandedTime == null || !mainVideoStream || readingKeyframesPromise.current) return; + try { + const promise = ffmpeg.readFrames({ filePath, aroundTime: debouncedCommandedTime, stream: mainVideoStream.index }); + readingKeyframesPromise.current = promise; + const newFrames = await promise; + // console.log(newFrames); + setNeighbouringFrames(newFrames); + } catch (err) { + console.error('Failed to read keyframes', err); + } finally { + readingKeyframesPromise.current = undefined; + } + } + run(); + }, [filePath, debouncedCommandedTime, mainVideoStream]); + const topBarHeight = '2rem'; const bottomBarHeight = '6rem'; @@ -1513,32 +1573,34 @@ const App = memo(() => { {filePath && ( - + - {renderOutFmt({ width: 60 })} +
{renderOutFmt({ height: 20 })}
- + - + )} @@ -1718,6 +1780,10 @@ const App = memo(() => { invertCutSegments={invertCutSegments} /> ))} + + {mainVideoStream && neighbouringFrames.filter(f => f.keyframe).map((f) => ( +
+ ))}
@@ -1802,14 +1868,14 @@ const App = memo(() => {
{renderInvertCutButton()} - setZoom(parseInt(e.target.value, 10)))}> {Array(10).fill().map((unused, z) => { const val = 2 ** z; return ( ); })} - +
@@ -1832,7 +1898,7 @@ const App = memo(() => { role="button" /> - {renderCaptureFormatButton()} + {renderCaptureFormatButton({ height: 20 })}