diff --git a/package.json b/package.json index 2531d0c7..192a3b77 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,6 @@ "moment": "^2.18.1", "mousetrap": "^1.6.1", "p-map": "^3.0.0", - "p-queue": "^6.2.0", "patch-package": "^6.2.1", "pretty-ms": "^6.0.0", "react": "^16.12.0", @@ -96,6 +95,7 @@ "read-chunk": "^2.0.0", "semver": "^7.1.3", "string-to-stream": "^1.1.1", + "strtok3": "^6.0.0", "trash": "^6.1.1" }, "eslintConfig": { diff --git a/src/App.jsx b/src/App.jsx index a79e5aa3..dfde6caa 100644 --- a/src/App.jsx +++ b/src/App.jsx @@ -6,7 +6,6 @@ import Lottie from 'react-lottie'; import { SideSheet, Button, Position, SegmentedControl, Select } from 'evergreen-ui'; import { useStateWithHistory } from 'react-use/lib/useStateWithHistory'; import useDebounce from 'react-use/lib/useDebounce'; -import PQueue from 'p-queue'; import filePathToUrl from 'file-url'; import Mousetrap from 'mousetrap'; import uuid from 'uuid'; @@ -22,6 +21,7 @@ import flatMap from 'lodash/flatMap'; import isEqual from 'lodash/isEqual'; +import Canvas from './Canvas'; import TopMenu from './TopMenu'; import HelpSheet from './HelpSheet'; import SettingsSheet from './SettingsSheet'; @@ -39,7 +39,7 @@ import allOutFormats from './outFormats'; import { captureFrameFromTag, captureFrameFfmpeg } from './capture-frame'; import { defaultProcessedCodecTypes, getStreamFps, isCuttingStart, isCuttingEnd, - getDefaultOutFormat, getFormatData, renderFrame, mergeFiles as ffmpegMergeFiles, renderThumbnails as ffmpegRenderThumbnails, + getDefaultOutFormat, getFormatData, mergeFiles as ffmpegMergeFiles, renderThumbnails as ffmpegRenderThumbnails, readFrames, renderWaveformPng, html5ifyDummy, cutMultiple, extractStreams, autoMergeSegments, getAllStreams, findNearestKeyFrameTime, html5ify as ffmpegHtml5ify, isStreamThumbnail, } from './ffmpeg'; @@ -121,12 +121,8 @@ const zoomMax = 2 ** 14; const videoStyle = { width: '100%', height: '100%', objectFit: 'contain' }; -const queue = new PQueue({ concurrency: 1 }); - - const App = memo(() => { // Per project state - const [framePath, setFramePath] = useState(); const [waveform, setWaveform] = useState(); const [html5FriendlyPath, setHtml5FriendlyPath] = useState(); const [working, setWorking] = useState(false); @@ -140,7 +136,6 @@ const App = memo(() => { const [rotation, setRotation] = useState(360); const [cutProgress, setCutProgress] = useState(); const [startTimeOffset, setStartTimeOffset] = useState(0); - const [rotationPreviewRequested, setRotationPreviewRequested] = useState(false); const [filePath, setFilePath] = useState(''); const [externalStreamFiles, setExternalStreamFiles] = useState([]); const [detectedFps, setDetectedFps] = useState(); @@ -235,7 +230,6 @@ const App = memo(() => { firstUpdateRef.current = false; }, []); - // Global state const [helpVisible, setHelpVisible] = useState(false); const [settingsVisible, setSettingsVisible] = useState(false); @@ -326,8 +320,12 @@ const App = memo(() => { if (dummyVideoPath) unlink(dummyVideoPath).catch(console.error); }, [dummyVideoPath]); + // 360 means we don't modify rotation + const isRotationSet = rotation !== 360; + const effectiveRotation = isRotationSet ? rotation : (mainVideoStream && mainVideoStream.tags && mainVideoStream.tags.rotate && parseInt(mainVideoStream.tags.rotate, 10)); + const zoomRel = useCallback((rel) => setZoom(z => Math.min(Math.max(z + rel, 1), zoomMax)), []); - const frameRenderEnabled = !!(rotationPreviewRequested || dummyVideoPath); + const canvasPlayerEnabled = !!(mainVideoStream && (isRotationSet || dummyVideoPath)); const comfortZoom = duration ? Math.max(duration / 100, 1) : undefined; const toggleComfortZoom = useCallback(() => { @@ -565,38 +563,6 @@ const App = memo(() => { save(); }, [debouncedCutSegments, edlFilePath, autoSaveProjectFile]); - // 360 means we don't modify rotation - const isRotationSet = rotation !== 360; - const effectiveRotation = isRotationSet ? rotation : undefined; - - useEffect(() => { - async function throttledRender() { - if (queue.size < 2) { - queue.add(async () => { - if (!frameRenderEnabled) return; - - if (playerTime == null || !filePath) return; - - try { - const framePathNew = await renderFrame(playerTime, filePath, effectiveRotation); - setFramePath(framePathNew); - } catch (err) { - console.error(err); - } - }); - } - - await queue.onIdle(); - } - - throttledRender(); - }, [ - filePath, playerTime, frameRenderEnabled, effectiveRotation, - ]); - - // Cleanup old - useEffect(() => () => URL.revokeObjectURL(framePath), [framePath]); - function onPlayingChange(val) { setPlaying(val); if (!val) { @@ -614,13 +580,11 @@ const App = memo(() => { const onTimeUpdate = useCallback((e) => { const { currentTime } = e.target; if (playerTime === currentTime) return; - setRotationPreviewRequested(false); // Reset this setPlayerTime(currentTime); }, [playerTime]); const increaseRotation = useCallback(() => { setRotation((r) => (r + 90) % 450); - setRotationPreviewRequested(true); }, []); const assureOutDirAccess = useCallback(async (outFilePath) => { @@ -812,7 +776,6 @@ const App = memo(() => { video.playbackRate = 1; setFileNameTitle(); - setFramePath(); setHtml5FriendlyPath(); setDummyVideoPath(); setWorking(false); @@ -830,7 +793,6 @@ const App = memo(() => { setRotation(360); setCutProgress(); setStartTimeOffset(0); - setRotationPreviewRequested(false); setFilePath(''); // Setting video src="" prevents memory leak in chromium setExternalStreamFiles([]); setDetectedFps(); @@ -1015,7 +977,7 @@ const App = memo(() => { outFormat: fileFormat, isCustomFormatSelected, videoDuration: duration, - rotation: effectiveRotation, + rotation: isRotationSet ? effectiveRotation : undefined, copyFileStreams, keyframeCut, segments: outSegments, @@ -1062,7 +1024,7 @@ const App = memo(() => { setWorking(false); } }, [ - effectiveRotation, outSegments, handleCutFailed, + effectiveRotation, outSegments, handleCutFailed, isRotationSet, working, duration, filePath, keyframeCut, autoMerge, customOutDir, fileFormat, haveInvalidSegs, copyFileStreams, numStreamsToCopy, exportExtraStreams, nonCopiedExtraStreams, outputDir, shortestFlag, isCustomFormatSelected, @@ -1075,7 +1037,7 @@ const App = memo(() => { const currentTime = currentTimeRef.current; const video = videoRef.current; const outPath = mustCaptureFfmpeg - ? await captureFrameFfmpeg({ customOutDir, videoPath: filePath, currentTime, captureFormat, duration: video.duration }) + ? await captureFrameFfmpeg({ customOutDir, videoPath: filePath, currentTime, captureFormat, duration }) : await captureFrameFromTag({ customOutDir, filePath, video, currentTime, captureFormat }); openDirToast({ dirPath: outputDir, text: `${i18n.t('Screenshot captured to:')} ${outPath}` }); @@ -1083,7 +1045,7 @@ const App = memo(() => { console.error(err); errorToast(i18n.t('Failed to capture frame')); } - }, [filePath, captureFormat, customOutDir, html5FriendlyPath, dummyVideoPath, outputDir]); + }, [filePath, captureFormat, customOutDir, html5FriendlyPath, dummyVideoPath, outputDir, duration]); const changePlaybackRate = useCallback((dir) => { const video = videoRef.current; @@ -1825,24 +1787,15 @@ const App = memo(() => { onError={onVideoError} /> - {framePath && frameRenderEnabled && ( - - )} + {canvasPlayerEnabled && } - {rotationPreviewRequested && ( + {isRotationSet && (
- {t('Lossless rotation preview')} + {t('Rotation preview')}
)} diff --git a/src/Canvas.jsx b/src/Canvas.jsx new file mode 100644 index 00000000..bebf34fe --- /dev/null +++ b/src/Canvas.jsx @@ -0,0 +1,33 @@ + +import React, { memo, useEffect, useRef, useMemo } from 'react'; + +import CanvasPlayer from './CanvasPlayer'; + +const Canvas = memo(({ rotate, filePath, width, height, playerTime, commandedTime, playing }) => { + const canvasRef = useRef(); + + const canvasPlayer = useMemo(() => CanvasPlayer({ path: filePath, width, height }), + [filePath, width, height]); + + useEffect(() => { + canvasPlayer.setCanvas(canvasRef.current); + + return () => { + canvasPlayer.setCanvas(); + if (canvasPlayer) canvasPlayer.dispose(); + }; + }, [canvasPlayer]); + + useEffect(() => { + if (playing) canvasPlayer.play(commandedTime); + else canvasPlayer.pause(playerTime); + }, [canvasPlayer, commandedTime, playerTime, playing]); + + return ( +
+ +
+ ); +}); + +export default Canvas; diff --git a/src/CanvasPlayer.js b/src/CanvasPlayer.js new file mode 100644 index 00000000..96f5b3d7 --- /dev/null +++ b/src/CanvasPlayer.js @@ -0,0 +1,105 @@ +import { encodeLiveRawStream, getOneRawFrame } from './ffmpeg'; + +// TODO keep everything in electron land? +const strtok3 = window.require('strtok3'); + +export default ({ path, width: inWidth, height: inHeight }) => { + let canvas; + + let terminated; + let cancel; + let commandedTime; + let playing; + + function drawOnCanvas(rgbaImage, width, height) { + if (!canvas || rgbaImage.length === 0) return; + + canvas.width = width; + canvas.height = height; + + const ctx = canvas.getContext('2d'); + // https://developer.mozilla.org/en-US/docs/Web/API/ImageData/ImageData + // https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/putImageData + ctx.putImageData(new ImageData(Uint8ClampedArray.from(rgbaImage), width, height), 0, 0); + } + + async function run() { + let process; + let cancelled; + + cancel = () => { + cancelled = true; + if (process) process.cancel(); + cancel = undefined; + }; + + if (playing) { + try { + const { process: processIn, channels, width, height } = encodeLiveRawStream({ path, inWidth, inHeight, seekTo: commandedTime }); + process = processIn; + + // process.stderr.on('data', data => console.log(data.toString('utf-8'))); + + const tokenizer = await strtok3.fromStream(process.stdout); + + const size = width * height * channels; + const buf = Buffer.allocUnsafe(size); + + while (!cancelled) { + // eslint-disable-next-line no-await-in-loop + await tokenizer.readBuffer(buf, { length: size }); + if (!cancelled) drawOnCanvas(buf, width, height); + } + } catch (err) { + if (!err.isCanceled) console.warn(err.message); + } + } else { + try { + const { process: processIn, width, height } = getOneRawFrame({ path, inWidth, inHeight, seekTo: commandedTime }); + process = processIn; + const { stdout: rgbaImage } = await process; + + if (!cancelled) drawOnCanvas(rgbaImage, width, height); + } catch (err) { + if (!err.isCanceled) console.warn(err.message); + } + } + } + + function command() { + if (cancel) cancel(); + run(); + } + + function pause(seekTo) { + if (terminated) return; + if (!playing && commandedTime === seekTo) return; + playing = false; + commandedTime = seekTo; + command(); + } + + function play(playFrom) { + if (terminated) return; + if (playing && commandedTime === playFrom) return; + playing = true; + commandedTime = playFrom; + command(); + } + + function setCanvas(c) { + canvas = c; + } + + function dispose() { + terminated = true; + if (cancel) cancel(); + } + + return { + play, + pause, + setCanvas, + dispose, + }; +}; diff --git a/src/ffmpeg.js b/src/ffmpeg.js index a51bdbc0..937819c2 100644 --- a/src/ffmpeg.js +++ b/src/ffmpeg.js @@ -217,7 +217,7 @@ async function cut({ ...cutToArgs, ]; - const rotationArgs = rotation !== undefined ? ['-metadata:s:v:0', `rotate=${rotation}`] : []; + const rotationArgs = rotation !== undefined ? ['-metadata:s:v:0', `rotate=${360 - rotation}`] : []; const ffmpegArgs = [ '-hide_banner', @@ -310,6 +310,7 @@ export async function cutMultiple({ return outFiles; } +// TODO merge with getFormatData export async function getDuration(filePath) { // https://superuser.com/questions/650291/how-to-get-video-duration-in-seconds const { stdout } = await runFfprobe(['-i', filePath, '-show_entries', 'format=duration', '-print_format', 'json']); @@ -648,35 +649,6 @@ export async function renderWaveformPng({ filePath, aroundTime, window, color }) } } -export async function renderFrame(timestamp, filePath, rotation) { - const transpose = { - 90: 'transpose=2', - 180: 'transpose=1,transpose=1', - 270: 'transpose=1', - }; - const args = [ - '-ss', timestamp, - ...(rotation !== undefined ? ['-noautorotate'] : []), - '-i', filePath, - // ...(rotation !== undefined ? ['-metadata:s:v:0', 'rotate=0'] : []), // Reset the rotation metadata first - ...(rotation !== undefined && rotation > 0 ? ['-vf', `${transpose[rotation]}`] : []), - '-f', 'image2', - '-vframes', '1', - '-q:v', '10', - '-', - // '-y', outPath, - ]; - - // console.time('ffmpeg'); - const ffmpegPath = getFfmpegPath(); - // console.timeEnd('ffmpeg'); - // console.log('ffmpeg', args); - const { stdout } = await execa(ffmpegPath, args, { encoding: null }); - - const blob = new Blob([stdout], { type: 'image/jpeg' }); - return URL.createObjectURL(blob); -} - export async function extractWaveform({ filePath, outPath }) { const numSegs = 10; const duration = 60 * 60; @@ -739,3 +711,66 @@ export function getStreamFps(stream) { } return undefined; } + +function createRawFfmpeg({ fps = 25, path, inWidth, inHeight, seekTo, oneFrameOnly, execaOpts }) { + // const fps = 25; // TODO + + const aspectRatio = inWidth / inHeight; + + let newWidth; + let newHeight; + if (inWidth > inHeight) { + newWidth = 320; + newHeight = Math.floor(newWidth / aspectRatio); + } else { + newHeight = 320; + newWidth = Math.floor(newHeight * aspectRatio); + } + + const args = [ + '-hide_banner', '-loglevel', 'panic', + + '-re', + + '-ss', seekTo, + + '-noautorotate', + + '-i', path, + + '-vf', `fps=${fps},scale=${newWidth}:${newHeight}:flags=lanczos`, + '-map', 'v:0', + '-vcodec', 'rawvideo', + '-pix_fmt', 'rgba', + + ...(oneFrameOnly ? ['-frames:v', '1'] : []), + + '-f', 'image2pipe', + '-', + ]; + + // console.log(args); + + return { + process: execa(getFfmpegPath(), args, execaOpts), + width: newWidth, + height: newHeight, + channels: 4, + }; +} + +export function getOneRawFrame({ path, inWidth, inHeight, seekTo }) { + const { process, width, height, channels } = createRawFfmpeg({ path, inWidth, inHeight, seekTo, oneFrameOnly: true, execaOpts: { encoding: null } }); + return { process, width, height, channels }; +} + +export function encodeLiveRawStream({ path, inWidth, inHeight, seekTo }) { + const { process, width, height, channels } = createRawFfmpeg({ path, inWidth, inHeight, seekTo, execaOpts: { encoding: null, buffer: false } }); + + return { + process, + width, + height, + channels, + }; +} diff --git a/yarn.lock b/yarn.lock index 4cec0bcd..dee84943 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1544,6 +1544,11 @@ dependencies: defer-to-connect "^1.0.1" +"@tokenizer/token@^0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.1.1.tgz#f0d92c12f87079ddfd1b29f614758b9696bc29e3" + integrity sha512-XO6INPbZCxdprl+9qa/AAbFFOMzzwqYxpjPgLICrMD6C2FCw6qfJOPcBk6JqqPLSaZ/Qx87qn4rpPmPMwaAK6w== + "@types/babel__core@^7.1.0": version "7.1.6" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.6.tgz#16ff42a5ae203c9af1c6e190ed1f30f83207b610" @@ -8973,14 +8978,6 @@ p-map@^3.0.0: dependencies: aggregate-error "^3.0.0" -p-queue@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.2.0.tgz#c8122b9514d2bbe5f16d8a47f17dc2f9a8ac7235" - integrity sha512-B2LXNONcyn/G6uz2UBFsGjmSa0e/br3jznlzhEyCXg56c7VhEpiT2pZxGOfv32Q3FSyugAdys9KGpsv3kV+Sbg== - dependencies: - eventemitter3 "^4.0.0" - p-timeout "^3.1.0" - p-reduce@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-reduce/-/p-reduce-1.0.0.tgz#18c2b0dd936a4690a529f8231f58a0fdb6a47dfa" @@ -8993,13 +8990,6 @@ p-retry@^3.0.1: dependencies: retry "^0.12.0" -p-timeout@^3.1.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" - integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== - dependencies: - p-finally "^1.0.0" - p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" @@ -9243,6 +9233,11 @@ pbkdf2@^3.0.3: safe-buffer "^5.0.1" sha.js "^2.4.8" +peek-readable@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/peek-readable/-/peek-readable-3.1.0.tgz#250b08b7de09db8573d7fd8ea475215bbff14348" + integrity sha512-KGuODSTV6hcgdZvDrIDBUkN0utcAVj1LL7FfGbM0viKTtCHmtZcuEJ+lGqsp0fTFkGqesdtemV2yUSMeyy3ddA== + pend@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" @@ -12026,6 +12021,16 @@ strong-data-uri@^1.0.5: dependencies: truncate "^2.0.1" +strtok3@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-6.0.0.tgz#d6b900863daeacfe6c1724c6e7bb36d7a58e83c8" + integrity sha512-ZXlmE22LZnIBvEU3n/kZGdh770fYFie65u5+2hLK9s74DoFtpkQIdBZVeYEzlolpGa+52G5IkzjUWn+iXynOEQ== + dependencies: + "@tokenizer/token" "^0.1.1" + "@types/debug" "^4.1.5" + debug "^4.1.1" + peek-readable "^3.1.0" + style-loader@0.23.1: version "0.23.1" resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-0.23.1.tgz#cb9154606f3e771ab6c4ab637026a1049174d925"