From 2510135db4e0f4a07d7ae7a9e6a185e40c2d215d Mon Sep 17 00:00:00 2001 From: Mikael Finstad Date: Mon, 13 Oct 2025 16:45:02 +0800 Subject: [PATCH] improve compat player --- src/main/compatPlayer.ts | 32 +- src/main/ffmpeg.ts | 46 +- src/renderer/src/App.tsx | 637 ++++++++++++------------- src/renderer/src/MediaSourcePlayer.tsx | 362 +++++++------- src/renderer/src/hooks/useVideo.ts | 4 - 5 files changed, 532 insertions(+), 549 deletions(-) diff --git a/src/main/compatPlayer.ts b/src/main/compatPlayer.ts index aa272917..f828c267 100644 --- a/src/main/compatPlayer.ts +++ b/src/main/compatPlayer.ts @@ -1,10 +1,10 @@ -import assert from 'node:assert'; import { ExecaError } from 'execa'; import logger from './logger.js'; -import { createMediaSourceProcess, readOneJpegFrame as readOneJpegFrameRaw } from './ffmpeg.js'; +import { createMediaSourceProcess } from './ffmpeg.js'; +// eslint-disable-next-line import/prefer-default-export export function createMediaSourceStream(params: Parameters[0]) { const abortController = new AbortController(); const { videoStreamIndex, audioStreamIndexes, seekTo } = params; @@ -18,7 +18,6 @@ export function createMediaSourceStream(params: Parameters { stdout.pause(); cleanup(); @@ -75,28 +76,3 @@ export function createMediaSourceStream(params: Parameters[0]) { - const abortController = new AbortController(); - const process = readOneJpegFrameRaw(params); - - // eslint-disable-next-line unicorn/prefer-add-event-listener - abortController.signal.onabort = () => process.kill('SIGKILL'); - - function abort() { - abortController.abort(); - } - - const promise = (async () => { - try { - const { stdout } = await process; - return stdout; - } catch (err) { - // @ts-expect-error todo - logger.error('renderOneJpegFrame', err.shortMessage); - throw new Error('Failed to render JPEG frame'); - } - })(); - - return { promise, abort }; -} diff --git a/src/main/ffmpeg.ts b/src/main/ffmpeg.ts index 654952d1..7f91ce76 100644 --- a/src/main/ffmpeg.ts +++ b/src/main/ffmpeg.ts @@ -532,34 +532,6 @@ export async function getDuration(filePath: string) { return parseFloat((await readFormatData(filePath)).duration); } -export function readOneJpegFrame({ path, seekTo, videoStreamIndex, rotate }: { - path: string, - seekTo: number, - videoStreamIndex: number, - rotate: number | undefined, -}) { - const args = [ - '-hide_banner', '-loglevel', 'error', - - '-ss', String(seekTo), - - ...(rotate != null ? ['-noautorotate'] : []), - - '-i', path, - - '-map', `0:${videoStreamIndex}`, - '-vcodec', 'mjpeg', - - '-frames:v', '1', - - '-f', 'image2pipe', - '-', - ]; - - logger.info(getFfCommandLine('ffmpeg', args)); - return runFfmpegProcess(args, undefined, { logCli: false }); -} - const enableLog = false; const encode = true; @@ -602,6 +574,18 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn return ['-filter_complex', graph.join(';')]; } + const videoEncodeArgs = [ + 'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', '-crf', '10', + + // should, in theory, set the private options of x264 or x265 automatically + // https://www.reddit.com/r/ffmpeg/comments/jlk2zn/how_to_encode_using_bt709/ + '-colorspace:v', 'bt709', '-color_primaries:v', 'bt709', '-color_trc:v', 'bt709', '-color_range:v', 'tv', + '-pix_fmt:v', 'yuv420p', + ]; + + // const videoEncodeArgs = ['h264_videotoolbox', '-b:v', '5M'] + + // https://stackoverflow.com/questions/16658873/how-to-minimize-the-delay-in-a-live-streaming-with-ffmpeg // https://unix.stackexchange.com/questions/25372/turn-off-buffering-in-pipe const args = [ @@ -625,12 +609,16 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn '-fps_mode', 'passthrough', + '-map_metadata', '-1', + '-map_chapters', '-1', + ...(encode ? [ ...getFilters(), ...(videoStreamIndex != null ? [ '-map', '[video]', - '-pix_fmt', 'yuv420p', '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', '-crf', '10', + '-c:v', ...videoEncodeArgs, + '-g', '1', // reduces latency and buffering ] : ['-vn']), diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 450da86f..9c0ad148 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -141,7 +141,7 @@ function App() { const [activeVideoStreamIndex, setActiveVideoStreamIndex] = useState(); const [activeAudioStreamIndexes, setActiveAudioStreamIndexes] = useState>(new Set()); const [activeSubtitleStreamIndex, setActiveSubtitleStreamIndex] = useState(); - const [hideMediaSourcePlayer, setHideMediaSourcePlayer] = useState(false); + const [hideCompatPlayer, setHideCompatPlayer] = useState(false); const [exportConfirmVisible, setExportConfirmVisible] = useState(false); const [cacheBuster, setCacheBuster] = useState(0); const [currentFileExportCount, setCurrentFileExportCount] = useState(0); @@ -184,7 +184,7 @@ function App() { const keyBindingByAction = useMemo(() => Object.fromEntries(keyBindings.map((binding) => [binding.action, binding])) as Record, [keyBindings]); const { working, setWorking, workingRef, abortWorking } = useLoading(); - const { videoRef, videoContainerRef, playbackRate, setPlaybackRate, outputPlaybackRate, setOutputPlaybackRate, commandedTime, seekAbs, playingRef, getRelevantTime, setPlaying, onSeeked, relevantTime, onStartPlaying, setCommandedTime, setCompatPlayerEventId, compatPlayerEventId, setOutputPlaybackRateState, commandedTimeRef, onStopPlaying, onVideoAbort, playerTime, setPlayerTime, playbackModeRef, playing, play, pause, seekRel } = useVideo({ filePath }); + const { videoRef, videoContainerRef, playbackRate, setPlaybackRate, outputPlaybackRate, setOutputPlaybackRate, commandedTime, seekAbs, playingRef, getRelevantTime, setPlaying, onSeeked, relevantTime, onStartPlaying, setCommandedTime, setOutputPlaybackRateState, commandedTimeRef, onStopPlaying, onVideoAbort, playerTime, setPlayerTime, playbackModeRef, playing, play, pause, seekRel } = useVideo({ filePath }); const { timecodePlaceholder, formatTimecode, formatTimeAndFrames, parseTimecode, getFrameCount } = useTimecode({ detectedFps, timecodeFormat }); const { loadSubtitle, subtitlesByStreamId, setSubtitlesByStreamId } = useSubtitles(); @@ -319,10 +319,8 @@ function App() { const zoomAbs = useCallback((fn: (v: number) => number) => setZoom((z) => Math.min(Math.max(fn(z), 1), zoomMax)), []); const zoomRel = useCallback((rel: number) => zoomAbs((z) => z + (rel * (1 + (z / 10)))), [zoomAbs]); const compatPlayerRequired = ( - // when using html5ified dummy video, we *have* to use canvas player - usingDummyVideo - // or if user selected an explicit video or audio stream, and the html5 player does not have any track index corresponding to the selected stream index - || ( + // if user selected an explicit video or audio stream, and the html5 player does not have any track index corresponding to the selected stream index + ( (activeVideoStreamIndex != null || activeAudioStreamIndexes.size === 1) && videoRef.current != null && !canHtml5PlayerPlayStreams(videoRef.current, activeVideoStreamIndex, [...activeAudioStreamIndexes][0]) @@ -332,7 +330,8 @@ function App() { ); // if user selected a rotation, but they might want to turn off the rotation preview // but allow the user to disable - const compatPlayerWanted = isRotationSet && !hideMediaSourcePlayer; + const compatPlayerWanted = (isRotationSet && !hideCompatPlayer) + || usingDummyVideo; const compatPlayerEnabled = (compatPlayerRequired || compatPlayerWanted) && (activeVideoStream != null || activeAudioStreams.length > 0); @@ -340,7 +339,7 @@ function App() { useEffect(() => { // Reset the user preference when we go from not having compat player to having it - if (compatPlayerEnabled) setHideMediaSourcePlayer(false); + if (compatPlayerEnabled) setHideCompatPlayer(false); }, [compatPlayerEnabled]); const comfortZoom = isDurationValid(fileDuration) ? Math.max(fileDuration / 100, 1) : undefined; @@ -441,7 +440,7 @@ function App() { const increaseRotation = useCallback(() => { setRotation((r) => (r + 90) % 450); - setHideMediaSourcePlayer(false); + setHideCompatPlayer(false); // Matroska is known not to work, so we warn user. See https://github.com/mifi/lossless-cut/discussions/661 const supportsRotation = !isMatroska(fileFormat); if (!supportsRotation) showNotification({ text: i18n.t('Lossless rotation might not work with this file format. You may try changing to MP4') }); @@ -554,14 +553,14 @@ function App() { const onActiveVideoStreamChange = useCallback((videoStreamIndex?: number) => { invariant(videoRef.current); - setHideMediaSourcePlayer(false); + setHideCompatPlayer(false); enableVideoTrack(videoRef.current, videoStreamIndex); setActiveVideoStreamIndex(videoStreamIndex); }, [videoRef]); const onActiveAudioStreamsChange = useCallback((audioStreamIndexes: Set) => { invariant(videoRef.current); - setHideMediaSourcePlayer(false); + setHideCompatPlayer(false); enableAudioTrack(videoRef.current, [...audioStreamIndexes][0]); setActiveAudioStreamIndexes(audioStreamIndexes); }, [videoRef]); @@ -618,7 +617,6 @@ function App() { setPlaying(false); playingRef.current = false; playbackModeRef.current = undefined; - setCompatPlayerEventId(0); setFileDuration(undefined); cutSegmentsHistory.go(0); setFileFormat(undefined); @@ -642,11 +640,11 @@ function App() { setActiveAudioStreamIndexes(new Set()); setActiveVideoStreamIndex(undefined); setActiveSubtitleStreamIndex(undefined); - setHideMediaSourcePlayer(false); + setHideCompatPlayer(false); setExportConfirmVisible(false); setOutputPlaybackRateState(1); setCurrentFileExportCount(0); - }, [videoRef, setCommandedTime, setPlaybackRate, setPlaying, playingRef, playbackModeRef, setCompatPlayerEventId, setFileDuration, cutSegmentsHistory, setFileFormat, setDetectedFileFormat, setCopyStreamIdsByFile, setThumbnails, setSubtitlesByStreamId, setOutputPlaybackRateState]); + }, [videoRef, setCommandedTime, setPlaybackRate, setPlaying, playingRef, playbackModeRef, setFileDuration, cutSegmentsHistory, setFileFormat, setDetectedFileFormat, setCopyStreamIdsByFile, setThumbnails, setSubtitlesByStreamId, setOutputPlaybackRateState]); const showUnsupportedFileMessage = useCallback(() => { @@ -699,6 +697,12 @@ function App() { setUsingDummyVideo(usesDummyVideo); }, [html5ify, html5ifyDummy]); + const handleHideCompatPlayerClick = useCallback(() => { + setHideCompatPlayer(true); + setPreviewFilePath(undefined); + setUsingDummyVideo(false); + }, []); + const convertFormatBatch = useCallback(async () => { if (batchFiles.length === 0) return; const filePaths = batchFiles.map((f) => f.path); @@ -1326,11 +1330,6 @@ function App() { const extractSelectedSegmentsFramesAsImages = useCallback(() => extractSegmentsFramesAsImages(selectedSegments), [extractSegmentsFramesAsImages, selectedSegments]); const userChangePlaybackRate = useCallback((dir: number, rateMultiplier?: number) => { - if (compatPlayerEnabled) { - toast.fire({ title: i18n.t('Unable to change playback rate right now'), timer: 1000 }); - return; - } - const video = videoRef.current; if (!playingRef.current) { video!.play(); @@ -1338,7 +1337,7 @@ function App() { const newRate = adjustRate(video!.playbackRate, dir, rateMultiplier); setPlaybackRate(newRate); } - }, [compatPlayerEnabled, playingRef, setPlaybackRate, videoRef]); + }, [playingRef, setPlaybackRate, videoRef]); const loadEdlFile = useCallback(async ({ path, type, append = false }: { path: string, type: EdlFileType, append?: boolean }) => { console.log('Loading EDL file', type, path, append); @@ -1497,7 +1496,7 @@ function App() { } // only show one toast, or else we will only show the last one - if (existingHtml5FriendlyFile) { + if (existingHtml5FriendlyFile && !existingHtml5FriendlyFile.usingDummyVideo) { showPreviewFileLoadedMessage(basename(existingHtml5FriendlyFile.path)); } else if (needsAutoHtml5ify) { showUnsupportedFileMessage(); @@ -2518,321 +2517,321 @@ function App() { return ( <> - - - -
- - -
- - {showLeftBar && ( - - )} - - - {/* Middle part (also shown in fullscreen): */} -
- {!isFileOpened && } - -
- {/* eslint-disable-next-line jsx-a11y/media-has-caption */} - - - {filePath != null && compatPlayerEnabled && } -
+ + + +
+ - {bigWaveformEnabled && } +
+ + {showLeftBar && ( + + )} + + + {/* Middle part (also shown in fullscreen): */} +
+ {!isFileOpened && } + +
+ {/* eslint-disable-next-line jsx-a11y/media-has-caption */} + + + {filePath != null && compatPlayerEnabled && } +
- {compatPlayerEnabled && ( -
- {isRotationSet ? ( - <> - - {t('Rotation preview')} - - ) : ( - <> - {t('FFmpeg-assisted playback')} - - )} + {bigWaveformEnabled && } + + {compatPlayerEnabled && ( +
+ {isRotationSet ? ( + <> + + {t('Rotation preview')} + + ) : ( + <> + {t('FFmpeg-assisted playback')} + + )} + +
incrementMediaSourceQuality()} title={t('Select playback quality')}>{mediaSourceQualities[mediaSourceQuality]}
+ + {!compatPlayerRequired && } +
+ )} + + {isFileOpened && ( +
+ + + {shouldShowPlaybackStreamSelector && ( + + )} + + {!showRightBar && ( + + )} +
+ )} +
-
incrementMediaSourceQuality()} title={t('Select playback quality')}>{mediaSourceQualities[mediaSourceQuality]}
+ + {showRightBar && isFileOpened && filePath != null && ( + + )} + +
- {!compatPlayerRequired && setHideMediaSourcePlayer(true)} />} -
- )} +
+ - {isFileOpened && ( -
- - - {shouldShowPlaybackStreamSelector && ( - - )} - - {!showRightBar && ( - - )} -
- )} +
- - {showRightBar && isFileOpened && filePath != null && ( - setTunerVisible(undefined)} />} + + {/* Dialogs */} + + + + setStreamsSelectorShown(false)} maxWidth={1000}> + {mainStreams && filePath != null && ( + )} - -
+ -
- - -
- - {tunerVisible != null && setTunerVisible(undefined)} />} - - {/* Dialogs */} - - - - setStreamsSelectorShown(false)} maxWidth={1000}> - {mainStreams && filePath != null && ( - + - )} - - - - - - - + - 0 && concatDialogVisible} onHide={() => setConcatDialogVisible(false)} paths={batchFilePaths} onConcat={userConcatFiles} setAlwaysConcatMultipleFiles={setAlwaysConcatMultipleFiles} alwaysConcatMultipleFiles={alwaysConcatMultipleFiles} exportCount={exportCount} maxLabelLength={maxLabelLength} /> + 0 && concatDialogVisible} onHide={() => setConcatDialogVisible(false)} paths={batchFilePaths} onConcat={userConcatFiles} setAlwaysConcatMultipleFiles={setAlwaysConcatMultipleFiles} alwaysConcatMultipleFiles={alwaysConcatMultipleFiles} exportCount={exportCount} maxLabelLength={maxLabelLength} /> - setKeyboardShortcutsVisible(false)} keyBindings={keyBindings} setKeyBindings={setKeyBindings} currentCutSeg={currentCutSeg} resetKeyBindings={resetKeyBindings} /> + setKeyboardShortcutsVisible(false)} keyBindings={keyBindings} setKeyBindings={setKeyBindings} currentCutSeg={currentCutSeg} resetKeyBindings={resetKeyBindings} /> - {/* This should probably be last, so that it's always on top */} - - {working && } - -
- - - + {/* This should probably be last, so that it's always on top */} + + {working && } + +
+ + +
diff --git a/src/renderer/src/MediaSourcePlayer.tsx b/src/renderer/src/MediaSourcePlayer.tsx index eadef800..7ed2fe5c 100644 --- a/src/renderer/src/MediaSourcePlayer.tsx +++ b/src/renderer/src/MediaSourcePlayer.tsx @@ -1,63 +1,69 @@ import { useEffect, useRef, useState, useCallback, useMemo, memo, CSSProperties, RefObject, ReactEventHandler, FocusEventHandler } from 'react'; import { Spinner } from 'evergreen-ui'; -import { useDebounce } from 'use-debounce'; import invariant from 'tiny-invariant'; +import debounce from 'lodash/debounce.js'; import isDev from './isDev'; import { ChromiumHTMLVideoElement } from './types'; import { FFprobeStream } from '../../../ffprobe'; -const { compatPlayer: { createMediaSourceStream, readOneJpegFrame } } = window.require('@electron/remote').require('./index.js'); +const { compatPlayer: { createMediaSourceStream } } = window.require('@electron/remote').require('./index.js'); -async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes, seekTo, signal, onPlayRequested, onCanPlay, getTargetTime, size, fps, rotate }: { +async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, audioStreamIndexes, seekTo, signal, size, fps, rotate, onCanPlay, onResetNeeded, onWaiting }: { path: string, - video: ChromiumHTMLVideoElement, + slaveVideo: ChromiumHTMLVideoElement, + masterVideo: ChromiumHTMLVideoElement, videoStreamIndex?: number | undefined, audioStreamIndexes: number[], seekTo: number, signal: AbortSignal, - onPlayRequested: () => void, - onCanPlay: () => void, - getTargetTime: () => number, size?: number | undefined, fps?: number | undefined, rotate: number | undefined, + onCanPlay: () => void, + onResetNeeded: () => void, + onWaiting: () => void, }) { let canPlay = false; let bufferEndTime: number | undefined; - let bufferStartTime = 0; + let bufferStartTime = seekTo; let stream: ReturnType | undefined; - let done = false; let interval: NodeJS.Timeout | undefined; + let interval2: NodeJS.Timeout | undefined; let objectUrl: string | undefined; let processChunkTimeout: NodeJS.Timeout; - function cleanup() { + signal.addEventListener('abort', () => { console.log('Cleanup'); - done = true; - video.pause(); + slaveVideo.pause(); if (interval != null) clearInterval(interval); + if (interval2 != null) clearInterval(interval2); if (processChunkTimeout != null) clearInterval(processChunkTimeout); stream?.abort(); if (objectUrl != null) URL.revokeObjectURL(objectUrl); - video.removeAttribute('src'); - } - - signal.addEventListener('abort', cleanup); + slaveVideo.removeAttribute('src'); + }); // See chrome://media-internals - const mediaSource = new MediaSource(); - let streamTimestamp: number | undefined; - let lastRemoveTimestamp = 0; + let lastRemoveTimestamp = seekTo; + + const setPlaybackRate = (r: number) => { + const maxAllowedPlaybackRate = 16; // or else we get an error in Chromium + const newAdjustedRate = Math.min(maxAllowedPlaybackRate, r * masterVideo.playbackRate); + if (slaveVideo.playbackRate === newAdjustedRate) { + return false; + } - function setStandardPlaybackRate() { - // set it a bit faster, so that we don't easily fall behind (better too fast than too slow) // eslint-disable-next-line no-param-reassign - video.playbackRate = 1.05; - } + slaveVideo.playbackRate = newAdjustedRate; + return true; + }; + + // set it a bit faster, so that we don't easily fall behind (better too fast than too slow) + const setStandardPlaybackRate = () => setPlaybackRate(1.05); setStandardPlaybackRate(); @@ -80,15 +86,20 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes throw new Error(`Unsupported MIME type or codec: ${mimeCodec}`); } + const mediaSource = new MediaSource(); + // console.log(mediaSource.readyState); // closed objectUrl = URL.createObjectURL(mediaSource); // eslint-disable-next-line no-param-reassign - video.src = objectUrl; + slaveVideo.src = objectUrl; await new Promise((resolve) => mediaSource.addEventListener('sourceopen', resolve, { once: true })); // console.log(mediaSource.readyState); // open const sourceBuffer = mediaSource.addSourceBuffer(mimeCodec); + sourceBuffer.timestampOffset = seekTo; + + signal.addEventListener('abort', () => sourceBuffer.abort()); const getBufferEndTime = () => { if (mediaSource.readyState !== 'open') { @@ -105,10 +116,6 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes return sourceBuffer.buffered.end(0); }; - sourceBuffer.addEventListener('updateend', () => { - onPlayRequested(); - }, { once: true }); - let firstChunkReceived = false; const processChunk = async () => { @@ -118,7 +125,8 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes console.log('End of stream'); return; } - if (done) return; + + if (signal.aborted) return; if (!firstChunkReceived) { firstChunkReceived = true; @@ -134,18 +142,46 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes sourceBuffer.addEventListener('error', (err) => console.error('sourceBuffer error, check DevTools ▶ More Tools ▶ Media', err)); - // video.addEventListener('loadeddata', () => console.log('loadeddata')); - // video.addEventListener('play', () => console.log('play')); - video.addEventListener('canplay', () => { + const handleCanPlay = () => { console.log('canplay'); - if (!canPlay) { - canPlay = true; - onCanPlay(); - } - }, { once: true }); + canPlay = true; + onCanPlay(); + }; + slaveVideo.addEventListener('canplay', handleCanPlay); + + const handleEnded = () => { + console.log('ended'); + }; + slaveVideo.addEventListener('ended', handleEnded); + + const handleStalled = () => { + console.log('stalled'); + }; + slaveVideo.addEventListener('stalled', handleStalled); + + const handleWaiting = () => { + if (slaveVideo.paused || slaveVideo.ended) return; // we don't care if paused + console.log('waiting'); + onWaiting(); + }; + + slaveVideo.addEventListener('waiting', handleWaiting); + + const handlePlaying = () => { + console.log('playing'); + }; + slaveVideo.addEventListener('playing', handlePlaying); + + signal.addEventListener('abort', () => { + slaveVideo.removeEventListener('canplay', handleCanPlay); + slaveVideo.removeEventListener('ended', handleEnded); + slaveVideo.removeEventListener('stalled', handleStalled); + slaveVideo.removeEventListener('waiting', handleWaiting); + slaveVideo.removeEventListener('playing', handlePlaying); + }); sourceBuffer.addEventListener('updateend', ({ timeStamp }) => { - if (done) return; + if (signal.aborted) return; streamTimestamp = timeStamp; // apparently this timestamp cannot be trusted much @@ -154,13 +190,10 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes bufferEndTime = getBufferEndTime(); - // console.log('updateend', { bufferEndTime }) if (bufferEndTime != null) { - const targetTime = getTargetTime(); - - const bufferedTime = bufferEndTime - lastRemoveTimestamp; + const bufferedDuration = bufferEndTime - lastRemoveTimestamp; - if (bufferedTime > bufferMaxSec && !sourceBuffer.updating) { + if (bufferedDuration > bufferMaxSec && !sourceBuffer.updating) { try { lastRemoveTimestamp = bufferEndTime; const removeTo = bufferEndTime - bufferMaxSec; @@ -173,7 +206,7 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes } } - const bufferAheadSec = bufferEndTime - targetTime; + const bufferAheadSec = bufferEndTime - masterVideo.currentTime; if (bufferAheadSec > bufferThrottleSec) { console.debug(`buffer ahead by ${bufferAheadSec}, throttling stream read`); processChunkTimeout = setTimeout(processChunk, 1000); @@ -188,195 +221,186 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes stream = createMediaSourceStream({ path, videoStreamIndex, audioStreamIndexes, seekTo, size, fps, rotate }); interval = setInterval(() => { + if (!canPlay) return; + if (mediaSource.readyState !== 'open') { console.warn('mediaSource.readyState was not open, but:', mediaSource.readyState); // else we will get: Uncaught DOMException: Failed to execute 'end' on 'TimeRanges': The index provided (0) is greater than or equal to the maximum bound (0). return; } - const targetTime = getTargetTime(); - const playbackDiff = targetTime != null ? targetTime - video.currentTime : undefined; - - const streamTimestampDiff = streamTimestamp != null && bufferEndTime != null ? (streamTimestamp / 1000) - bufferEndTime : undefined; // not really needed, but log for curiosity - console.debug('bufferStartTime', bufferStartTime, 'bufferEndTime', bufferEndTime, 'targetTime', targetTime, 'playback:', video.currentTime, 'playbackDiff:', playbackDiff, 'streamTimestamp diff:', streamTimestampDiff); - - if (!canPlay || targetTime == null) return; + console.log(`bufferStartTime: ${bufferStartTime}, bufferEndTime: ${bufferEndTime}, master time: ${masterVideo.currentTime}, slave time: ${slaveVideo.currentTime} (diff: ${masterVideo.currentTime - slaveVideo.currentTime}), streamTimestamp: ${streamTimestamp}`); + // console.log(sourceBuffer.buffered.length, sourceBuffer.buffered.start(0), sourceBuffer.buffered.end(0)) if (sourceBuffer.buffered.length !== 1) { // not sure why this would happen or how to handle this console.warn('sourceBuffer.buffered.length was', sourceBuffer.buffered.length); } + }, 1000); - if ((video.paused || video.ended) && !done) { - console.warn('Resuming unexpectedly paused video'); - onPlayRequested(); - } + // Synchronize state between the two video elements + interval2 = setInterval(async () => { + try { + const maxSecAfterBufferToWaitFor = 5; + if (masterVideo.currentTime < bufferStartTime || (bufferEndTime != null && masterVideo.currentTime - bufferEndTime > maxSecAfterBufferToWaitFor)) { + console.log('Seeked before/after buffered range, resetting playback'); + onResetNeeded(); + return; + } + + if (masterVideo.paused || masterVideo.ended) { + const resolution = 1000; + if (Math.round(slaveVideo.currentTime * resolution) !== Math.round(masterVideo.currentTime * resolution)) { + // eslint-disable-next-line no-param-reassign + slaveVideo.currentTime = masterVideo.currentTime; + } + } else { // playing + // make sure the playback keeps up while playing + // or when seeking while playing + // https://stackoverflow.com/questions/23301496/how-to-keep-a-live-mediasource-video-stream-in-sync + const playbackDiff = masterVideo.currentTime - slaveVideo.currentTime; + if (Math.abs(playbackDiff) > 1) { + console.log(`Playback ${playbackDiff > 0 ? 'behind' : 'ahead'} master player time by ${playbackDiff}s, jumping to desired time`); + // eslint-disable-next-line no-param-reassign + slaveVideo.currentTime = masterVideo.currentTime; + setStandardPlaybackRate(); + } else if (playbackDiff != null && playbackDiff > 0.3) { + // eslint-disable-next-line no-param-reassign + if (setPlaybackRate(1.5)) { + console.warn(`Playback behind by ${playbackDiff}s, speeding up playback`); + } + } else { + setStandardPlaybackRate(); + } + } - // make sure the playback keeps up - // https://stackoverflow.com/questions/23301496/how-to-keep-a-live-mediasource-video-stream-in-sync - if (playbackDiff != null && playbackDiff > 1) { - console.warn(`playback severely behind by ${playbackDiff}s, seeking to desired time`); - // eslint-disable-next-line no-param-reassign - video.currentTime = targetTime; - setStandardPlaybackRate(); - } else if (playbackDiff != null && playbackDiff > 0.3) { - console.warn(`playback behind by ${playbackDiff}s, speeding up playback`); - // eslint-disable-next-line no-param-reassign - video.playbackRate = 1.5; - } else { - setStandardPlaybackRate(); + if (slaveVideo.volume !== masterVideo.volume) { + // eslint-disable-next-line no-param-reassign + slaveVideo.volume = masterVideo.volume; + } + + const masterStopped = masterVideo.paused || masterVideo.ended; + const slaveStopped = slaveVideo.paused || slaveVideo.ended; + + if (slaveStopped && !masterStopped) { + await slaveVideo.play(); + } else if (!slaveStopped && masterStopped) { + slaveVideo.pause(); + } + } catch (err) { + console.error('play/pause failed', err); } - }, 200); + }, 30); // todo requestAnimationFrame? // OK, everything initialized and ready to stream! processChunk(); } -async function createPauseImage({ path, seekTo, videoStreamIndex, image, rotate, signal }: { - path: string, - seekTo: number, - videoStreamIndex: number, - image: HTMLImageElement | null, - rotate: number | undefined, - signal: AbortSignal, -}) { - const { promise, abort } = readOneJpegFrame({ path, seekTo, videoStreamIndex, rotate }); - signal.addEventListener('abort', () => abort()); - const jpegImage = await promise; - - invariant(image); - if (image.src) URL.revokeObjectURL(image.src); - // eslint-disable-next-line no-param-reassign - image.src = URL.createObjectURL(new Blob([Buffer.from(jpegImage)], { type: 'image/jpeg' })); -} - -function MediaSourcePlayer({ rotate, filePath, playerTime, videoStream, audioStreams, commandedTime, playing, eventId, masterVideoRef, mediaSourceQuality, playbackVolume }: { +function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, masterVideoRef, mediaSourceQuality }: { rotate: number | undefined, filePath: string, - playerTime: number, videoStream: FFprobeStream | undefined, audioStreams: FFprobeStream[], - commandedTime: number, - playing: boolean, - eventId: number, masterVideoRef: RefObject, mediaSourceQuality: number, - playbackVolume: number, }) { const videoRef = useRef(null); - const imgRef = useRef(null); + const canvasRef = useRef(null); const [loading, setLoading] = useState(true); + const [showCanvas, setShowCanvas] = useState(false); const onVideoError = useCallback>((error) => { console.error('video error', error); }, []); - const state = useMemo(() => (playing - ? { startTime: commandedTime, playing, eventId } - : { startTime: playerTime, playing, eventId } - ), [commandedTime, eventId, playerTime, playing]); - const audioStreamIndexes = useMemo(() => audioStreams.map((s) => s.index), [audioStreams]); - const [debouncedState] = useDebounce(state, 300, { - equalityFn: (a, b) => a.startTime === b.startTime && a.playing === b.playing && a.eventId === b.eventId, - leading: true, - }); - - useEffect(() => { - // console.log('debouncedState', debouncedState); - }, [debouncedState]); - - const onPlayRequested = useCallback(async () => { - try { - await videoRef.current?.play(); - } catch (err) { - console.error('play failed', err); - } - }, []); - useEffect(() => { - setLoading(true); - - let { startTime: seekTo } = debouncedState; - const video = videoRef.current; invariant(video != null); const masterVideo = masterVideoRef.current; invariant(masterVideo != null); - if (seekTo >= masterVideo.duration) seekTo = 0; // start over if reached end + const canvas = canvasRef.current; + invariant(canvas != null); - if (seekTo == null) { - return () => undefined; - } + let abortController: AbortController; + let startDebounced: () => void; - const onCanPlay = () => { - setLoading(false); - }; - const getTargetTime = () => masterVideoRef.current!.currentTime - seekTo; + const start = async () => { + abortController = new AbortController(); - const abortController = new AbortController(); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + const ctx = canvas.getContext('2d'); + ctx?.drawImage(video, 0, 0, canvas.width, canvas.height); + setShowCanvas(true); + setLoading(true); - (async () => { - try { - // When playing, we use a secondary video element, but when paused we use an img - if (debouncedState.playing) { - let size: number | undefined; - if (videoStream != null) { - if (mediaSourceQuality === 0) size = 800; - else if (mediaSourceQuality === 1) size = 420; - } + const seekTo = masterVideo.currentTime; - let fps: number | undefined; - if (mediaSourceQuality === 0) fps = 30; - else if (mediaSourceQuality === 1) fps = 15; + try { + let size: number | undefined; + if (videoStream != null) { + if (mediaSourceQuality === 0) size = 800; + else if (mediaSourceQuality === 1) size = 420; + } - await startPlayback({ signal: abortController.signal, path: filePath, video, videoStreamIndex: videoStream?.index, audioStreamIndexes, seekTo, onPlayRequested, onCanPlay, getTargetTime, size, fps, rotate }); - } else if (videoStream != null) { // paused - try { - await createPauseImage({ signal: abortController.signal, path: filePath, videoStreamIndex: videoStream.index, seekTo, image: imgRef.current, rotate }); - } finally { + let fps: number | undefined; + if (mediaSourceQuality === 0) fps = 30; + else if (mediaSourceQuality === 1) fps = 15; + + await startPlayback({ + signal: abortController.signal, + path: filePath, + slaveVideo: video, + masterVideo, + videoStreamIndex: videoStream?.index, + audioStreamIndexes, + seekTo, + size, + fps, + rotate, + onCanPlay: () => { setLoading(false); - } - } + setShowCanvas(false); + }, + onResetNeeded: () => { + abortController.abort(); + startDebounced(); + }, + onWaiting: () => { + setLoading(true); + }, + }); } catch (err) { console.error('Preview failed', err); } - })(); + }; + + startDebounced = debounce(start, 500, { leading: true, trailing: true }); + + startDebounced(); return () => abortController.abort(); // Important that we also have eventId in the deps, so that we can restart the preview when the eventId changes - }, [debouncedState.startTime, debouncedState.eventId, filePath, masterVideoRef, onPlayRequested, debouncedState.playing, videoStream, mediaSourceQuality, audioStreamIndexes, rotate, debouncedState]); - - useEffect(() => { - if (videoRef.current) videoRef.current.volume = playbackVolume; - }, [playbackVolume]); + }, [audioStreamIndexes, filePath, masterVideoRef, mediaSourceQuality, rotate, videoStream]); - const onFocus = useCallback>((e) => { + const onFocus = useCallback>((e) => { // prevent video element from stealing focus in fullscreen mode https://github.com/mifi/lossless-cut/issues/543#issuecomment-1868167775 e.target.blur(); }, []); - const { videoStyle, imgStyle } = useMemo<{ videoStyle: CSSProperties, imgStyle: CSSProperties }>(() => { - const sharedStyle: CSSProperties = { - position: 'absolute', left: 0, right: 0, top: 0, bottom: 0, display: 'block', width: '100%', height: '100%', objectFit: 'contain', transform: rotate ? `rotate(${rotate}deg)` : undefined, - }; - - const shouldShowImage = !debouncedState.playing; - - return { - videoStyle: { ...sharedStyle, visibility: loading || !debouncedState.playing ? 'hidden' : undefined }, - imgStyle: { ...sharedStyle, visibility: shouldShowImage ? undefined : 'hidden' }, - }; - }, [loading, debouncedState.playing, rotate]); + const videoStyle = useMemo(() => ({ + position: 'absolute', left: 0, right: 0, top: 0, bottom: 0, display: 'block', width: '100%', height: '100%', objectFit: 'contain', transform: rotate ? `rotate(${rotate}deg)` : undefined, + }), [rotate]); return (
{/* eslint-disable-next-line jsx-a11y/media-has-caption */} -