improve compat player

pull/2352/head
Mikael Finstad 9 months ago
parent 25101299d4
commit 2510135db4
No known key found for this signature in database
GPG Key ID: 25AB36E3E81CBC26

@ -1,10 +1,10 @@
import assert from 'node:assert';
import { ExecaError } from 'execa';
import logger from './logger.js';
import { createMediaSourceProcess, readOneJpegFrame as readOneJpegFrameRaw } from './ffmpeg.js';
import { createMediaSourceProcess } from './ffmpeg.js';
// eslint-disable-next-line import/prefer-default-export
export function createMediaSourceStream(params: Parameters<typeof createMediaSourceProcess>[0]) {
const abortController = new AbortController();
const { videoStreamIndex, audioStreamIndexes, seekTo } = params;
@ -18,7 +18,6 @@ export function createMediaSourceStream(params: Parameters<typeof createMediaSou
};
const { stdout } = process;
assert(stdout != null);
stdout.pause();
@ -29,6 +28,8 @@ export function createMediaSourceStream(params: Parameters<typeof createMediaSou
cleanup();
resolve(null);
};
// poor man's backpressure handling: we only read one chunk at a time
const onData = (chunk: Buffer) => {
stdout.pause();
cleanup();
@ -75,28 +76,3 @@ export function createMediaSourceStream(params: Parameters<typeof createMediaSou
return { abort, readChunk };
}
export function readOneJpegFrame(params: Parameters<typeof readOneJpegFrameRaw>[0]) {
const abortController = new AbortController();
const process = readOneJpegFrameRaw(params);
// eslint-disable-next-line unicorn/prefer-add-event-listener
abortController.signal.onabort = () => process.kill('SIGKILL');
function abort() {
abortController.abort();
}
const promise = (async () => {
try {
const { stdout } = await process;
return stdout;
} catch (err) {
// @ts-expect-error todo
logger.error('renderOneJpegFrame', err.shortMessage);
throw new Error('Failed to render JPEG frame');
}
})();
return { promise, abort };
}

@ -532,34 +532,6 @@ export async function getDuration(filePath: string) {
return parseFloat((await readFormatData(filePath)).duration);
}
export function readOneJpegFrame({ path, seekTo, videoStreamIndex, rotate }: {
path: string,
seekTo: number,
videoStreamIndex: number,
rotate: number | undefined,
}) {
const args = [
'-hide_banner', '-loglevel', 'error',
'-ss', String(seekTo),
...(rotate != null ? ['-noautorotate'] : []),
'-i', path,
'-map', `0:${videoStreamIndex}`,
'-vcodec', 'mjpeg',
'-frames:v', '1',
'-f', 'image2pipe',
'-',
];
logger.info(getFfCommandLine('ffmpeg', args));
return runFfmpegProcess(args, undefined, { logCli: false });
}
const enableLog = false;
const encode = true;
@ -602,6 +574,18 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn
return ['-filter_complex', graph.join(';')];
}
const videoEncodeArgs = [
'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', '-crf', '10',
// should, in theory, set the private options of x264 or x265 automatically
// https://www.reddit.com/r/ffmpeg/comments/jlk2zn/how_to_encode_using_bt709/
'-colorspace:v', 'bt709', '-color_primaries:v', 'bt709', '-color_trc:v', 'bt709', '-color_range:v', 'tv',
'-pix_fmt:v', 'yuv420p',
];
// const videoEncodeArgs = ['h264_videotoolbox', '-b:v', '5M']
// https://stackoverflow.com/questions/16658873/how-to-minimize-the-delay-in-a-live-streaming-with-ffmpeg
// https://unix.stackexchange.com/questions/25372/turn-off-buffering-in-pipe
const args = [
@ -625,12 +609,16 @@ export function createMediaSourceProcess({ path, videoStreamIndex, audioStreamIn
'-fps_mode', 'passthrough',
'-map_metadata', '-1',
'-map_chapters', '-1',
...(encode ? [
...getFilters(),
...(videoStreamIndex != null ? [
'-map', '[video]',
'-pix_fmt', 'yuv420p', '-c:v', 'libx264', '-preset', 'ultrafast', '-tune', 'zerolatency', '-crf', '10',
'-c:v', ...videoEncodeArgs,
'-g', '1', // reduces latency and buffering
] : ['-vn']),

@ -141,7 +141,7 @@ function App() {
const [activeVideoStreamIndex, setActiveVideoStreamIndex] = useState<number>();
const [activeAudioStreamIndexes, setActiveAudioStreamIndexes] = useState<Set<number>>(new Set());
const [activeSubtitleStreamIndex, setActiveSubtitleStreamIndex] = useState<number>();
const [hideMediaSourcePlayer, setHideMediaSourcePlayer] = useState(false);
const [hideCompatPlayer, setHideCompatPlayer] = useState(false);
const [exportConfirmVisible, setExportConfirmVisible] = useState(false);
const [cacheBuster, setCacheBuster] = useState(0);
const [currentFileExportCount, setCurrentFileExportCount] = useState(0);
@ -184,7 +184,7 @@ function App() {
const keyBindingByAction = useMemo(() => Object.fromEntries(keyBindings.map((binding) => [binding.action, binding])) as Record<KeyboardAction, KeyBinding>, [keyBindings]);
const { working, setWorking, workingRef, abortWorking } = useLoading();
const { videoRef, videoContainerRef, playbackRate, setPlaybackRate, outputPlaybackRate, setOutputPlaybackRate, commandedTime, seekAbs, playingRef, getRelevantTime, setPlaying, onSeeked, relevantTime, onStartPlaying, setCommandedTime, setCompatPlayerEventId, compatPlayerEventId, setOutputPlaybackRateState, commandedTimeRef, onStopPlaying, onVideoAbort, playerTime, setPlayerTime, playbackModeRef, playing, play, pause, seekRel } = useVideo({ filePath });
const { videoRef, videoContainerRef, playbackRate, setPlaybackRate, outputPlaybackRate, setOutputPlaybackRate, commandedTime, seekAbs, playingRef, getRelevantTime, setPlaying, onSeeked, relevantTime, onStartPlaying, setCommandedTime, setOutputPlaybackRateState, commandedTimeRef, onStopPlaying, onVideoAbort, playerTime, setPlayerTime, playbackModeRef, playing, play, pause, seekRel } = useVideo({ filePath });
const { timecodePlaceholder, formatTimecode, formatTimeAndFrames, parseTimecode, getFrameCount } = useTimecode({ detectedFps, timecodeFormat });
const { loadSubtitle, subtitlesByStreamId, setSubtitlesByStreamId } = useSubtitles();
@ -319,10 +319,8 @@ function App() {
const zoomAbs = useCallback((fn: (v: number) => number) => setZoom((z) => Math.min(Math.max(fn(z), 1), zoomMax)), []);
const zoomRel = useCallback((rel: number) => zoomAbs((z) => z + (rel * (1 + (z / 10)))), [zoomAbs]);
const compatPlayerRequired = (
// when using html5ified dummy video, we *have* to use canvas player
usingDummyVideo
// or if user selected an explicit video or audio stream, and the html5 player does not have any track index corresponding to the selected stream index
|| (
// if user selected an explicit video or audio stream, and the html5 player does not have any track index corresponding to the selected stream index
(
(activeVideoStreamIndex != null || activeAudioStreamIndexes.size === 1)
&& videoRef.current != null
&& !canHtml5PlayerPlayStreams(videoRef.current, activeVideoStreamIndex, [...activeAudioStreamIndexes][0])
@ -332,7 +330,8 @@ function App() {
);
// if user selected a rotation, but they might want to turn off the rotation preview
// but allow the user to disable
const compatPlayerWanted = isRotationSet && !hideMediaSourcePlayer;
const compatPlayerWanted = (isRotationSet && !hideCompatPlayer)
|| usingDummyVideo;
const compatPlayerEnabled = (compatPlayerRequired || compatPlayerWanted) && (activeVideoStream != null || activeAudioStreams.length > 0);
@ -340,7 +339,7 @@ function App() {
useEffect(() => {
// Reset the user preference when we go from not having compat player to having it
if (compatPlayerEnabled) setHideMediaSourcePlayer(false);
if (compatPlayerEnabled) setHideCompatPlayer(false);
}, [compatPlayerEnabled]);
const comfortZoom = isDurationValid(fileDuration) ? Math.max(fileDuration / 100, 1) : undefined;
@ -441,7 +440,7 @@ function App() {
const increaseRotation = useCallback(() => {
setRotation((r) => (r + 90) % 450);
setHideMediaSourcePlayer(false);
setHideCompatPlayer(false);
// Matroska is known not to work, so we warn user. See https://github.com/mifi/lossless-cut/discussions/661
const supportsRotation = !isMatroska(fileFormat);
if (!supportsRotation) showNotification({ text: i18n.t('Lossless rotation might not work with this file format. You may try changing to MP4') });
@ -554,14 +553,14 @@ function App() {
const onActiveVideoStreamChange = useCallback((videoStreamIndex?: number) => {
invariant(videoRef.current);
setHideMediaSourcePlayer(false);
setHideCompatPlayer(false);
enableVideoTrack(videoRef.current, videoStreamIndex);
setActiveVideoStreamIndex(videoStreamIndex);
}, [videoRef]);
const onActiveAudioStreamsChange = useCallback((audioStreamIndexes: Set<number>) => {
invariant(videoRef.current);
setHideMediaSourcePlayer(false);
setHideCompatPlayer(false);
enableAudioTrack(videoRef.current, [...audioStreamIndexes][0]);
setActiveAudioStreamIndexes(audioStreamIndexes);
}, [videoRef]);
@ -618,7 +617,6 @@ function App() {
setPlaying(false);
playingRef.current = false;
playbackModeRef.current = undefined;
setCompatPlayerEventId(0);
setFileDuration(undefined);
cutSegmentsHistory.go(0);
setFileFormat(undefined);
@ -642,11 +640,11 @@ function App() {
setActiveAudioStreamIndexes(new Set());
setActiveVideoStreamIndex(undefined);
setActiveSubtitleStreamIndex(undefined);
setHideMediaSourcePlayer(false);
setHideCompatPlayer(false);
setExportConfirmVisible(false);
setOutputPlaybackRateState(1);
setCurrentFileExportCount(0);
}, [videoRef, setCommandedTime, setPlaybackRate, setPlaying, playingRef, playbackModeRef, setCompatPlayerEventId, setFileDuration, cutSegmentsHistory, setFileFormat, setDetectedFileFormat, setCopyStreamIdsByFile, setThumbnails, setSubtitlesByStreamId, setOutputPlaybackRateState]);
}, [videoRef, setCommandedTime, setPlaybackRate, setPlaying, playingRef, playbackModeRef, setFileDuration, cutSegmentsHistory, setFileFormat, setDetectedFileFormat, setCopyStreamIdsByFile, setThumbnails, setSubtitlesByStreamId, setOutputPlaybackRateState]);
const showUnsupportedFileMessage = useCallback(() => {
@ -699,6 +697,12 @@ function App() {
setUsingDummyVideo(usesDummyVideo);
}, [html5ify, html5ifyDummy]);
const handleHideCompatPlayerClick = useCallback(() => {
setHideCompatPlayer(true);
setPreviewFilePath(undefined);
setUsingDummyVideo(false);
}, []);
const convertFormatBatch = useCallback(async () => {
if (batchFiles.length === 0) return;
const filePaths = batchFiles.map((f) => f.path);
@ -1326,11 +1330,6 @@ function App() {
const extractSelectedSegmentsFramesAsImages = useCallback(() => extractSegmentsFramesAsImages(selectedSegments), [extractSegmentsFramesAsImages, selectedSegments]);
const userChangePlaybackRate = useCallback((dir: number, rateMultiplier?: number) => {
if (compatPlayerEnabled) {
toast.fire({ title: i18n.t('Unable to change playback rate right now'), timer: 1000 });
return;
}
const video = videoRef.current;
if (!playingRef.current) {
video!.play();
@ -1338,7 +1337,7 @@ function App() {
const newRate = adjustRate(video!.playbackRate, dir, rateMultiplier);
setPlaybackRate(newRate);
}
}, [compatPlayerEnabled, playingRef, setPlaybackRate, videoRef]);
}, [playingRef, setPlaybackRate, videoRef]);
const loadEdlFile = useCallback(async ({ path, type, append = false }: { path: string, type: EdlFileType, append?: boolean }) => {
console.log('Loading EDL file', type, path, append);
@ -1497,7 +1496,7 @@ function App() {
}
// only show one toast, or else we will only show the last one
if (existingHtml5FriendlyFile) {
if (existingHtml5FriendlyFile && !existingHtml5FriendlyFile.usingDummyVideo) {
showPreviewFileLoadedMessage(basename(existingHtml5FriendlyFile.path));
} else if (needsAutoHtml5ify) {
showUnsupportedFileMessage();
@ -2518,321 +2517,321 @@ function App() {
return (
<>
<AppContext.Provider value={appContext}>
<SegColorsContext.Provider value={segColorsContext}>
<UserSettingsContext.Provider value={userSettingsContext}>
<ThemeProvider value={theme}>
<div className={darkMode ? 'dark-theme' : undefined} style={{ ...baseColorStyle, display: 'flex', flexDirection: 'column', height: '100vh', transition: darkModeTransition }}>
<TopMenu
filePath={filePath}
fileFormat={fileFormat}
changeEnabledStreamsFilter={changeEnabledStreamsFilter}
applyEnabledStreamsFilter={applyEnabledStreamsFilter}
enabledStreamsFilter={enabledStreamsFilter}
clearOutDir={clearOutDir}
isCustomFormatSelected={isCustomFormatSelected}
renderOutFmt={renderOutFmt}
toggleSettings={toggleSettings}
numStreamsToCopy={numStreamsToCopy}
numStreamsTotal={numStreamsTotal}
setStreamsSelectorShown={setStreamsSelectorShown}
selectedSegments={segmentsOrInverse.selected}
/>
<div style={{ flexGrow: 1, display: 'flex', overflowY: 'hidden' }}>
<AnimatePresence>
{showLeftBar && (
<BatchFilesList
selectedBatchFiles={selectedBatchFiles}
filePath={filePath}
width={leftBarWidth}
batchFiles={batchFiles}
setBatchFiles={setBatchFiles}
onBatchFileSelect={onBatchFileSelect}
batchListRemoveFile={batchListRemoveFile}
closeBatch={closeBatch}
onMergeFilesClick={concatBatch}
onBatchConvertToSupportedFormatClick={convertFormatBatch}
onDrop={handleBatchFilesDrop}
/>
)}
</AnimatePresence>
{/* Middle part (also shown in fullscreen): */}
<div style={{ position: 'relative', flexGrow: 1, overflow: 'hidden' }} ref={videoContainerRef}>
{!isFileOpened && <NoFileLoaded mifiLink={mifiLink} currentCutSeg={currentCutSeg} onClick={openFilesDialog} darkMode={darkMode} keyBindingByAction={keyBindingByAction} />}
<div className="no-user-select" style={{ position: 'absolute', top: 0, left: 0, right: 0, bottom: 0, visibility: !isFileOpened || !hasVideo || bigWaveformEnabled ? 'hidden' : undefined }} onWheel={onTimelineWheel}>
{/* eslint-disable-next-line jsx-a11y/media-has-caption */}
<video
className={styles['video']}
tabIndex={-1}
muted={playbackVolume === 0 || compatPlayerEnabled}
ref={videoRef}
style={videoStyle}
src={fileUri}
onPlay={onStartPlaying}
onPause={onStopPlaying}
onAbort={onVideoAbort}
onDurationChange={onDurationChange}
onTimeUpdate={onTimeUpdate}
onError={onVideoError}
onClick={onVideoClick}
onDoubleClick={toggleFullscreenVideo}
onFocusCapture={onVideoFocus}
onSeeked={onSeeked}
>
{renderSubtitles()}
</video>
{filePath != null && compatPlayerEnabled && <MediaSourcePlayer rotate={effectiveRotation} filePath={filePath} videoStream={activeVideoStream} audioStreams={activeAudioStreams} playerTime={playerTime ?? 0} commandedTime={commandedTime} playing={playing} eventId={compatPlayerEventId} masterVideoRef={videoRef} mediaSourceQuality={mediaSourceQuality} playbackVolume={playbackVolume} />}
</div>
<SegColorsContext.Provider value={segColorsContext}>
<UserSettingsContext.Provider value={userSettingsContext}>
<ThemeProvider value={theme}>
<div className={darkMode ? 'dark-theme' : undefined} style={{ ...baseColorStyle, display: 'flex', flexDirection: 'column', height: '100vh', transition: darkModeTransition }}>
<TopMenu
filePath={filePath}
fileFormat={fileFormat}
changeEnabledStreamsFilter={changeEnabledStreamsFilter}
applyEnabledStreamsFilter={applyEnabledStreamsFilter}
enabledStreamsFilter={enabledStreamsFilter}
clearOutDir={clearOutDir}
isCustomFormatSelected={isCustomFormatSelected}
renderOutFmt={renderOutFmt}
toggleSettings={toggleSettings}
numStreamsToCopy={numStreamsToCopy}
numStreamsTotal={numStreamsTotal}
setStreamsSelectorShown={setStreamsSelectorShown}
selectedSegments={segmentsOrInverse.selected}
/>
{bigWaveformEnabled && <BigWaveform waveforms={waveforms} relevantTime={relevantTime} playing={playing} fileDurationNonZero={fileDurationNonZero} zoom={zoomUnrounded} seekRel={seekRel} darkMode={darkMode} />}
<div style={{ flexGrow: 1, display: 'flex', overflowY: 'hidden' }}>
<AnimatePresence>
{showLeftBar && (
<BatchFilesList
selectedBatchFiles={selectedBatchFiles}
filePath={filePath}
width={leftBarWidth}
batchFiles={batchFiles}
setBatchFiles={setBatchFiles}
onBatchFileSelect={onBatchFileSelect}
batchListRemoveFile={batchListRemoveFile}
closeBatch={closeBatch}
onMergeFilesClick={concatBatch}
onBatchConvertToSupportedFormatClick={convertFormatBatch}
onDrop={handleBatchFilesDrop}
/>
)}
</AnimatePresence>
{/* Middle part (also shown in fullscreen): */}
<div style={{ position: 'relative', flexGrow: 1, overflow: 'hidden' }} ref={videoContainerRef}>
{!isFileOpened && <NoFileLoaded mifiLink={mifiLink} currentCutSeg={currentCutSeg} onClick={openFilesDialog} darkMode={darkMode} keyBindingByAction={keyBindingByAction} />}
<div className="no-user-select" style={{ position: 'absolute', top: 0, left: 0, right: 0, bottom: 0, visibility: !isFileOpened || !hasVideo || bigWaveformEnabled ? 'hidden' : undefined }} onWheel={onTimelineWheel}>
{/* eslint-disable-next-line jsx-a11y/media-has-caption */}
<video
className={styles['video']}
tabIndex={-1}
muted={playbackVolume === 0 || compatPlayerEnabled}
ref={videoRef}
style={videoStyle}
src={fileUri}
onPlay={onStartPlaying}
onPause={onStopPlaying}
onAbort={onVideoAbort}
onDurationChange={onDurationChange}
onTimeUpdate={onTimeUpdate}
onError={onVideoError}
onClick={onVideoClick}
onDoubleClick={toggleFullscreenVideo}
onFocusCapture={onVideoFocus}
onSeeked={onSeeked}
>
{renderSubtitles()}
</video>
{filePath != null && compatPlayerEnabled && <MediaSourcePlayer rotate={effectiveRotation} filePath={filePath} videoStream={activeVideoStream} audioStreams={activeAudioStreams} masterVideoRef={videoRef} mediaSourceQuality={mediaSourceQuality} />}
</div>
{compatPlayerEnabled && (
<div style={{ position: 'absolute', top: 0, right: 0, left: 0, marginTop: '1em', marginLeft: '1em', color: 'white', opacity: 0.7, display: 'flex', alignItems: 'center', pointerEvents: 'none' }}>
{isRotationSet ? (
<>
<MdRotate90DegreesCcw size={26} style={{ marginRight: 5 }} />
{t('Rotation preview')}
</>
) : (
<>
{t('FFmpeg-assisted playback')}
</>
)}
{bigWaveformEnabled && <BigWaveform waveforms={waveforms} relevantTime={relevantTime} playing={playing} fileDurationNonZero={fileDurationNonZero} zoom={zoomUnrounded} seekRel={seekRel} darkMode={darkMode} />}
{compatPlayerEnabled && (
<div style={{ position: 'absolute', top: 0, right: 0, left: 0, marginTop: '1em', marginLeft: '1em', color: 'white', opacity: 0.7, display: 'flex', alignItems: 'center', pointerEvents: 'none' }}>
{isRotationSet ? (
<>
<MdRotate90DegreesCcw size={26} style={{ marginRight: 5 }} />
{t('Rotation preview')}
</>
) : (
<>
{t('FFmpeg-assisted playback')}
</>
)}
<div style={{ cursor: 'pointer', pointerEvents: 'initial', color: 'white', opacity: 0.7, padding: '.2em', marginLeft: '.5em' }} role="button" onClick={() => incrementMediaSourceQuality()} title={t('Select playback quality')}>{mediaSourceQualities[mediaSourceQuality]}</div>
{!compatPlayerRequired && <FaRegTimesCircle role="button" style={{ cursor: 'pointer', pointerEvents: 'initial', verticalAlign: 'middle', padding: '.2em' }} onClick={handleHideCompatPlayerClick} />}
</div>
)}
{isFileOpened && (
<div className="no-user-select" style={{ position: 'absolute', right: 0, bottom: 0, marginBottom: 10, display: 'flex', alignItems: 'flex-end' }}>
<VolumeControl playbackVolume={playbackVolume} setPlaybackVolume={setPlaybackVolume} onToggleMutedClick={toggleMuted} />
{shouldShowPlaybackStreamSelector && (
<PlaybackStreamSelector subtitleStreams={subtitleStreams} videoStreams={videoStreams} audioStreams={audioStreams} activeSubtitleStreamIndex={activeSubtitleStreamIndex} activeVideoStreamIndex={activeVideoStreamIndex} activeAudioStreamIndexes={activeAudioStreamIndexes} onActiveSubtitleChange={onActiveSubtitleChange} onActiveVideoStreamChange={onActiveVideoStreamChange} onActiveAudioStreamsChange={onActiveAudioStreamsChange} />
)}
{!showRightBar && (
<FaAngleLeft
title={t('Show sidebar')}
size={30}
role="button"
style={{ marginRight: 10, color: 'var(--gray-12)', opacity: 0.7 }}
onClick={toggleSegmentsList}
/>
)}
</div>
)}
</div>
<div style={{ cursor: 'pointer', pointerEvents: 'initial', color: 'white', opacity: 0.7, padding: '.2em', marginLeft: '.5em' }} role="button" onClick={() => incrementMediaSourceQuality()} title={t('Select playback quality')}>{mediaSourceQualities[mediaSourceQuality]}</div>
<AnimatePresence>
{showRightBar && isFileOpened && filePath != null && (
<SegmentList
width={rightBarWidth}
currentSegIndex={currentSegIndexSafe}
cutSegments={cutSegments}
inverseCutSegments={inverseCutSegments}
getFrameCount={getFrameCount}
formatTimecode={formatTimecode}
onSegClick={setCurrentSegIndex}
updateSegOrder={updateSegOrder}
updateSegOrders={updateSegOrders}
onLabelSegment={labelSegment}
currentCutSeg={currentCutSeg}
firstSegmentAtCursor={firstSegmentAtCursor}
addSegment={addSegment}
onDuplicateSegmentClick={duplicateSegment}
removeSegment={removeSegment}
onRemoveSelected={removeSelectedSegments}
toggleSegmentsList={toggleSegmentsList}
splitCurrentSegment={splitCurrentSegment}
selectedSegments={segmentsOrInverse.selected}
onSelectSingleSegment={selectOnlySegment}
onToggleSegmentSelected={toggleSegmentSelected}
onDeselectAllSegments={deselectAllSegments}
onSelectAllSegments={selectAllSegments}
onInvertSelectedSegments={invertSelectedSegments}
onExtractSegmentsFramesAsImages={extractSegmentsFramesAsImages}
onExtractSelectedSegmentsFramesAsImages={extractSelectedSegmentsFramesAsImages}
jumpSegStart={jumpSegStart}
jumpSegEnd={jumpSegEnd}
onSelectSegmentsByLabel={selectSegmentsByLabel}
onSelectSegmentsByExpr={selectSegmentsByExpr}
onSelectAllMarkers={selectAllMarkers}
onMutateSegmentsByExpr={mutateSegmentsByExpr}
onLabelSelectedSegments={labelSelectedSegments}
updateSegAtIndex={updateSegAtIndex}
editingSegmentTags={editingSegmentTags}
editingSegmentTagsSegmentIndex={editingSegmentTagsSegmentIndex}
setEditingSegmentTags={setEditingSegmentTags}
setEditingSegmentTagsSegmentIndex={setEditingSegmentTagsSegmentIndex}
onEditSegmentTags={onEditSegmentTags}
/>
)}
</AnimatePresence>
</div>
{!compatPlayerRequired && <FaRegTimesCircle role="button" style={{ cursor: 'pointer', pointerEvents: 'initial', verticalAlign: 'middle', padding: '.2em' }} onClick={() => setHideMediaSourcePlayer(true)} />}
</div>
)}
<div className="no-user-select" style={bottomStyle}>
<Timeline
shouldShowKeyframes={shouldShowKeyframes}
waveforms={waveforms}
overviewWaveform={overviewWaveform}
shouldShowWaveform={shouldShowWaveform}
waveformEnabled={waveformEnabled}
waveformHeight={waveformHeight}
showThumbnails={showThumbnails}
neighbouringKeyFrames={neighbouringKeyFrames}
thumbnails={thumbnailsSorted}
playerTime={playerTime}
commandedTime={commandedTime}
relevantTime={relevantTime}
commandedTimeRef={commandedTimeRef}
startTimeOffset={startTimeOffset}
zoom={zoom}
seekAbs={seekAbs}
fileDurationNonZero={fileDurationNonZero}
cutSegments={cutSegments}
setCurrentSegIndex={setCurrentSegIndex}
currentSegIndexSafe={currentSegIndexSafe}
inverseCutSegments={inverseCutSegments}
formatTimecode={formatTimecode}
formatTimeAndFrames={formatTimeAndFrames}
zoomWindowStartTime={zoomWindowStartTime}
zoomWindowEndTime={zoomWindowEndTime}
onZoomWindowStartTimeChange={setZoomWindowStartTime}
onGenerateOverviewWaveformClick={onGenerateOverviewWaveformClick}
playing={playing}
isFileOpened={isFileOpened}
onWheel={onTimelineWheel}
goToTimecode={goToTimecode}
darkMode={darkMode}
/>
{isFileOpened && (
<div className="no-user-select" style={{ position: 'absolute', right: 0, bottom: 0, marginBottom: 10, display: 'flex', alignItems: 'flex-end' }}>
<VolumeControl playbackVolume={playbackVolume} setPlaybackVolume={setPlaybackVolume} onToggleMutedClick={toggleMuted} />
{shouldShowPlaybackStreamSelector && (
<PlaybackStreamSelector subtitleStreams={subtitleStreams} videoStreams={videoStreams} audioStreams={audioStreams} activeSubtitleStreamIndex={activeSubtitleStreamIndex} activeVideoStreamIndex={activeVideoStreamIndex} activeAudioStreamIndexes={activeAudioStreamIndexes} onActiveSubtitleChange={onActiveSubtitleChange} onActiveVideoStreamChange={onActiveVideoStreamChange} onActiveAudioStreamsChange={onActiveAudioStreamsChange} />
)}
{!showRightBar && (
<FaAngleLeft
title={t('Show sidebar')}
size={30}
role="button"
style={{ marginRight: 10, color: 'var(--gray-12)', opacity: 0.7 }}
onClick={toggleSegmentsList}
/>
)}
</div>
)}
<BottomBar
zoom={zoom}
setZoom={zoomAbs}
timelineToggleComfortZoom={timelineToggleComfortZoom}
hasVideo={hasVideo}
isRotationSet={isRotationSet}
rotation={rotation}
areWeCutting={areWeCutting}
increaseRotation={increaseRotation}
cleanupFilesDialog={cleanupFilesDialog}
captureSnapshot={captureSnapshot}
onExportPress={onExportPress}
segmentsToExport={segmentsToExport}
seekAbs={seekAbs}
currentSegIndexSafe={currentSegIndexSafe}
cutSegments={cutSegments}
currentCutSeg={currentCutSeg}
selectedSegments={selectedSegments}
setCutStart={setCutStart}
setCutEnd={setCutEnd}
setCurrentSegIndex={setCurrentSegIndex}
jumpCutEnd={jumpCutEnd}
jumpCutStart={jumpCutStart}
jumpTimelineStart={jumpTimelineStart}
jumpTimelineEnd={jumpTimelineEnd}
startTimeOffset={startTimeOffset}
setCutTime={setCutTime}
playing={playing}
shortStep={shortStep}
seekClosestKeyframe={seekClosestKeyframe}
togglePlay={togglePlay}
showThumbnails={showThumbnails}
toggleShowThumbnails={toggleShowThumbnails}
toggleWaveformMode={toggleWaveformMode}
waveformMode={waveformMode}
hasAudio={hasAudio}
keyframesEnabled={keyframesEnabled}
toggleShowKeyframes={toggleShowKeyframes}
detectedFps={detectedFps}
toggleLoopSelectedSegments={toggleLoopSelectedSegments}
isFileOpened={isFileOpened}
darkMode={darkMode}
toggleDarkMode={toggleDarkMode}
outputPlaybackRate={outputPlaybackRate}
setOutputPlaybackRate={setOutputPlaybackRate}
formatTimecode={formatTimecode}
parseTimecode={parseTimecode}
playbackRate={playbackRate}
currentFrame={currentFrame}
/>
</div>
<AnimatePresence>
{showRightBar && isFileOpened && filePath != null && (
<SegmentList
width={rightBarWidth}
currentSegIndex={currentSegIndexSafe}
cutSegments={cutSegments}
inverseCutSegments={inverseCutSegments}
getFrameCount={getFrameCount}
{tunerVisible != null && <ValueTuners type={tunerVisible} onFinished={() => setTunerVisible(undefined)} />}
{/* Dialogs */}
<ExportConfirm areWeCutting={areWeCutting} segmentsOrInverse={segmentsOrInverse} segmentsToExport={segmentsToExport} willMerge={willMerge} visible={exportConfirmVisible} onClosePress={closeExportConfirm} onExportConfirm={onExportConfirm} renderOutFmt={renderOutFmt} outputDir={outputDir} numStreamsTotal={numStreamsTotal} numStreamsToCopy={numStreamsToCopy} onShowStreamsSelectorClick={handleShowStreamsSelectorClick} outFormat={fileFormat} setOutSegTemplate={setOutSegTemplate} outSegTemplate={outSegTemplateOrDefault} mergedFileTemplate={mergedFileTemplateOrDefault} setMergedFileTemplate={setMergedFileTemplate} generateOutSegFileNames={generateOutSegFileNames} generateMergedFileNames={generateMergedFileNames} currentSegIndexSafe={currentSegIndexSafe} mainCopiedThumbnailStreams={mainCopiedThumbnailStreams} needSmartCut={needSmartCut} isEncoding={isEncoding} encBitrate={encBitrate} setEncBitrate={setEncBitrate} toggleSettings={toggleSettings} outputPlaybackRate={outputPlaybackRate} lossyMode={lossyMode} />
<Sheet visible={streamsSelectorShown} onClosePress={() => setStreamsSelectorShown(false)} maxWidth={1000}>
{mainStreams && filePath != null && (
<StreamsSelector
mainFilePath={filePath}
mainFileFormatData={mainFileFormatData}
mainFileChapters={mainFileChapters}
allFilesMeta={allFilesMeta}
externalFilesMeta={externalFilesMeta}
setExternalFilesMeta={setExternalFilesMeta}
showAddStreamSourceDialog={showIncludeExternalStreamsDialog}
mainFileStreams={mainStreams}
isCopyingStreamId={isCopyingStreamId}
toggleCopyStreamId={toggleCopyStreamId}
setCopyStreamIdsForPath={setCopyStreamIdsForPath}
onExtractAllStreamsPress={extractAllStreams}
onExtractStreamPress={extractSingleStream}
shortestFlag={shortestFlag}
setShortestFlag={setShortestFlag}
nonCopiedExtraStreams={nonCopiedExtraStreams}
customTagsByFile={customTagsByFile}
setCustomTagsByFile={setCustomTagsByFile}
paramsByStreamId={paramsByStreamId}
updateStreamParams={updateStreamParams}
formatTimecode={formatTimecode}
onSegClick={setCurrentSegIndex}
updateSegOrder={updateSegOrder}
updateSegOrders={updateSegOrders}
onLabelSegment={labelSegment}
currentCutSeg={currentCutSeg}
firstSegmentAtCursor={firstSegmentAtCursor}
addSegment={addSegment}
onDuplicateSegmentClick={duplicateSegment}
removeSegment={removeSegment}
onRemoveSelected={removeSelectedSegments}
toggleSegmentsList={toggleSegmentsList}
splitCurrentSegment={splitCurrentSegment}
selectedSegments={segmentsOrInverse.selected}
onSelectSingleSegment={selectOnlySegment}
onToggleSegmentSelected={toggleSegmentSelected}
onDeselectAllSegments={deselectAllSegments}
onSelectAllSegments={selectAllSegments}
onInvertSelectedSegments={invertSelectedSegments}
onExtractSegmentsFramesAsImages={extractSegmentsFramesAsImages}
onExtractSelectedSegmentsFramesAsImages={extractSelectedSegmentsFramesAsImages}
jumpSegStart={jumpSegStart}
jumpSegEnd={jumpSegEnd}
onSelectSegmentsByLabel={selectSegmentsByLabel}
onSelectSegmentsByExpr={selectSegmentsByExpr}
onSelectAllMarkers={selectAllMarkers}
onMutateSegmentsByExpr={mutateSegmentsByExpr}
onLabelSelectedSegments={labelSelectedSegments}
updateSegAtIndex={updateSegAtIndex}
editingSegmentTags={editingSegmentTags}
editingSegmentTagsSegmentIndex={editingSegmentTagsSegmentIndex}
setEditingSegmentTags={setEditingSegmentTags}
setEditingSegmentTagsSegmentIndex={setEditingSegmentTagsSegmentIndex}
onEditSegmentTags={onEditSegmentTags}
loadSubtitleTrackToSegments={loadSubtitleTrackToSegments}
toggleCopyStreamIds={toggleCopyStreamIds}
changeEnabledStreamsFilter={changeEnabledStreamsFilter}
toggleCopyAllStreamsForPath={toggleCopyAllStreamsForPath}
onStreamSourceFileDrop={handleStreamSourceFileDrop}
/>
)}
</AnimatePresence>
</div>
</Sheet>
<div className="no-user-select" style={bottomStyle}>
<Timeline
shouldShowKeyframes={shouldShowKeyframes}
waveforms={waveforms}
overviewWaveform={overviewWaveform}
shouldShowWaveform={shouldShowWaveform}
waveformEnabled={waveformEnabled}
waveformHeight={waveformHeight}
showThumbnails={showThumbnails}
neighbouringKeyFrames={neighbouringKeyFrames}
thumbnails={thumbnailsSorted}
playerTime={playerTime}
commandedTime={commandedTime}
relevantTime={relevantTime}
commandedTimeRef={commandedTimeRef}
startTimeOffset={startTimeOffset}
zoom={zoom}
seekAbs={seekAbs}
fileDurationNonZero={fileDurationNonZero}
cutSegments={cutSegments}
setCurrentSegIndex={setCurrentSegIndex}
currentSegIndexSafe={currentSegIndexSafe}
inverseCutSegments={inverseCutSegments}
formatTimecode={formatTimecode}
formatTimeAndFrames={formatTimeAndFrames}
zoomWindowStartTime={zoomWindowStartTime}
zoomWindowEndTime={zoomWindowEndTime}
onZoomWindowStartTimeChange={setZoomWindowStartTime}
onGenerateOverviewWaveformClick={onGenerateOverviewWaveformClick}
playing={playing}
isFileOpened={isFileOpened}
onWheel={onTimelineWheel}
goToTimecode={goToTimecode}
darkMode={darkMode}
<LastCommandsSheet
visible={lastCommandsVisible}
onTogglePress={toggleLastCommands}
ffmpegCommandLog={ffmpegCommandLog}
setFfmpegCommandLog={setFfmpegCommandLog}
/>
<BottomBar
zoom={zoom}
setZoom={zoomAbs}
timelineToggleComfortZoom={timelineToggleComfortZoom}
hasVideo={hasVideo}
isRotationSet={isRotationSet}
rotation={rotation}
areWeCutting={areWeCutting}
increaseRotation={increaseRotation}
cleanupFilesDialog={cleanupFilesDialog}
captureSnapshot={captureSnapshot}
onExportPress={onExportPress}
segmentsToExport={segmentsToExport}
seekAbs={seekAbs}
currentSegIndexSafe={currentSegIndexSafe}
cutSegments={cutSegments}
currentCutSeg={currentCutSeg}
selectedSegments={selectedSegments}
setCutStart={setCutStart}
setCutEnd={setCutEnd}
setCurrentSegIndex={setCurrentSegIndex}
jumpCutEnd={jumpCutEnd}
jumpCutStart={jumpCutStart}
jumpTimelineStart={jumpTimelineStart}
jumpTimelineEnd={jumpTimelineEnd}
startTimeOffset={startTimeOffset}
setCutTime={setCutTime}
playing={playing}
shortStep={shortStep}
seekClosestKeyframe={seekClosestKeyframe}
togglePlay={togglePlay}
showThumbnails={showThumbnails}
toggleShowThumbnails={toggleShowThumbnails}
toggleWaveformMode={toggleWaveformMode}
waveformMode={waveformMode}
hasAudio={hasAudio}
keyframesEnabled={keyframesEnabled}
toggleShowKeyframes={toggleShowKeyframes}
detectedFps={detectedFps}
toggleLoopSelectedSegments={toggleLoopSelectedSegments}
isFileOpened={isFileOpened}
darkMode={darkMode}
toggleDarkMode={toggleDarkMode}
outputPlaybackRate={outputPlaybackRate}
setOutputPlaybackRate={setOutputPlaybackRate}
formatTimecode={formatTimecode}
parseTimecode={parseTimecode}
playbackRate={playbackRate}
currentFrame={currentFrame}
/>
</div>
{tunerVisible != null && <ValueTuners type={tunerVisible} onFinished={() => setTunerVisible(undefined)} />}
{/* Dialogs */}
<ExportConfirm areWeCutting={areWeCutting} segmentsOrInverse={segmentsOrInverse} segmentsToExport={segmentsToExport} willMerge={willMerge} visible={exportConfirmVisible} onClosePress={closeExportConfirm} onExportConfirm={onExportConfirm} renderOutFmt={renderOutFmt} outputDir={outputDir} numStreamsTotal={numStreamsTotal} numStreamsToCopy={numStreamsToCopy} onShowStreamsSelectorClick={handleShowStreamsSelectorClick} outFormat={fileFormat} setOutSegTemplate={setOutSegTemplate} outSegTemplate={outSegTemplateOrDefault} mergedFileTemplate={mergedFileTemplateOrDefault} setMergedFileTemplate={setMergedFileTemplate} generateOutSegFileNames={generateOutSegFileNames} generateMergedFileNames={generateMergedFileNames} currentSegIndexSafe={currentSegIndexSafe} mainCopiedThumbnailStreams={mainCopiedThumbnailStreams} needSmartCut={needSmartCut} isEncoding={isEncoding} encBitrate={encBitrate} setEncBitrate={setEncBitrate} toggleSettings={toggleSettings} outputPlaybackRate={outputPlaybackRate} lossyMode={lossyMode} />
<Sheet visible={streamsSelectorShown} onClosePress={() => setStreamsSelectorShown(false)} maxWidth={1000}>
{mainStreams && filePath != null && (
<StreamsSelector
mainFilePath={filePath}
mainFileFormatData={mainFileFormatData}
mainFileChapters={mainFileChapters}
allFilesMeta={allFilesMeta}
externalFilesMeta={externalFilesMeta}
setExternalFilesMeta={setExternalFilesMeta}
showAddStreamSourceDialog={showIncludeExternalStreamsDialog}
mainFileStreams={mainStreams}
isCopyingStreamId={isCopyingStreamId}
toggleCopyStreamId={toggleCopyStreamId}
setCopyStreamIdsForPath={setCopyStreamIdsForPath}
onExtractAllStreamsPress={extractAllStreams}
onExtractStreamPress={extractSingleStream}
shortestFlag={shortestFlag}
setShortestFlag={setShortestFlag}
nonCopiedExtraStreams={nonCopiedExtraStreams}
customTagsByFile={customTagsByFile}
setCustomTagsByFile={setCustomTagsByFile}
paramsByStreamId={paramsByStreamId}
updateStreamParams={updateStreamParams}
formatTimecode={formatTimecode}
loadSubtitleTrackToSegments={loadSubtitleTrackToSegments}
toggleCopyStreamIds={toggleCopyStreamIds}
changeEnabledStreamsFilter={changeEnabledStreamsFilter}
toggleCopyAllStreamsForPath={toggleCopyAllStreamsForPath}
onStreamSourceFileDrop={handleStreamSourceFileDrop}
<Sheet visible={settingsVisible} onClosePress={toggleSettings}>
<Settings
onTunerRequested={onTunerRequested}
onKeyboardShortcutsDialogRequested={toggleKeyboardShortcuts}
askForCleanupChoices={askForCleanupChoices}
toggleStoreProjectInWorkingDir={toggleStoreProjectInWorkingDir}
simpleMode={simpleMode}
clearOutDir={clearOutDir}
/>
)}
</Sheet>
<LastCommandsSheet
visible={lastCommandsVisible}
onTogglePress={toggleLastCommands}
ffmpegCommandLog={ffmpegCommandLog}
setFfmpegCommandLog={setFfmpegCommandLog}
/>
<Sheet visible={settingsVisible} onClosePress={toggleSettings}>
<Settings
onTunerRequested={onTunerRequested}
onKeyboardShortcutsDialogRequested={toggleKeyboardShortcuts}
askForCleanupChoices={askForCleanupChoices}
toggleStoreProjectInWorkingDir={toggleStoreProjectInWorkingDir}
simpleMode={simpleMode}
clearOutDir={clearOutDir}
/>
</Sheet>
</Sheet>
<ConcatDialog isShown={batchFiles.length > 0 && concatDialogVisible} onHide={() => setConcatDialogVisible(false)} paths={batchFilePaths} onConcat={userConcatFiles} setAlwaysConcatMultipleFiles={setAlwaysConcatMultipleFiles} alwaysConcatMultipleFiles={alwaysConcatMultipleFiles} exportCount={exportCount} maxLabelLength={maxLabelLength} />
<ConcatDialog isShown={batchFiles.length > 0 && concatDialogVisible} onHide={() => setConcatDialogVisible(false)} paths={batchFilePaths} onConcat={userConcatFiles} setAlwaysConcatMultipleFiles={setAlwaysConcatMultipleFiles} alwaysConcatMultipleFiles={alwaysConcatMultipleFiles} exportCount={exportCount} maxLabelLength={maxLabelLength} />
<KeyboardShortcuts isShown={keyboardShortcutsVisible} onHide={() => setKeyboardShortcutsVisible(false)} keyBindings={keyBindings} setKeyBindings={setKeyBindings} currentCutSeg={currentCutSeg} resetKeyBindings={resetKeyBindings} />
<KeyboardShortcuts isShown={keyboardShortcutsVisible} onHide={() => setKeyboardShortcutsVisible(false)} keyBindings={keyBindings} setKeyBindings={setKeyBindings} currentCutSeg={currentCutSeg} resetKeyBindings={resetKeyBindings} />
{/* This should probably be last, so that it's always on top */}
<AnimatePresence>
{working && <Working text={working.text} progress={progress} onAbortClick={abortWorking} />}
</AnimatePresence>
</div>
</ThemeProvider>
</UserSettingsContext.Provider>
</SegColorsContext.Provider>
{/* This should probably be last, so that it's always on top */}
<AnimatePresence>
{working && <Working text={working.text} progress={progress} onAbortClick={abortWorking} />}
</AnimatePresence>
</div>
</ThemeProvider>
</UserSettingsContext.Provider>
</SegColorsContext.Provider>
</AppContext.Provider>
<div id="swal2-container-wrapper" className={darkMode ? 'dark-theme' : undefined} style={baseColorStyle} />

@ -1,63 +1,69 @@
import { useEffect, useRef, useState, useCallback, useMemo, memo, CSSProperties, RefObject, ReactEventHandler, FocusEventHandler } from 'react';
import { Spinner } from 'evergreen-ui';
import { useDebounce } from 'use-debounce';
import invariant from 'tiny-invariant';
import debounce from 'lodash/debounce.js';
import isDev from './isDev';
import { ChromiumHTMLVideoElement } from './types';
import { FFprobeStream } from '../../../ffprobe';
const { compatPlayer: { createMediaSourceStream, readOneJpegFrame } } = window.require('@electron/remote').require('./index.js');
const { compatPlayer: { createMediaSourceStream } } = window.require('@electron/remote').require('./index.js');
async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes, seekTo, signal, onPlayRequested, onCanPlay, getTargetTime, size, fps, rotate }: {
async function startPlayback({ path, slaveVideo, masterVideo, videoStreamIndex, audioStreamIndexes, seekTo, signal, size, fps, rotate, onCanPlay, onResetNeeded, onWaiting }: {
path: string,
video: ChromiumHTMLVideoElement,
slaveVideo: ChromiumHTMLVideoElement,
masterVideo: ChromiumHTMLVideoElement,
videoStreamIndex?: number | undefined,
audioStreamIndexes: number[],
seekTo: number,
signal: AbortSignal,
onPlayRequested: () => void,
onCanPlay: () => void,
getTargetTime: () => number,
size?: number | undefined,
fps?: number | undefined,
rotate: number | undefined,
onCanPlay: () => void,
onResetNeeded: () => void,
onWaiting: () => void,
}) {
let canPlay = false;
let bufferEndTime: number | undefined;
let bufferStartTime = 0;
let bufferStartTime = seekTo;
let stream: ReturnType<typeof createMediaSourceStream> | undefined;
let done = false;
let interval: NodeJS.Timeout | undefined;
let interval2: NodeJS.Timeout | undefined;
let objectUrl: string | undefined;
let processChunkTimeout: NodeJS.Timeout;
function cleanup() {
signal.addEventListener('abort', () => {
console.log('Cleanup');
done = true;
video.pause();
slaveVideo.pause();
if (interval != null) clearInterval(interval);
if (interval2 != null) clearInterval(interval2);
if (processChunkTimeout != null) clearInterval(processChunkTimeout);
stream?.abort();
if (objectUrl != null) URL.revokeObjectURL(objectUrl);
video.removeAttribute('src');
}
signal.addEventListener('abort', cleanup);
slaveVideo.removeAttribute('src');
});
// See chrome://media-internals
const mediaSource = new MediaSource();
let streamTimestamp: number | undefined;
let lastRemoveTimestamp = 0;
let lastRemoveTimestamp = seekTo;
const setPlaybackRate = (r: number) => {
const maxAllowedPlaybackRate = 16; // or else we get an error in Chromium
const newAdjustedRate = Math.min(maxAllowedPlaybackRate, r * masterVideo.playbackRate);
if (slaveVideo.playbackRate === newAdjustedRate) {
return false;
}
function setStandardPlaybackRate() {
// set it a bit faster, so that we don't easily fall behind (better too fast than too slow)
// eslint-disable-next-line no-param-reassign
video.playbackRate = 1.05;
}
slaveVideo.playbackRate = newAdjustedRate;
return true;
};
// set it a bit faster, so that we don't easily fall behind (better too fast than too slow)
const setStandardPlaybackRate = () => setPlaybackRate(1.05);
setStandardPlaybackRate();
@ -80,15 +86,20 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes
throw new Error(`Unsupported MIME type or codec: ${mimeCodec}`);
}
const mediaSource = new MediaSource();
// console.log(mediaSource.readyState); // closed
objectUrl = URL.createObjectURL(mediaSource);
// eslint-disable-next-line no-param-reassign
video.src = objectUrl;
slaveVideo.src = objectUrl;
await new Promise((resolve) => mediaSource.addEventListener('sourceopen', resolve, { once: true }));
// console.log(mediaSource.readyState); // open
const sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
sourceBuffer.timestampOffset = seekTo;
signal.addEventListener('abort', () => sourceBuffer.abort());
const getBufferEndTime = () => {
if (mediaSource.readyState !== 'open') {
@ -105,10 +116,6 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes
return sourceBuffer.buffered.end(0);
};
sourceBuffer.addEventListener('updateend', () => {
onPlayRequested();
}, { once: true });
let firstChunkReceived = false;
const processChunk = async () => {
@ -118,7 +125,8 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes
console.log('End of stream');
return;
}
if (done) return;
if (signal.aborted) return;
if (!firstChunkReceived) {
firstChunkReceived = true;
@ -134,18 +142,46 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes
sourceBuffer.addEventListener('error', (err) => console.error('sourceBuffer error, check DevTools ▶ More Tools ▶ Media', err));
// video.addEventListener('loadeddata', () => console.log('loadeddata'));
// video.addEventListener('play', () => console.log('play'));
video.addEventListener('canplay', () => {
const handleCanPlay = () => {
console.log('canplay');
if (!canPlay) {
canPlay = true;
onCanPlay();
}
}, { once: true });
canPlay = true;
onCanPlay();
};
slaveVideo.addEventListener('canplay', handleCanPlay);
const handleEnded = () => {
console.log('ended');
};
slaveVideo.addEventListener('ended', handleEnded);
const handleStalled = () => {
console.log('stalled');
};
slaveVideo.addEventListener('stalled', handleStalled);
const handleWaiting = () => {
if (slaveVideo.paused || slaveVideo.ended) return; // we don't care if paused
console.log('waiting');
onWaiting();
};
slaveVideo.addEventListener('waiting', handleWaiting);
const handlePlaying = () => {
console.log('playing');
};
slaveVideo.addEventListener('playing', handlePlaying);
signal.addEventListener('abort', () => {
slaveVideo.removeEventListener('canplay', handleCanPlay);
slaveVideo.removeEventListener('ended', handleEnded);
slaveVideo.removeEventListener('stalled', handleStalled);
slaveVideo.removeEventListener('waiting', handleWaiting);
slaveVideo.removeEventListener('playing', handlePlaying);
});
sourceBuffer.addEventListener('updateend', ({ timeStamp }) => {
if (done) return;
if (signal.aborted) return;
streamTimestamp = timeStamp; // apparently this timestamp cannot be trusted much
@ -154,13 +190,10 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes
bufferEndTime = getBufferEndTime();
// console.log('updateend', { bufferEndTime })
if (bufferEndTime != null) {
const targetTime = getTargetTime();
const bufferedTime = bufferEndTime - lastRemoveTimestamp;
const bufferedDuration = bufferEndTime - lastRemoveTimestamp;
if (bufferedTime > bufferMaxSec && !sourceBuffer.updating) {
if (bufferedDuration > bufferMaxSec && !sourceBuffer.updating) {
try {
lastRemoveTimestamp = bufferEndTime;
const removeTo = bufferEndTime - bufferMaxSec;
@ -173,7 +206,7 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes
}
}
const bufferAheadSec = bufferEndTime - targetTime;
const bufferAheadSec = bufferEndTime - masterVideo.currentTime;
if (bufferAheadSec > bufferThrottleSec) {
console.debug(`buffer ahead by ${bufferAheadSec}, throttling stream read`);
processChunkTimeout = setTimeout(processChunk, 1000);
@ -188,195 +221,186 @@ async function startPlayback({ path, video, videoStreamIndex, audioStreamIndexes
stream = createMediaSourceStream({ path, videoStreamIndex, audioStreamIndexes, seekTo, size, fps, rotate });
interval = setInterval(() => {
if (!canPlay) return;
if (mediaSource.readyState !== 'open') {
console.warn('mediaSource.readyState was not open, but:', mediaSource.readyState);
// else we will get: Uncaught DOMException: Failed to execute 'end' on 'TimeRanges': The index provided (0) is greater than or equal to the maximum bound (0).
return;
}
const targetTime = getTargetTime();
const playbackDiff = targetTime != null ? targetTime - video.currentTime : undefined;
const streamTimestampDiff = streamTimestamp != null && bufferEndTime != null ? (streamTimestamp / 1000) - bufferEndTime : undefined; // not really needed, but log for curiosity
console.debug('bufferStartTime', bufferStartTime, 'bufferEndTime', bufferEndTime, 'targetTime', targetTime, 'playback:', video.currentTime, 'playbackDiff:', playbackDiff, 'streamTimestamp diff:', streamTimestampDiff);
if (!canPlay || targetTime == null) return;
console.log(`bufferStartTime: ${bufferStartTime}, bufferEndTime: ${bufferEndTime}, master time: ${masterVideo.currentTime}, slave time: ${slaveVideo.currentTime} (diff: ${masterVideo.currentTime - slaveVideo.currentTime}), streamTimestamp: ${streamTimestamp}`);
// console.log(sourceBuffer.buffered.length, sourceBuffer.buffered.start(0), sourceBuffer.buffered.end(0))
if (sourceBuffer.buffered.length !== 1) {
// not sure why this would happen or how to handle this
console.warn('sourceBuffer.buffered.length was', sourceBuffer.buffered.length);
}
}, 1000);
if ((video.paused || video.ended) && !done) {
console.warn('Resuming unexpectedly paused video');
onPlayRequested();
}
// Synchronize state between the two video elements
interval2 = setInterval(async () => {
try {
const maxSecAfterBufferToWaitFor = 5;
if (masterVideo.currentTime < bufferStartTime || (bufferEndTime != null && masterVideo.currentTime - bufferEndTime > maxSecAfterBufferToWaitFor)) {
console.log('Seeked before/after buffered range, resetting playback');
onResetNeeded();
return;
}
if (masterVideo.paused || masterVideo.ended) {
const resolution = 1000;
if (Math.round(slaveVideo.currentTime * resolution) !== Math.round(masterVideo.currentTime * resolution)) {
// eslint-disable-next-line no-param-reassign
slaveVideo.currentTime = masterVideo.currentTime;
}
} else { // playing
// make sure the playback keeps up while playing
// or when seeking while playing
// https://stackoverflow.com/questions/23301496/how-to-keep-a-live-mediasource-video-stream-in-sync
const playbackDiff = masterVideo.currentTime - slaveVideo.currentTime;
if (Math.abs(playbackDiff) > 1) {
console.log(`Playback ${playbackDiff > 0 ? 'behind' : 'ahead'} master player time by ${playbackDiff}s, jumping to desired time`);
// eslint-disable-next-line no-param-reassign
slaveVideo.currentTime = masterVideo.currentTime;
setStandardPlaybackRate();
} else if (playbackDiff != null && playbackDiff > 0.3) {
// eslint-disable-next-line no-param-reassign
if (setPlaybackRate(1.5)) {
console.warn(`Playback behind by ${playbackDiff}s, speeding up playback`);
}
} else {
setStandardPlaybackRate();
}
}
// make sure the playback keeps up
// https://stackoverflow.com/questions/23301496/how-to-keep-a-live-mediasource-video-stream-in-sync
if (playbackDiff != null && playbackDiff > 1) {
console.warn(`playback severely behind by ${playbackDiff}s, seeking to desired time`);
// eslint-disable-next-line no-param-reassign
video.currentTime = targetTime;
setStandardPlaybackRate();
} else if (playbackDiff != null && playbackDiff > 0.3) {
console.warn(`playback behind by ${playbackDiff}s, speeding up playback`);
// eslint-disable-next-line no-param-reassign
video.playbackRate = 1.5;
} else {
setStandardPlaybackRate();
if (slaveVideo.volume !== masterVideo.volume) {
// eslint-disable-next-line no-param-reassign
slaveVideo.volume = masterVideo.volume;
}
const masterStopped = masterVideo.paused || masterVideo.ended;
const slaveStopped = slaveVideo.paused || slaveVideo.ended;
if (slaveStopped && !masterStopped) {
await slaveVideo.play();
} else if (!slaveStopped && masterStopped) {
slaveVideo.pause();
}
} catch (err) {
console.error('play/pause failed', err);
}
}, 200);
}, 30); // todo requestAnimationFrame?
// OK, everything initialized and ready to stream!
processChunk();
}
async function createPauseImage({ path, seekTo, videoStreamIndex, image, rotate, signal }: {
path: string,
seekTo: number,
videoStreamIndex: number,
image: HTMLImageElement | null,
rotate: number | undefined,
signal: AbortSignal,
}) {
const { promise, abort } = readOneJpegFrame({ path, seekTo, videoStreamIndex, rotate });
signal.addEventListener('abort', () => abort());
const jpegImage = await promise;
invariant(image);
if (image.src) URL.revokeObjectURL(image.src);
// eslint-disable-next-line no-param-reassign
image.src = URL.createObjectURL(new Blob([Buffer.from(jpegImage)], { type: 'image/jpeg' }));
}
function MediaSourcePlayer({ rotate, filePath, playerTime, videoStream, audioStreams, commandedTime, playing, eventId, masterVideoRef, mediaSourceQuality, playbackVolume }: {
function MediaSourcePlayer({ rotate, filePath, videoStream, audioStreams, masterVideoRef, mediaSourceQuality }: {
rotate: number | undefined,
filePath: string,
playerTime: number,
videoStream: FFprobeStream | undefined,
audioStreams: FFprobeStream[],
commandedTime: number,
playing: boolean,
eventId: number,
masterVideoRef: RefObject<HTMLVideoElement>,
mediaSourceQuality: number,
playbackVolume: number,
}) {
const videoRef = useRef<HTMLVideoElement>(null);
const imgRef = useRef<HTMLImageElement>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
const [loading, setLoading] = useState(true);
const [showCanvas, setShowCanvas] = useState(false);
const onVideoError = useCallback<ReactEventHandler<HTMLVideoElement>>((error) => {
console.error('video error', error);
}, []);
const state = useMemo(() => (playing
? { startTime: commandedTime, playing, eventId }
: { startTime: playerTime, playing, eventId }
), [commandedTime, eventId, playerTime, playing]);
const audioStreamIndexes = useMemo(() => audioStreams.map((s) => s.index), [audioStreams]);
const [debouncedState] = useDebounce(state, 300, {
equalityFn: (a, b) => a.startTime === b.startTime && a.playing === b.playing && a.eventId === b.eventId,
leading: true,
});
useEffect(() => {
// console.log('debouncedState', debouncedState);
}, [debouncedState]);
const onPlayRequested = useCallback(async () => {
try {
await videoRef.current?.play();
} catch (err) {
console.error('play failed', err);
}
}, []);
useEffect(() => {
setLoading(true);
let { startTime: seekTo } = debouncedState;
const video = videoRef.current;
invariant(video != null);
const masterVideo = masterVideoRef.current;
invariant(masterVideo != null);
if (seekTo >= masterVideo.duration) seekTo = 0; // start over if reached end
const canvas = canvasRef.current;
invariant(canvas != null);
if (seekTo == null) {
return () => undefined;
}
let abortController: AbortController;
let startDebounced: () => void;
const onCanPlay = () => {
setLoading(false);
};
const getTargetTime = () => masterVideoRef.current!.currentTime - seekTo;
const start = async () => {
abortController = new AbortController();
const abortController = new AbortController();
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const ctx = canvas.getContext('2d');
ctx?.drawImage(video, 0, 0, canvas.width, canvas.height);
setShowCanvas(true);
setLoading(true);
(async () => {
try {
// When playing, we use a secondary video element, but when paused we use an img
if (debouncedState.playing) {
let size: number | undefined;
if (videoStream != null) {
if (mediaSourceQuality === 0) size = 800;
else if (mediaSourceQuality === 1) size = 420;
}
const seekTo = masterVideo.currentTime;
let fps: number | undefined;
if (mediaSourceQuality === 0) fps = 30;
else if (mediaSourceQuality === 1) fps = 15;
try {
let size: number | undefined;
if (videoStream != null) {
if (mediaSourceQuality === 0) size = 800;
else if (mediaSourceQuality === 1) size = 420;
}
await startPlayback({ signal: abortController.signal, path: filePath, video, videoStreamIndex: videoStream?.index, audioStreamIndexes, seekTo, onPlayRequested, onCanPlay, getTargetTime, size, fps, rotate });
} else if (videoStream != null) { // paused
try {
await createPauseImage({ signal: abortController.signal, path: filePath, videoStreamIndex: videoStream.index, seekTo, image: imgRef.current, rotate });
} finally {
let fps: number | undefined;
if (mediaSourceQuality === 0) fps = 30;
else if (mediaSourceQuality === 1) fps = 15;
await startPlayback({
signal: abortController.signal,
path: filePath,
slaveVideo: video,
masterVideo,
videoStreamIndex: videoStream?.index,
audioStreamIndexes,
seekTo,
size,
fps,
rotate,
onCanPlay: () => {
setLoading(false);
}
}
setShowCanvas(false);
},
onResetNeeded: () => {
abortController.abort();
startDebounced();
},
onWaiting: () => {
setLoading(true);
},
});
} catch (err) {
console.error('Preview failed', err);
}
})();
};
startDebounced = debounce(start, 500, { leading: true, trailing: true });
startDebounced();
return () => abortController.abort();
// Important that we also have eventId in the deps, so that we can restart the preview when the eventId changes
}, [debouncedState.startTime, debouncedState.eventId, filePath, masterVideoRef, onPlayRequested, debouncedState.playing, videoStream, mediaSourceQuality, audioStreamIndexes, rotate, debouncedState]);
useEffect(() => {
if (videoRef.current) videoRef.current.volume = playbackVolume;
}, [playbackVolume]);
}, [audioStreamIndexes, filePath, masterVideoRef, mediaSourceQuality, rotate, videoStream]);
const onFocus = useCallback<FocusEventHandler<HTMLVideoElement | HTMLImageElement>>((e) => {
const onFocus = useCallback<FocusEventHandler<HTMLVideoElement>>((e) => {
// prevent video element from stealing focus in fullscreen mode https://github.com/mifi/lossless-cut/issues/543#issuecomment-1868167775
e.target.blur();
}, []);
const { videoStyle, imgStyle } = useMemo<{ videoStyle: CSSProperties, imgStyle: CSSProperties }>(() => {
const sharedStyle: CSSProperties = {
position: 'absolute', left: 0, right: 0, top: 0, bottom: 0, display: 'block', width: '100%', height: '100%', objectFit: 'contain', transform: rotate ? `rotate(${rotate}deg)` : undefined,
};
const shouldShowImage = !debouncedState.playing;
return {
videoStyle: { ...sharedStyle, visibility: loading || !debouncedState.playing ? 'hidden' : undefined },
imgStyle: { ...sharedStyle, visibility: shouldShowImage ? undefined : 'hidden' },
};
}, [loading, debouncedState.playing, rotate]);
const videoStyle = useMemo<CSSProperties>(() => ({
position: 'absolute', left: 0, right: 0, top: 0, bottom: 0, display: 'block', width: '100%', height: '100%', objectFit: 'contain', transform: rotate ? `rotate(${rotate}deg)` : undefined,
}), [rotate]);
return (
<div style={{ width: '100%', height: '100%', left: 0, right: 0, top: 0, bottom: 0, position: 'absolute', overflow: 'hidden', background: 'black', pointerEvents: 'none' }}>
{/* eslint-disable-next-line jsx-a11y/media-has-caption */}
<video style={videoStyle} ref={videoRef} playsInline onError={onVideoError} tabIndex={-1} onFocusCapture={onFocus} />
{videoStream != null && <img alt="" width={videoStream.width} height={videoStream.height} ref={imgRef} style={imgStyle} tabIndex={-1} onFocusCapture={onFocus} />}
<video style={{ ...videoStyle, visibility: showCanvas ? 'hidden' : 'initial' }} ref={videoRef} playsInline onError={onVideoError} tabIndex={-1} onFocusCapture={onFocus} />
<canvas style={{ ...videoStyle, display: showCanvas ? 'initial' : 'none' }} ref={canvasRef} />
{loading && (
<div style={{ position: 'absolute', top: 0, bottom: 0, left: 0, right: 0, display: 'flex', justifyContent: 'center', alignItems: 'center' }}><Spinner /></div>

@ -4,7 +4,6 @@ import { showPlaybackFailedMessage } from '../swal';
export default ({ filePath }: { filePath: string | undefined }) => {
const [commandedTime, setCommandedTimeRaw] = useState(0);
const [compatPlayerEventId, setCompatPlayerEventId] = useState(0);
const [playbackRate, setPlaybackRateState] = useState(1);
const [outputPlaybackRate, setOutputPlaybackRateState] = useState(1);
const [playerTime, setPlayerTime] = useState<number>();
@ -68,7 +67,6 @@ export default ({ filePath }: { filePath: string | undefined }) => {
smoothSeek(outVal);
setCommandedTime(outVal);
if (video.currentTime === val) setCompatPlayerEventId((id) => id + 1); // To make sure that we can seek even to the same commanded time that we are already add (e.g. loop current segment)
}, [setCommandedTime, smoothSeek]);
// Relevant time is the player's playback position if we're currently playing - if not, it's the user's commanded time.
@ -146,8 +144,6 @@ export default ({ filePath }: { filePath: string | undefined }) => {
relevantTime,
getRelevantTime,
onVideoAbort,
compatPlayerEventId,
setCompatPlayerEventId,
setOutputPlaybackRateState,
playbackModeRef,
playerTime,

Loading…
Cancel
Save