Improvements

- add low FPS support for HEVC/H265 and PRORES using ffmpeg per-frame rendering
- rotation previews
- refactoring

package.json

@@ -66,6 +66,7 @@
     "mime-types": "^2.1.14",
     "moment": "^2.18.1",
     "mousetrap": "^1.6.1",
+    "p-queue": "^6.2.0",
     "prop-types": "^15.6.2",
     "react": "^16.7.0",
     "react-dom": "^16.7.0",

src/ffmpeg.js

@@ -31,11 +31,18 @@ function getFfmpegPath() {
     });
 }
 
-async function getFFprobePath() {
+async function runFfprobe(args) {
   const ffmpegPath = await getFfmpegPath();
-  return path.join(path.dirname(ffmpegPath), getWithExt('ffprobe'));
+  const ffprobePath = path.join(path.dirname(ffmpegPath), getWithExt('ffprobe'));
+  return execa(ffprobePath, args);
 }
 
+async function runFfmpeg(args) {
+  const ffmpegPath = await getFfmpegPath();
+  return execa(ffmpegPath, args);
+}
+
+
 function handleProgress(process, cutDuration, onProgress) {
   const rl = readline.createInterface({ input: process.stderr });
   rl.on('line', (line) => {
@@ -171,10 +178,38 @@ async function html5ify(filePath, outPath, encodeVideo) {
 
   console.log('ffmpeg', ffmpegArgs.join(' '));
 
-  const ffmpegPath = await getFfmpegPath();
-  const process = execa(ffmpegPath, ffmpegArgs);
-  const result = await process;
-  console.log(result.stdout);
+  const { stdout } = await runFfmpeg(ffmpegArgs);
+  console.log(stdout);
+
+  await transferTimestamps(filePath, outPath);
+}
+
+async function getDuration(filePpath) {
+  // https://superuser.com/questions/650291/how-to-get-video-duration-in-seconds
+  const { stdout } = await runFfprobe(['-i', filePpath, '-show_entries', 'format=duration', '-print_format', 'json']);
+  return parseFloat(JSON.parse(stdout).format.duration);
+}
+
+// This is just used to load something into the player with correct length,
+// so user can seek and then we render frames using ffmpeg
+async function html5ifyDummy(filePath, outPath) {
+  console.log('Making HTML5 friendly dummy', { filePath, outPath });
+
+  const duration = await getDuration(filePath);
+
+  const ffmpegArgs = [
+    // This is just a fast way of generating an empty dummy file
+    // TODO use existing audio track file if it has one
+    '-f', 'lavfi', '-i', 'anullsrc=channel_layout=stereo:sample_rate=44100',
+    '-t', duration,
+    '-acodec', 'flac',
+    '-y', outPath,
+  ];
+
+  console.log('ffmpeg', ffmpegArgs.join(' '));
+
+  const { stdout } = await runFfmpeg(ffmpegArgs);
+  console.log(stdout);
 
   await transferTimestamps(filePath, outPath);
 }
@@ -246,11 +281,10 @@ function determineOutputFormat(ffprobeFormats, ft) {
 async function getFormat(filePath) {
   console.log('getFormat', filePath);
 
-  const ffprobePath = await getFFprobePath();
-  const result = await execa(ffprobePath, [
+  const { stdout } = await runFfprobe([
     '-of', 'json', '-show_format', '-i', filePath,
   ]);
-  const formatsStr = JSON.parse(result.stdout).format.format_name;
+  const formatsStr = JSON.parse(stdout).format.format_name;
   console.log('formats', formatsStr);
   const formats = (formatsStr || '').split(',');
 
@@ -263,12 +297,11 @@ async function getFormat(filePath) {
 }
 
 async function getAllStreams(filePath) {
-  const ffprobePath = await getFFprobePath();
-  const result = await execa(ffprobePath, [
+  const { stdout } = await runFfprobe([
     '-of', 'json', '-show_entries', 'stream', '-i', filePath,
   ]);
 
-  return JSON.parse(result.stdout);
+  return JSON.parse(stdout);
 }
 
 function mapCodecToOutputFormat(codec, type) {
@@ -325,17 +358,49 @@ async function extractAllStreams({ customOutDir, filePath }) {
   console.log(ffmpegArgs);
 
   // TODO progress
+  const { stdout } = await runFfmpeg(ffmpegArgs);
+  console.log(stdout);
+}
+
+async function renderFrame(timestamp, filePath, rotation) {
+  const transpose = {
+    90: 'transpose=2',
+    180: 'transpose=1,transpose=1',
+    270: 'transpose=1',
+  };
+  const args = [
+    '-ss', timestamp,
+    ...(rotation !== undefined ? ['-noautorotate'] : []),
+    '-i', filePath,
+    // ...(rotation !== undefined ? ['-metadata:s:v:0', 'rotate=0'] : []), // Reset the rotation metadata first
+    ...(rotation !== undefined && rotation > 0 ? ['-vf', `${transpose[rotation]}`] : []),
+    '-f', 'image2',
+    '-vframes', '1',
+    '-q:v', '10',
+    '-',
+    // '-y', outPath,
+  ];
+
+  // console.time('ffmpeg');
   const ffmpegPath = await getFfmpegPath();
-  const process = execa(ffmpegPath, ffmpegArgs);
-  const result = await process;
-  console.log(result.stdout);
+  // console.timeEnd('ffmpeg');
+  console.log('ffmpeg', args);
+  const { stdout } = await execa(ffmpegPath, args, { encoding: null });
+
+  const blob = new Blob([stdout], { type: 'image/jpeg' });
+  const url = URL.createObjectURL(blob);
+  return url;
 }
 
+
 module.exports = {
   cutMultiple,
   getFormat,
   html5ify,
+  html5ifyDummy,
   mergeAnyFiles,
   autoMergeSegments,
   extractAllStreams,
+  renderFrame,
+  getAllStreams,
 };

src/renderer.jsx

@@ -13,6 +13,8 @@ const uuid = require('uuid');
 const React = require('react');
 const ReactDOM = require('react-dom');
 const classnames = require('classnames');
+const { default: PQueue } = require('p-queue');
+
 
 const HelpSheet = require('./HelpSheet');
 const TimelineSeg = require('./TimelineSeg');
@@ -72,10 +74,17 @@ function createSegment({ start, end } = {}) {
   };
 }
 
+function doesPlayerSupportFile(streams) {
+  // TODO improve, whitelist supported codecs instead
+  return !streams.find(s => ['hevc', 'prores'].includes(s.codec_name));
+  // return true;
+}
+
 const getInitialLocalState = () => ({
   working: false,
   filePath: '', // Setting video src="" prevents memory leak in chromium
   html5FriendlyPath: undefined,
+  userHtml5ified: false,
   playing: false,
   currentTime: undefined,
   duration: undefined,
@@ -85,9 +94,12 @@ const getInitialLocalState = () => ({
   cutEndTimeManual: undefined,
   fileFormat: undefined,
   detectedFileFormat: undefined,
+  streams: [],
   rotation: 360,
   cutProgress: undefined,
   startTimeOffset: 0,
+  framePath: undefined,
+  rotationPreviewRequested: false,
 });
 
 const globalState = {
@@ -108,6 +120,8 @@ class App extends React.Component {
       ...globalState,
     };
 
+    this.queue = new PQueue({ concurrency: 1 });
+
     const load = async (filePath, html5FriendlyPath) => {
       const { working } = this.state;
 
@@ -127,10 +141,27 @@ class App extends React.Component {
           errorToast('Unsupported file');
           return;
         }
+
+        const { streams } = await ffmpeg.getAllStreams(filePath);
+
         setFileNameTitle(filePath);
         this.setState({
-          filePath, html5FriendlyPath, fileFormat, detectedFileFormat: fileFormat,
+          streams,
+          filePath,
+          html5FriendlyPath,
+          fileFormat,
+          detectedFileFormat: fileFormat,
         });
+
+        if (html5FriendlyPath) {
+          this.setState({ userHtml5ified: true });
+        } else if (!doesPlayerSupportFile(streams)) {
+          const { customOutDir } = this.state;
+          const html5ifiedDummyPath = getOutPath(customOutDir, filePath, 'html5ified-dummy.mkv');
+          await ffmpeg.html5ifyDummy(filePath, html5ifiedDummyPath);
+          this.setState({ html5FriendlyPath: html5ifiedDummyPath });
+          this.throttledRenderFrame(0);
+        }
       } catch (err) {
         if (err.code === 1 || err.code === 'ENOENT') {
           errorToast('Unsupported file');
@@ -241,7 +272,7 @@ class App extends React.Component {
     electron.ipcRenderer.send('renderer-ready');
   }
 
-  onPlay(playing) {
+  onPlayingChange(playing) {
     this.setState({ playing });
 
     if (!playing) {
@@ -253,6 +284,16 @@ class App extends React.Component {
     this.setState({ duration });
   }
 
+  onTimeUpdate = (e) => {
+    const { currentTime } = e.target;
+    if (this.state.currentTime === currentTime) return;
+
+    this.setState({ rotationPreviewRequested: false }); // Reset this
+    this.setState({ currentTime }, () => {
+      this.throttledRenderFrame();
+    });
+  }
+
   onCutProgress = (cutProgress) => {
     this.setState({ cutProgress });
   }
@@ -289,6 +330,10 @@ class App extends React.Component {
     return this.state.rotation;
   }
 
+  getEffectiveRotation() {
+    return this.isRotationSet() ? this.getRotation() : undefined;
+  }
+
   getRotationStr() {
     return `${this.getRotation()}°`;
   }
@@ -330,8 +375,38 @@ class App extends React.Component {
     return (this.state.currentTime || 0) + this.state.startTimeOffset;
   }
 
+  frameRenderEnabled = () => {
+    const { rotationPreviewRequested, userHtml5ified, streams } = this.state;
+    if (rotationPreviewRequested) return true;
+    return !userHtml5ified && !doesPlayerSupportFile(streams);
+  }
+
+  /* eslint-disable react/sort-comp */
+  throttledRenderFrame = async () => {
+    if (this.queue.size < 2) {
+      this.queue.add(async () => {
+        if (!this.frameRenderEnabled()) return;
+
+        const { filePath, currentTime } = this.state;
+        const rotation = this.getEffectiveRotation();
+        if (currentTime == null || !filePath) return;
+
+        try {
+          if (this.state.framePath) URL.revokeObjectURL(this.state.framePath);
+          const framePath = await ffmpeg.renderFrame(currentTime, filePath, rotation);
+          this.setState({ framePath });
+        } catch (err) {
+          console.error(err);
+        }
+      });
+    }
+
+    await this.queue.onIdle();
+  };
+
   increaseRotation = () => {
     this.setState(({ rotation }) => ({ rotation: (rotation + 90) % 450 }));
+    this.setState({ rotationPreviewRequested: true }, () => this.throttledRenderFrame());
   }
 
   toggleCaptureFormat = () => {
@@ -438,7 +513,7 @@ class App extends React.Component {
       return;
     }
 
-    const rotation = this.isRotationSet() ? this.getRotation() : undefined;
+    const rotation = this.getEffectiveRotation();
 
     const cutStartTime = this.getCutStartTime();
     const cutEndTime = this.getCutEndTime();
@@ -622,16 +697,35 @@ class App extends React.Component {
         </div>
         )}
 
+        {this.state.rotationPreviewRequested && (
+          <div style={{
+            position: 'absolute', zIndex: 1, top: '1em', right: '1em', color: 'white',
+          }}
+          >
+            Lossless rotation preview
+          </div>
+        )}
+
         {/* eslint-disable jsx-a11y/media-has-caption */}
         <div id="player">
           <video
             src={this.getFileUri()}
             onRateChange={this.playbackRateChange}
-            onPlay={() => this.onPlay(true)}
-            onPause={() => this.onPlay(false)}
+            onPlay={() => this.onPlayingChange(true)}
+            onPause={() => this.onPlayingChange(false)}
             onDurationChange={e => this.onDurationChange(e.target.duration)}
-            onTimeUpdate={e => this.setState({ currentTime: e.target.currentTime })}
+            onTimeUpdate={this.onTimeUpdate}
           />
+
+          {this.state.framePath && this.frameRenderEnabled() && (
+            <img
+              style={{
+                width: '100%', height: '100%', objectFit: 'contain', left: 0, right: 0, top: 0, bottom: 0, position: 'absolute', background: 'black',
+              }}
+              src={this.state.framePath}
+              alt=""
+            />
+          )}
         </div>
         {/* eslint-enable jsx-a11y/media-has-caption */}
 

yarn.lock

@@ -1943,6 +1943,11 @@ esutils@^2.0.2:
   resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.2.tgz#0abf4f1caa5bcb1f7a9d8acc6dea4faaa04bac9b"
   integrity sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=
 
+eventemitter3@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.0.tgz#d65176163887ee59f386d64c82610b696a4a74eb"
+  integrity sha512-qerSRB0p+UDEssxTtm6EDKcE7W4OaoisfIMl4CngyEhjpYglocpNg6UEqCvemdGhosAsg4sO2dXJOdyBifPGCg==
+
 execa@^0.10.0:
   version "0.10.0"
   resolved "https://registry.yarnpkg.com/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50"
@@ -3610,6 +3615,21 @@ p-map@^1.1.1, p-map@^1.2.0:
   resolved "https://registry.yarnpkg.com/p-map/-/p-map-1.2.0.tgz#e4e94f311eabbc8633a1e79908165fca26241b6b"
   integrity sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA==
 
+p-queue@^6.2.0:
+  version "6.2.0"
+  resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.2.0.tgz#c8122b9514d2bbe5f16d8a47f17dc2f9a8ac7235"
+  integrity sha512-B2LXNONcyn/G6uz2UBFsGjmSa0e/br3jznlzhEyCXg56c7VhEpiT2pZxGOfv32Q3FSyugAdys9KGpsv3kV+Sbg==
+  dependencies:
+    eventemitter3 "^4.0.0"
+    p-timeout "^3.1.0"
+
+p-timeout@^3.1.0:
+  version "3.2.0"
+  resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe"
+  integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==
+  dependencies:
+    p-finally "^1.0.0"
+
 p-try@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3"