retry EPERM (windows antivirus)

fixes #1704

src/edlStore.js

@@ -5,39 +5,39 @@ import { parseSrt, formatSrt, parseCuesheet, parseXmeml, parseFcpXml, parseCsv,
 import { askForYouTubeInput, showOpenDialog } from './dialogs';
 import { getOutPath } from './util';
 
-const fs = window.require('fs-extra');
+const { readFile, writeFile } = window.require('fs/promises');
 const cueParser = window.require('cue-parser');
 const { basename } = window.require('path');
 
 const { dialog } = window.require('@electron/remote');
 
 export async function loadCsvSeconds(path) {
-  return parseCsv(await fs.readFile(path, 'utf-8'), parseCsvTime);
+  return parseCsv(await readFile(path, 'utf-8'), parseCsvTime);
 }
 
 export async function loadCsvFrames(path, fps) {
   if (!fps) throw new Error('The loaded file has an unknown framerate');
-  return parseCsv(await fs.readFile(path, 'utf-8'), getFrameValParser(fps));
+  return parseCsv(await readFile(path, 'utf-8'), getFrameValParser(fps));
 }
 
 export async function loadXmeml(path) {
-  return parseXmeml(await fs.readFile(path, 'utf-8'));
+  return parseXmeml(await readFile(path, 'utf-8'));
 }
 
 export async function loadFcpXml(path) {
-  return parseFcpXml(await fs.readFile(path, 'utf-8'));
+  return parseFcpXml(await readFile(path, 'utf-8'));
 }
 
 export async function loadDvAnalyzerSummaryTxt(path) {
-  return parseDvAnalyzerSummaryTxt(await fs.readFile(path, 'utf-8'));
+  return parseDvAnalyzerSummaryTxt(await readFile(path, 'utf-8'));
 }
 
 export async function loadPbf(path) {
-  return parsePbf(await fs.readFile(path));
+  return parsePbf(await readFile(path));
 }
 
 export async function loadMplayerEdl(path) {
-  return parseMplayerEdl(await fs.readFile(path, 'utf-8'));
+  return parseMplayerEdl(await readFile(path, 'utf-8'));
 }
 
 export async function loadCue(path) {
@@ -45,41 +45,40 @@ export async function loadCue(path) {
 }
 
 export async function loadSrt(path) {
-  return parseSrt(await fs.readFile(path, 'utf-8'));
+  return parseSrt(await readFile(path, 'utf-8'));
 }
 
 export async function saveCsv(path, cutSegments) {
-  await fs.writeFile(path, await formatCsvSeconds(cutSegments));
+  await writeFile(path, await formatCsvSeconds(cutSegments));
 }
 
 export async function saveCsvHuman(path, cutSegments) {
-  await fs.writeFile(path, await formatCsvHuman(cutSegments));
+  await writeFile(path, await formatCsvHuman(cutSegments));
 }
 
 export async function saveCsvFrames({ path, cutSegments, getFrameCount }) {
-  await fs.writeFile(path, await formatCsvFrames({ cutSegments, getFrameCount }));
+  await writeFile(path, await formatCsvFrames({ cutSegments, getFrameCount }));
 }
 
 export async function saveTsv(path, cutSegments) {
-  await fs.writeFile(path, await formatTsv(cutSegments));
+  await writeFile(path, await formatTsv(cutSegments));
 }
 
 export async function saveSrt(path, cutSegments) {
-  await fs.writeFile(path, await formatSrt(cutSegments));
+  await writeFile(path, await formatSrt(cutSegments));
 }
 
-
 export async function saveLlcProject({ savePath, filePath, cutSegments }) {
   const projectData = {
     version: 1,
     mediaFileName: basename(filePath),
     cutSegments: cutSegments.map(({ start, end, name, tags }) => ({ start, end, name, tags })),
   };
-  await fs.writeFile(savePath, JSON5.stringify(projectData, null, 2));
+  await writeFile(savePath, JSON5.stringify(projectData, null, 2));
 }
 
 export async function loadLlcProject(path) {
-  return JSON5.parse(await fs.readFile(path));
+  return JSON5.parse(await readFile(path));
 }
 
 

src/hooks/useFfmpegOperations.js

@@ -3,7 +3,7 @@ import flatMap from 'lodash/flatMap';
 import sum from 'lodash/sum';
 import pMap from 'p-map';
 
-import { getSuffixedOutPath, transferTimestamps, getOutFileExtension, getOutDir, deleteDispositionValue, getHtml5ifiedPath } from '../util';
+import { getSuffixedOutPath, transferTimestamps, getOutFileExtension, getOutDir, deleteDispositionValue, getHtml5ifiedPath, unlinkWithRetry } from '../util';
 import { isCuttingStart, isCuttingEnd, runFfmpegWithProgress, getFfCommandLine, getDuration, createChaptersFromSegments, readFileMeta, cutEncodeSmartPart, getExperimentalArgs, html5ify as ffmpegHtml5ify, getVideoTimescaleArgs, logStdoutStderr, runFfmpegConcat } from '../ffmpeg';
 import { getMapStreamsArgs, getStreamIdsToCopy } from '../util/streams';
 import { getSmartCutParams } from '../smartcut';
@@ -11,7 +11,7 @@ import { isDurationValid } from '../segments';
 
 const { join, resolve, dirname } = window.require('path');
 const { pathExists } = window.require('fs-extra');
-const { writeFile, unlink, mkdir } = window.require('fs/promises');
+const { writeFile, mkdir } = window.require('fs/promises');
 
 async function writeChaptersFfmetadata(outDir, chapters) {
   if (!chapters || chapters.length === 0) return undefined;
@@ -52,9 +52,9 @@ function getMatroskaFlags() {
 
 const getChaptersInputArgs = (ffmetadataPath) => (ffmetadataPath ? ['-f', 'ffmetadata', '-i', ffmetadataPath] : []);
 
-const tryDeleteFiles = async (paths) => pMap(paths, (path) => {
+async function tryDeleteFiles(paths) {
-  unlink(path).catch((err) => console.error('Failed to delete', path, err));
+  return pMap(paths, (path) => unlinkWithRetry(path).catch((err) => console.error('Failed to delete', path, err)), { concurrency: 5 });
-}, { concurrency: 5 });
+}
 
 function useFfmpegOperations({ filePath, treatInputFileModifiedTimeAsStart, treatOutputFileModifiedTimeAsStart, needSmartCut, enableOverwriteOutput, outputPlaybackRate }) {
   const shouldSkipExistingFile = useCallback(async (path) => {

src/hooks/useFrameCapture.js

@@ -1,14 +1,13 @@
 import dataUriToBuffer from 'data-uri-to-buffer';
 import pMap from 'p-map';
 
-import { getSuffixedOutPath, getOutDir, transferTimestamps, getSuffixedFileName, getOutPath, escapeRegExp } from '../util';
+import { getSuffixedOutPath, getOutDir, transferTimestamps, getSuffixedFileName, getOutPath, escapeRegExp, fsOperationWithRetry } from '../util';
 import { getNumDigits } from '../segments';
 
 import { captureFrame as ffmpegCaptureFrame, captureFrames as ffmpegCaptureFrames } from '../ffmpeg';
 
-const fs = window.require('fs-extra');
 const mime = window.require('mime-types');
-const { rename, readdir } = window.require('fs/promises');
+const { rename, readdir, writeFile } = window.require('fs/promises');
 
 
 function getFrameFromVideo(video, format, quality) {
@@ -63,7 +62,7 @@ export default ({ formatTimecode, treatOutputFileModifiedTimeAsStart }) => {
       const duration = formatTimecode({ seconds: fromTime + (frameNum / fps), fileNameFriendly: true });
       const renameFromPath = getOutPath({ customOutDir, filePath, fileName });
       const renameToPath = getOutPath({ customOutDir, filePath, fileName: getSuffixedFileName(filePath, getSuffix(duration, captureFormat)) });
-      await rename(renameFromPath, renameToPath);
+      await fsOperationWithRetry(async () => rename(renameFromPath, renameToPath));
       return renameToPath;
     }, { concurrency: 1 });
 
@@ -87,7 +86,7 @@ export default ({ formatTimecode, treatOutputFileModifiedTimeAsStart }) => {
     const time = formatTimecode({ seconds: currentTime, fileNameFriendly: true });
 
     const outPath = getSuffixedOutPath({ customOutDir, filePath, nameSuffix: `${time}.${ext}` });
-    await fs.writeFile(outPath, buf);
+    await writeFile(outPath, buf);
 
     await transferTimestamps({ inPath: filePath, outPath, cutFrom: currentTime, treatOutputFileModifiedTimeAsStart });
     return outPath;

src/util.js

@@ -11,13 +11,11 @@ import { ffmpegExtractWindow } from './util/constants';
 
 const { dirname, parse: parsePath, join, extname, isAbsolute, resolve, basename } = window.require('path');
 const fsExtra = window.require('fs-extra');
-const { stat, readdir } = window.require('fs/promises');
+const { stat, readdir, utimes, unlink } = window.require('fs/promises');
 const os = window.require('os');
 const { ipcRenderer } = window.require('electron');
 const remote = window.require('@electron/remote');
 
-const { unlink } = fsExtra;
-
 
 const trashFile = async (path) => ipcRenderer.invoke('tryTrashItem', path);
 
@@ -95,6 +93,30 @@ export async function dirExists(dirPath) {
   return (await pathExists(dirPath)) && (await fsExtra.lstat(dirPath)).isDirectory();
 }
 
+// const testFailFsOperation = isDev;
+const testFailFsOperation = false;
+
+// Retry because sometimes write operations fail on windows due to the file being locked for various reasons (often anti-virus) #272 #1797 #1704
+export async function fsOperationWithRetry(operation, { signal, retries = 10, minTimeout = 100, maxTimeout = 2000, ...opts }) {
+  return pRetry(async () => {
+    if (testFailFsOperation && Math.random() > 0.3) throw Object.assign(new Error('test delete failure'), { code: 'EPERM' });
+    await operation();
+  }, {
+    retries,
+    signal,
+    minTimeout,
+    maxTimeout,
+    // mimic fs.rm `maxRetries` https://nodejs.org/api/fs.html#fspromisesrmpath-options
+    shouldRetry: (err) => err instanceof Error && 'code' in err && ['EBUSY', 'EMFILE', 'ENFILE', 'EPERM'].includes(err.code),
+    ...opts,
+  });
+}
+
+// example error: index-18074aaf.js:166 Failed to delete C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-00.01.04.915-00.01.07.424-seg1.mp4 Error: EPERM: operation not permitted, unlink 'C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-00.01.04.915-00.01.07.424-seg1.mp4'
+export const unlinkWithRetry = async (path, options) => fsOperationWithRetry(async () => unlink(path), { ...options, onFailedAttempt: (error) => console.warn('Retrying delete', path, error.attemptNumber) });
+// example error: index-18074aaf.js:160 Error: EPERM: operation not permitted, utime 'C:\Users\USERNAME\Desktop\RC\New folder\2023-12-27 21-45-22 (GMT p5)-merged-1703933052361-cut-merged-1703933070237.mp4'
+export const utimesWithRetry = async (path, atime, mtime, options) => fsOperationWithRetry(async () => utimes(path, atime, mtime), { ...options, onFailedAttempt: (error) => console.warn('Retrying utimes', path, error.attemptNumber) });
+
 export async function transferTimestamps({ inPath, outPath, cutFrom = 0, cutTo = 0, duration = 0, treatInputFileModifiedTimeAsStart = true, treatOutputFileModifiedTimeAsStart }) {
   if (treatOutputFileModifiedTimeAsStart == null) return; // null means disabled;
 
@@ -115,7 +137,7 @@ export async function transferTimestamps({ inPath, outPath, cutFrom = 0, cutTo =
 
   try {
     const { atime, mtime } = await stat(inPath);
-    await fsExtra.utimes(outPath, calculateTime((atime.getTime() / 1000)), calculateTime((mtime.getTime() / 1000)));
+    await utimesWithRetry(outPath, calculateTime((atime.getTime() / 1000)), calculateTime((mtime.getTime() / 1000)));
   } catch (err) {
     console.error('Failed to set output file modified time', err);
   }
@@ -239,13 +261,10 @@ export function getHtml5ifiedPath(cod, fp, type) {
 export async function deleteFiles({ paths, deleteIfTrashFails, signal }) {
   const failedToTrashFiles = [];
 
-  // const testFail = isDev;
-  const testFail = false;
-
   // eslint-disable-next-line no-restricted-syntax
   for (const path of paths) {
     try {
-      if (testFail) throw new Error('test trash failure');
+      if (testFailFsOperation) throw new Error('test trash failure');
       // eslint-disable-next-line no-await-in-loop
       await trashFile(path);
       signal.throwIfAborted();
@@ -267,19 +286,7 @@ export async function deleteFiles({ paths, deleteIfTrashFails, signal }) {
     if (!value) return;
   }
 
-  // Retry because sometimes it fails on windows #272 #1797
+  await pMap(failedToTrashFiles, async (path) => unlinkWithRetry(path, { signal }), { concurrency: 5 });
-  await pMap(failedToTrashFiles, async (path) => {
-    await pRetry(async () => {
-      if (testFail) throw new Error('test delete failure');
-      await unlink(path);
-    }, {
-      retries: 3,
-      signal,
-      onFailedAttempt: async () => {
-        console.warn('Retrying delete', path);
-      },
-    });
-  }, { concurrency: 1 });
 }
 
 export const deleteDispositionValue = 'llc_disposition_remove';