Combined output json processing from youtube-dl between subscriptions and one-off downloads

Added workaround for missing video in playlist to one-off downloads

backend/downloader.js

@@ -314,13 +314,14 @@ async function downloadQueuedFile(download_uid) {
             let difference = (end_time - start_time)/1000;
             logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
             clearInterval(download_checker);
-            if (err) {
+            const parsed_output = utils.parseOutputJSON(output, err);
+            if (!parsed_output) {
                 logger.error(err.stderr);
                 await handleDownloadError(download, err.stderr, 'unknown_error');
                 resolve(false);
                 return;
-            } else if (output) {
-                if (output.length === 0 || output[0].length === 0) {
+            } else if (parsed_output) {
+                if (parsed_output.length === 0 || parsed_output[0].length === 0) {
                     // ERROR!
                     const error_message = `No output received for video download, check if it exists in your archive.`;
                     await handleDownloadError(download, error_message, 'no_output');
@@ -329,17 +330,7 @@ async function downloadQueuedFile(download_uid) {
                     return;
                 }
 
-                for (let i = 0; i < output.length; i++) {
-                    let output_json = null;
-                    try {
-                        // we have to do this because sometimes there will be leading characters before the actual json
-                        const start_idx = output[i].indexOf('{"');
-                        const clean_output = output[i].slice(start_idx, output[i].length);
-                        output_json = JSON.parse(clean_output);
-                    } catch(e) {
-                        output_json = null;
-                    }
-
+                for (const output_json of parsed_output) {
                     if (!output_json) {
                         continue;
                     }
@@ -564,33 +555,9 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
         new_args.push('--dump-json');
 
         youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
-            if (output) {
-                let outputs = [];
-                try {
-                    for (let i = 0; i < output.length; i++) {
-                        let output_json = null;
-                        try {
-                            output_json = JSON.parse(output[i]);
-                        } catch(e) {
-                            output_json = null;
-                        }
-    
-                        if (!output_json) {
-                            continue;
-                        }
-
-                        outputs.push(output_json);
-                    }
-                    resolve(outputs.length === 1 ? outputs[0] : outputs);
-                } catch(e) {
-                    const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
-                    logger.error(error);
-                    if (download_uid) {
-                        const download = await db_api.getRecord('download_queue', {uid: download_uid});
-                        await handleDownloadError(download, error, 'parse_failed');
-                    }
-                    resolve(null);
-                }
+            const parsed_output = utils.parseOutputJSON(output, err);
+            if (parsed_output) {
+                resolve(parsed_output);
             } else {
                 let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
                 if (err.stderr) error_message += `\n\n${err.stderr}`;

backend/subscriptions.js

@@ -254,26 +254,15 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
             }
 
             logger.verbose('Subscription: finished check for ' + sub.name);
-            if (err && !output) {
-                logger.error(err.stderr ? err.stderr : err.message);
-                if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) {
-                    logger.info('An error was encountered with at least one video, backup method will be used.')
-                    try {
-                        const outputs = err.stdout.split(/\r\n|\r|\n/);
-                        const files_to_download = await handleOutputJSON(outputs, sub, user_uid);
-                        resolve(files_to_download);
-                    } catch(e) {
-                        logger.error('Backup method failed. See error below:');
-                        logger.error(e);
-                    }
-                } else {
+            const processed_output = utils.parseOutputJSON(output, err);
+            if (!processed_output) {
                 logger.error('Subscription check failed!');
+                resolve(null);
+                return;
             }
-                resolve(false);
-            } else if (output) {
-                const files_to_download = await handleOutputJSON(output, sub, user_uid);
+            const files_to_download = await handleOutputJSON(processed_output, sub, user_uid);
             resolve(files_to_download);
-           }
+            return;
         });
     }, err => {
         logger.error(err);
@@ -281,31 +270,17 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
     });
 }
 
-async function handleOutputJSON(output, sub, user_uid) {
+async function handleOutputJSON(output_jsons, sub, user_uid) {
     if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
         await setFreshUploads(sub, user_uid);
         checkVideosForFreshUploads(sub, user_uid);
     }
 
-    if (output.length === 0 || (output.length === 1 && output[0] === '')) {
+    if (output_jsons.length === 0 || (output_jsons.length === 1 && output_jsons[0] === '')) {
         logger.verbose('No additional videos to download for ' + sub.name);
         return [];
     }
 
-    const output_jsons = [];
-    for (let i = 0; i < output.length; i++) {
-        let output_json = null;
-        try {
-            output_json = JSON.parse(output[i]);
-            output_jsons.push(output_json);
-        } catch(e) {
-            output_json = null;
-        }
-        if (!output_json) {
-            continue;
-        }
-    }
-
     const files_to_download = await getFilesToDownload(sub, output_jsons);
     const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);
 

backend/utils.js

@@ -530,6 +530,37 @@ exports.getDirectoriesInDirectory = async (basePath) => {
     }
 }
 
+exports.parseOutputJSON = (output, err) => {
+    let split_output = [];
+    // const output_jsons = [];
+    if (err && !output) {
+        if (!err.stderr.includes('This video is unavailable') && !err.stderr.includes('Private video')) {
+            return null;
+        }
+        logger.info('An error was encountered with at least one video, backup method will be used.')
+        try {
+            split_output = err.stdout.split(/\r\n|\r|\n/);
+        } catch (e) {
+            logger.error('Backup method failed. See error below:');
+            logger.error(e);
+            return null;
+        }
+    }   else {
+        for (const output_item of output) {
+            // we have to do this because sometimes there will be leading characters before the actual json
+            const start_idx = output_item.indexOf('{"');
+            const clean_output = output_item.slice(start_idx, output_item.length);
+            split_output.push(clean_output);
+        }
+    }
+
+    try {
+        return split_output.map(split_output_str => JSON.parse(split_output_str));
+    } catch(e) {
+        return null;
+    }
+}
+
 // objects
 
 function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {