diff --git a/backend/downloader.js b/backend/downloader.js index 8658502..1c57534 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -314,13 +314,14 @@ async function downloadQueuedFile(download_uid) { let difference = (end_time - start_time)/1000; logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); clearInterval(download_checker); - if (err) { + const parsed_output = utils.parseOutputJSON(output, err); + if (!parsed_output) { logger.error(err.stderr); await handleDownloadError(download, err.stderr, 'unknown_error'); resolve(false); return; - } else if (output) { - if (output.length === 0 || output[0].length === 0) { + } else if (parsed_output) { + if (parsed_output.length === 0 || parsed_output[0].length === 0) { // ERROR! const error_message = `No output received for video download, check if it exists in your archive.`; await handleDownloadError(download, error_message, 'no_output'); @@ -329,17 +330,7 @@ async function downloadQueuedFile(download_uid) { return; } - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - // we have to do this because sometimes there will be leading characters before the actual json - const start_idx = output[i].indexOf('{"'); - const clean_output = output[i].slice(start_idx, output[i].length); - output_json = JSON.parse(clean_output); - } catch(e) { - output_json = null; - } - + for (const output_json of parsed_output) { if (!output_json) { continue; } @@ -564,33 +555,9 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => { new_args.push('--dump-json'); youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => { - if (output) { - let outputs = []; - try { - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - } catch(e) { - output_json = null; - } - - if (!output_json) { - continue; - } - - outputs.push(output_json); - } - resolve(outputs.length === 1 ? outputs[0] : outputs); - } catch(e) { - const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`; - logger.error(error); - if (download_uid) { - const download = await db_api.getRecord('download_queue', {uid: download_uid}); - await handleDownloadError(download, error, 'parse_failed'); - } - resolve(null); - } + const parsed_output = utils.parseOutputJSON(output, err); + if (parsed_output) { + resolve(parsed_output); } else { let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; if (err.stderr) error_message += `\n\n${err.stderr}`; diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 03c07b4..c1bbefe 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -254,26 +254,15 @@ exports.getVideosForSub = async (sub, user_uid = null) => { } logger.verbose('Subscription: finished check for ' + sub.name); - if (err && !output) { - logger.error(err.stderr ? err.stderr : err.message); - if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) { - logger.info('An error was encountered with at least one video, backup method will be used.') - try { - const outputs = err.stdout.split(/\r\n|\r|\n/); - const files_to_download = await handleOutputJSON(outputs, sub, user_uid); - resolve(files_to_download); - } catch(e) { - logger.error('Backup method failed. See error below:'); - logger.error(e); - } - } else { - logger.error('Subscription check failed!'); - } - resolve(false); - } else if (output) { - const files_to_download = await handleOutputJSON(output, sub, user_uid); - resolve(files_to_download); - } + const processed_output = utils.parseOutputJSON(output, err); + if (!processed_output) { + logger.error('Subscription check failed!'); + resolve(null); + return; + } + const files_to_download = await handleOutputJSON(processed_output, sub, user_uid); + resolve(files_to_download); + return; }); }, err => { logger.error(err); @@ -281,31 +270,17 @@ exports.getVideosForSub = async (sub, user_uid = null) => { }); } -async function handleOutputJSON(output, sub, user_uid) { +async function handleOutputJSON(output_jsons, sub, user_uid) { if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) { await setFreshUploads(sub, user_uid); checkVideosForFreshUploads(sub, user_uid); } - if (output.length === 0 || (output.length === 1 && output[0] === '')) { + if (output_jsons.length === 0 || (output_jsons.length === 1 && output_jsons[0] === '')) { logger.verbose('No additional videos to download for ' + sub.name); return []; } - const output_jsons = []; - for (let i = 0; i < output.length; i++) { - let output_json = null; - try { - output_json = JSON.parse(output[i]); - output_jsons.push(output_json); - } catch(e) { - output_json = null; - } - if (!output_json) { - continue; - } - } - const files_to_download = await getFilesToDownload(sub, output_jsons); const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid); diff --git a/backend/utils.js b/backend/utils.js index 157b6f0..8c6a1ae 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -530,6 +530,37 @@ exports.getDirectoriesInDirectory = async (basePath) => { } } +exports.parseOutputJSON = (output, err) => { + let split_output = []; + // const output_jsons = []; + if (err && !output) { + if (!err.stderr.includes('This video is unavailable') && !err.stderr.includes('Private video')) { + return null; + } + logger.info('An error was encountered with at least one video, backup method will be used.') + try { + split_output = err.stdout.split(/\r\n|\r|\n/); + } catch (e) { + logger.error('Backup method failed. See error below:'); + logger.error(e); + return null; + } + } else { + for (const output_item of output) { + // we have to do this because sometimes there will be leading characters before the actual json + const start_idx = output_item.indexOf('{"'); + const clean_output = output_item.slice(start_idx, output_item.length); + split_output.push(clean_output); + } + } + + try { + return split_output.map(split_output_str => JSON.parse(split_output_str)); + } catch(e) { + return null; + } +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {