aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Combined output json processing from youtube-dl between subscriptions and one-off downloads

Browse Source 
Added workaround for missing video in playlist to one-off downloads
pull/935/head
Isaac Abadi 2 years ago
parent 2a1af69f1f
commit 8212acbac6
3 changed files with 50 additions and 77 deletions
Show Stats Download Patch File Download Diff File

49
backend/downloader.js
Unescape Escape View File

@ -314,13 +314,14 @@ async function downloadQueuedFile(download_uid) {
let difference = (end_time - start_time)/1000; let difference = (end_time - start_time)/1000;
logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`); logger.debug(`${type === 'audio' ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
clearInterval(download_checker); clearInterval(download_checker);
if (err) { const parsed_output = utils.parseOutputJSON(output, err);
if (!parsed_output) {
logger.error(err.stderr); logger.error(err.stderr);
await handleDownloadError(download, err.stderr, 'unknown_error'); await handleDownloadError(download, err.stderr, 'unknown_error');
resolve(false); resolve(false);
return; return;
} else if (output) { } else if (parsed_output) {
if (output.length === 0 || output[0].length === 0) { if (parsed_output.length === 0 || parsed_output[0].length === 0) {
// ERROR! // ERROR!
const error_message = `No output received for video download, check if it exists in your archive.`; const error_message = `No output received for video download, check if it exists in your archive.`;
await handleDownloadError(download, error_message, 'no_output'); await handleDownloadError(download, error_message, 'no_output');
@ -329,17 +330,7 @@ async function downloadQueuedFile(download_uid) {
return; return;
} }
for (let i = 0; i < output.length; i++) { for (const output_json of parsed_output) {
let output_json = null;
try {
// we have to do this because sometimes there will be leading characters before the actual json
const start_idx = output[i].indexOf('{"');
const clean_output = output[i].slice(start_idx, output[i].length);
output_json = JSON.parse(clean_output);
} catch(e) {
output_json = null;
}
if (!output_json) { if (!output_json) {
continue; continue;
} }
@ -564,33 +555,9 @@ exports.getVideoInfoByURL = async (url, args = [], download_uid = null) => {
new_args.push('--dump-json'); new_args.push('--dump-json');
youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => { youtubedl.exec(url, new_args, {maxBuffer: Infinity}, async (err, output) => {
if (output) { const parsed_output = utils.parseOutputJSON(output, err);
let outputs = []; if (parsed_output) {
try { resolve(parsed_output);
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
outputs.push(output_json);
}
resolve(outputs.length === 1 ? outputs[0] : outputs);
} catch(e) {
const error = `Error while retrieving info on video with URL ${url} with the following message: output JSON could not be parsed. Output JSON: ${output}`;
logger.error(error);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error, 'parse_failed');
}
resolve(null);
}
} else { } else {
let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`; let error_message = `Error while retrieving info on video with URL ${url} with the following message: ${err}`;
if (err.stderr) error_message += `\n\n${err.stderr}`; if (err.stderr) error_message += `\n\n${err.stderr}`;

47
backend/subscriptions.js
Unescape Escape View File

@ -254,26 +254,15 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
} }
logger.verbose('Subscription: finished check for ' + sub.name); logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) { const processed_output = utils.parseOutputJSON(output, err);
logger.error(err.stderr ? err.stderr : err.message); if (!processed_output) {
if (err.stderr.includes('This video is unavailable') || err.stderr.includes('Private video')) { logger.error('Subscription check failed!');
logger.info('An error was encountered with at least one video, backup method will be used.') resolve(null);
try { return;
const outputs = err.stdout.split(/\r\n|\r|\n/); }
const files_to_download = await handleOutputJSON(outputs, sub, user_uid); const files_to_download = await handleOutputJSON(processed_output, sub, user_uid);
resolve(files_to_download); resolve(files_to_download);
} catch(e) { return;
logger.error('Backup method failed. See error below:');
logger.error(e);
}
} else {
logger.error('Subscription check failed!');
}
resolve(false);
} else if (output) {
const files_to_download = await handleOutputJSON(output, sub, user_uid);
resolve(files_to_download);
}
}); });
}, err => { }, err => {
logger.error(err); logger.error(err);
@ -281,31 +270,17 @@ exports.getVideosForSub = async (sub, user_uid = null) => {
}); });
} }
async function handleOutputJSON(output, sub, user_uid) { async function handleOutputJSON(output_jsons, sub, user_uid) {
if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) { if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
await setFreshUploads(sub, user_uid); await setFreshUploads(sub, user_uid);
checkVideosForFreshUploads(sub, user_uid); checkVideosForFreshUploads(sub, user_uid);
} }
if (output.length === 0 || (output.length === 1 && output[0] === '')) { if (output_jsons.length === 0 || (output_jsons.length === 1 && output_jsons[0] === '')) {
logger.verbose('No additional videos to download for ' + sub.name); logger.verbose('No additional videos to download for ' + sub.name);
return []; return [];
} }
const output_jsons = [];
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
output_jsons.push(output_json);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
}
const files_to_download = await getFilesToDownload(sub, output_jsons); const files_to_download = await getFilesToDownload(sub, output_jsons);
const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid); const base_download_options = exports.generateOptionsForSubscriptionDownload(sub, user_uid);

31
backend/utils.js
Unescape Escape View File

@ -530,6 +530,37 @@ exports.getDirectoriesInDirectory = async (basePath) => {
} }
} }
exports.parseOutputJSON = (output, err) => {
let split_output = [];
// const output_jsons = [];
if (err && !output) {
if (!err.stderr.includes('This video is unavailable') && !err.stderr.includes('Private video')) {
return null;
}
logger.info('An error was encountered with at least one video, backup method will be used.')
try {
split_output = err.stdout.split(/\r\n|\r|\n/);
} catch (e) {
logger.error('Backup method failed. See error below:');
logger.error(e);
return null;
}
} else {
for (const output_item of output) {
// we have to do this because sometimes there will be leading characters before the actual json
const start_idx = output_item.indexOf('{"');
const clean_output = output_item.slice(start_idx, output_item.length);
split_output.push(clean_output);
}
}
try {
return split_output.map(split_output_str => JSON.parse(split_output_str));
} catch(e) {
return null;
}
}
// objects // objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {

Write Preview
Loading…
Cancel
Save