From c8a80460560c4d4fdaa7554fb22002dea64067d9 Mon Sep 17 00:00:00 2001 From: Isaac Grynsztein Date: Mon, 25 May 2020 15:35:49 -0400 Subject: [PATCH] Fixed bug where if a subscription name was missing, the wrong folder was used Fixed bug that caused high CPU usage when the subscriptions check took longer than the interval in the settings (thus they piled on) --- backend/app.js | 32 +++++++++++++++--------- backend/subscriptions.js | 53 ++++++++++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 26 deletions(-) diff --git a/backend/app.js b/backend/app.js index d7b6a68..8177f20 100644 --- a/backend/app.js +++ b/backend/app.js @@ -165,7 +165,9 @@ var validDownloadingAgents = [ 'ffmpeg', 'httpie', 'wget' -] +]; + +const subscription_timeouts = {}; // don't overwrite config if it already happened.. NOT // let alreadyWritten = db.get('configWriteFlag').value(); @@ -616,15 +618,14 @@ function loadConfigValues() { logger.transports[2].level = logger_level; } -function calculateSubcriptionRetrievalDelay(amount) { - // frequency is 5 mins - let frequency_in_ms = subscriptionsCheckInterval * 1000; - let minimum_frequency = 60 * 1000; - const first_frequency = frequency_in_ms/amount; - return (first_frequency < minimum_frequency) ? minimum_frequency : first_frequency; +function calculateSubcriptionRetrievalDelay(subscriptions_amount) { + // frequency is once every 5 mins by default + let interval_in_ms = subscriptionsCheckInterval * 1000; + const subinterval_in_ms = interval_in_ms/subscriptions_amount; + return subinterval_in_ms; } -function watchSubscriptions() { +async function watchSubscriptions() { let subscriptions = null; const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); @@ -646,10 +647,19 @@ function watchSubscriptions() { let current_delay = 0; for (let i = 0; i < subscriptions.length; i++) { let sub = subscriptions[i]; + + // don't check the sub if the last check for the same subscription has not completed + if (subscription_timeouts[sub.id]) { + logger.verbose(`Subscription: skipped checking ${sub.name} as the last check for ${sub.name} has not completed.`); + continue; + } + logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval); - setTimeout(() => { - subscriptions_api.getVideosForSub(sub, sub.user_uid); + setTimeout(async () => { + await subscriptions_api.getVideosForSub(sub, sub.user_uid); + subscription_timeouts[sub.id] = false; }, current_delay); + subscription_timeouts[sub.id] = true; current_delay += delay_interval; if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0; } @@ -2212,7 +2222,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { else base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); - let appended_base_path = path.join(base_path, subscription.isPlaylist ? 'playlists' : 'channels', subscription.name, '/'); + let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/'); let files; try { files = recFindByExt(appended_base_path, 'mp4'); diff --git a/backend/subscriptions.js b/backend/subscriptions.js index e5ba918..811c771 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -240,10 +240,10 @@ async function getVideosForSub(sub, user_uid = null) { if (sub.name) { appendedBasePath = getAppendedBasePath(sub, basePath); } else { - appendedBasePath = basePath + (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s'); + appendedBasePath = path.join(basePath, (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s')); } - let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-annotations', '--write-thumbnail', '--write-info-json', '--print-json']; + let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-info-json', '--print-json']; let archive_dir = null; let archive_path = null; @@ -275,10 +275,28 @@ async function getVideosForSub(sub, user_uid = null) { } // get videos - logger.verbose('Subscribe: getting videos for subscription ' + sub.name); + logger.verbose('Subscription: getting videos for subscription ' + sub.name); youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { - if (err) { + logger.verbose('Subscription: finished check for ' + sub.name); + if (err && !output) { logger.error(err.stderr); + if (err.stderr.includes('This video is unavailable')) { + logger.info('An error was encountered with at least one video, backup method will be used.') + try { + const outputs = err.stdout.split(/\r\n|\r|\n/); + for (let i = 0; i < outputs.length; i++) { + const output = JSON.parse(outputs[i]); + handleOutputJSON(sub, sub_db, output, i === 0) + if (err.stderr.includes(output['id']) && archive_path) { + // we found a video that errored! add it to the archive to prevent future errors + fs.appendFileSync(archive_path, output['id']); + } + } + } catch(e) { + logger.error('Backup method failed. See error below:'); + logger.error(e); + } + } resolve(false); } else if (output) { if (output.length === 0 || (output.length === 1 && output[0] === '')) { @@ -296,17 +314,8 @@ async function getVideosForSub(sub, user_uid = null) { continue; } - if (sub.streamingOnly) { - if (i === 0) { - sub_db.assign({videos: []}).write(); - } - - // remove unnecessary info - output_json.formats = null; - - // add to db - sub_db.get('videos').push(output_json).write(); - } + const reset_videos = i === 0; + handleOutputJSON(sub, sub_db, output_json, reset_videos); // TODO: Potentially store downloaded files in db? @@ -317,6 +326,20 @@ async function getVideosForSub(sub, user_uid = null) { }); } +function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) { + if (sub.streamingOnly) { + if (reset_videos) { + sub_db.assign({videos: []}).write(); + } + + // remove unnecessary info + output_json.formats = null; + + // add to db + sub_db.get('videos').push(output_json).write(); + } +} + function getAllSubscriptions(user_uid = null) { if (user_uid) return users_db.get('users').find({uid: user_uid}).get('subscriptions').value();