YoutubeDL-Material/backend/subscriptions.js

const FileSync = require('lowdb/adapters/FileSync')

var fs = require('fs-extra');
const { uuid } = require('uuidv4');
var path = require('path');

var youtubedl = require('youtube-dl');
const config_api = require('./config');
const twitch_api = require('./twitch');
var utils = require('./utils');

const debugMode = process.env.YTDL_MODE === 'debug';

var logger = null;
var db = null;
var users_db = null;
let db_api = null;

function setDB(input_db_api) { db_api = input_db_api }
function setLogger(input_logger) { logger = input_logger; }

function initialize(input_db_api, input_logger) {
    setDB(input_db_api);
    setLogger(input_logger);
}

async function subscribe(sub, user_uid = null) {
    const result_obj = {
        success: false,
        error: ''
    };
    return new Promise(async resolve => {
        // sub should just have url and name. here we will get isPlaylist and path
        sub.isPlaylist = sub.url.includes('playlist');
        sub.videos = [];

        let url_exists = !!(await db_api.getRecord('subscriptions', {url: sub.url, user_uid: user_uid}));

        if (!sub.name && url_exists) {
            logger.error(`Sub with the same URL "${sub.url}" already exists -- please provide a custom name for this new subscription.`);
            result_obj.error = 'Subcription with URL ' + sub.url + ' already exists! Custom name is required.';
            resolve(result_obj);
            return;
        }

        sub['user_uid'] = user_uid ? user_uid : undefined;
        await db_api.insertRecordIntoTable('subscriptions', sub);

        let success = await getSubscriptionInfo(sub, user_uid);

        if (success) {
            getVideosForSub(sub, user_uid);
        } else {
            logger.error('Subscribe: Failed to get subscription info. Subscribe failed.')
        };

        result_obj.success = success;
        result_obj.sub = sub;
        resolve(result_obj);
    });

}

async function getSubscriptionInfo(sub, user_uid = null) {
    let basePath = null;
    if (user_uid)
        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
    else
        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');

    // get videos
    let downloadConfig = ['--dump-json', '--playlist-end', '1'];
    let useCookies = config_api.getConfigItem('ytdl_use_cookies');
    if (useCookies) {
        if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
            downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
        } else {
            logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
        }
    }

    return new Promise(async resolve => {
        youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
            if (debugMode) {
                logger.info('Subscribe: got info for subscription ' + sub.id);
            }
            if (err) {
                logger.error(err.stderr);
                resolve(false);
            } else if (output) {
                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
                    logger.verbose('Could not get info for ' + sub.id);
                    resolve(false);
                }
                for (let i = 0; i < output.length; i++) {
                    let output_json = null;
                    try {
                        output_json = JSON.parse(output[i]);
                    } catch(e) {
                        output_json = null;
                    }
                    if (!output_json) {
                        continue;
                    }
                    if (!sub.name) {
                        if (sub.isPlaylist) {
                            sub.name = output_json.playlist_title ? output_json.playlist_title : output_json.playlist;
                        } else {
                            sub.name = output_json.uploader;
                        }
                        // if it's now valid, update
                        if (sub.name) {
                            await db_api.updateRecord('subscriptions', {id: sub.id}, {name: sub.name});
                        }
                    }

                    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
                    if (useArchive && !sub.archive) {
                        // must create the archive
                        const archive_dir = path.join(__dirname, basePath, 'archives', sub.name);
                        const archive_path = path.join(archive_dir, 'archive.txt');

                        // creates archive directory and text file if it doesn't exist
                        fs.ensureDirSync(archive_dir);
                        fs.ensureFileSync(archive_path);

                        // updates subscription
                        sub.archive = archive_dir;

                        await db_api.updateRecord('subscriptions', {id: sub.id}, {archive: archive_dir});
                    }

                    // TODO: get even more info

                    resolve(true);
                }
                resolve(false);
            }
        });
    });
}

async function unsubscribe(sub, deleteMode, user_uid = null) {
    let basePath = null;
    if (user_uid)
        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
    else
        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
    let result_obj = { success: false, error: '' };

    let id = sub.id;

    const sub_files = await db_api.getRecords('files', {sub_id: id});
    for (let i = 0; i < sub_files.length; i++) {
        const sub_file = sub_files[i];
        if (config_api.descriptors[sub_file['uid']]) {
            try {
                for (let i = 0; i < config_api.descriptors[sub_file['uid']].length; i++) {
                    config_api.descriptors[sub_file['uid']][i].destroy();
                }
            } catch(e) {
    
            }
        }
    }

    await db_api.removeRecord('subscriptions', {id: id});
    await db_api.removeAllRecords('files', {sub_id: id});

    // failed subs have no name, on unsubscribe they shouldn't error
    if (!sub.name) {
        return;
    }

    const appendedBasePath = getAppendedBasePath(sub, basePath);
    if (deleteMode && (await fs.pathExists(appendedBasePath))) {
        if (sub.archive && (await fs.pathExists(sub.archive))) {
            const archive_file_path = path.join(sub.archive, 'archive.txt');
            // deletes archive if it exists
            if (await fs.pathExists(archive_file_path)) {
                await fs.unlink(archive_file_path);
            }
            await fs.rmdir(sub.archive);
        }
        await fs.remove(appendedBasePath);
    }
}

async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
    // TODO: combine this with deletefile
    let basePath = null;
    basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
                        : config_api.getConfigItem('ytdl_subscriptions_base_path');
    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
    const appendedBasePath = getAppendedBasePath(sub, basePath);
    const name = file;
    let retrievedID = null;

    await db_api.removeRecord('files', {uid: file_uid});

    let filePath = appendedBasePath;
    const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
    var jsonPath = path.join(__dirname,filePath,name+'.info.json');
    var videoFilePath = path.join(__dirname,filePath,name+ext);
    var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
    var altImageFilePath = path.join(__dirname,filePath,name+'.webp');

    const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
        fs.pathExists(jsonPath),
        fs.pathExists(videoFilePath),
        fs.pathExists(imageFilePath),
        fs.pathExists(altImageFilePath),
    ]);

    if (jsonExists) {
        retrievedID = JSON.parse(await fs.readFile(jsonPath, 'utf8'))['id'];
        await fs.unlink(jsonPath);
    }

    if (imageFileExists) {
        await fs.unlink(imageFilePath);
    }

    if (altImageFileExists) {
        await fs.unlink(altImageFilePath);
    }

    if (videoFileExists) {
        await fs.unlink(videoFilePath);
        if ((await fs.pathExists(jsonPath)) || (await fs.pathExists(videoFilePath))) {
            return false;
        } else {
            // check if the user wants the video to be redownloaded (deleteForever === false)
            if (!deleteForever && useArchive && sub.archive && retrievedID) {
                const archive_path = path.join(sub.archive, 'archive.txt')
                // if archive exists, remove line with video ID
                if (await fs.pathExists(archive_path)) {
                    utils.removeIDFromArchive(archive_path, retrievedID);
                }
            }
            return true;
        }
    } else {
        // TODO: tell user that the file didn't exist
        return true;
    }
}

async function getVideosForSub(sub, user_uid = null) {
    const latest_sub_obj = await getSubscription(sub.id);
    if (!latest_sub_obj || latest_sub_obj['downloading']) {
        return false;
    }

    updateSubscriptionProperty(sub, {downloading: true}, user_uid);

    // get basePath
    let basePath = null;
    if (user_uid)
        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
    else
        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');

    let appendedBasePath = getAppendedBasePath(sub, basePath);
    fs.ensureDirSync(appendedBasePath);

    let multiUserMode = null;
    if (user_uid) {
        multiUserMode = {
            user: user_uid,
            file_path: appendedBasePath
        }
    }

    const downloadConfig = await generateArgsForSubscription(sub, user_uid);

    // get videos
    logger.verbose(`Subscription: getting videos for subscription ${sub.name} with args: ${downloadConfig.join(',')}`);

    return new Promise(async resolve => {
        const preimported_file_paths = [];
        const PREIMPORT_INTERVAL = 5000;
        const preregister_check = setInterval(async () => {
            if (sub.streamingOnly) return;
            await db_api.preimportUnregisteredSubscriptionFile(sub, appendedBasePath);
        }, PREIMPORT_INTERVAL);
        youtubedl.exec(sub.url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
            // cleanup
            updateSubscriptionProperty(sub, {downloading: false}, user_uid);
            clearInterval(preregister_check);

            logger.verbose('Subscription: finished check for ' + sub.name);
            if (err && !output) {
                logger.error(err.stderr ? err.stderr : err.message);
                if (err.stderr.includes('This video is unavailable')) {
                    logger.info('An error was encountered with at least one video, backup method will be used.')
                    try {
                        const outputs = err.stdout.split(/\r\n|\r|\n/);
                        for (let i = 0; i < outputs.length; i++) {
                            const output = JSON.parse(outputs[i]);
                            await handleOutputJSON(sub, output, i === 0, multiUserMode)
                            if (err.stderr.includes(output['id']) && archive_path) {
                                // we found a video that errored! add it to the archive to prevent future errors
                                if (sub.archive) {
                                    archive_dir = sub.archive;
                                    archive_path = path.join(archive_dir, 'archive.txt')
                                    fs.appendFileSync(archive_path, output['id']);
                                }
                            }
                        }
                    } catch(e) {
                        logger.error('Backup method failed. See error below:');
                        logger.error(e);
                    }
                }
                resolve(false);
            } else if (output) {
                if (output.length === 0 || (output.length === 1 && output[0] === '')) {
                    logger.verbose('No additional videos to download for ' + sub.name);
                    resolve(true);
                    return;
                }
                for (let i = 0; i < output.length; i++) {
                    let output_json = null;
                    try {
                        output_json = JSON.parse(output[i]);
                    } catch(e) {
                        output_json = null;
                    }
                    if (!output_json) {
                        continue;
                    }

                    const reset_videos = i === 0;
                    await handleOutputJSON(sub, output_json, multiUserMode, preimported_file_paths, reset_videos);
                }

                if (config_api.getConfigItem('ytdl_subscriptions_redownload_fresh_uploads')) {
                    await setFreshUploads(sub, user_uid);
                    checkVideosForFreshUploads(sub, user_uid);
                }

                resolve(true);
            }
        });
    }, err => {
        logger.error(err);
        updateSubscriptionProperty(sub, {downloading: false}, user_uid);
        clearInterval(preregister_check);
    });
}

async function generateArgsForSubscription(sub, user_uid, redownload = false, desired_path = null) {
    // get basePath
    let basePath = null;
    if (user_uid)
        basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
    else
        basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');

    const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');

    let appendedBasePath = getAppendedBasePath(sub, basePath);

    const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';

    let fullOutput = `${appendedBasePath}/${file_output}.%(ext)s`;
    if (desired_path) {
        fullOutput = `${desired_path}.%(ext)s`;
    } else if (sub.custom_output) {
        fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
    }

    let downloadConfig = ['-o', fullOutput, !redownload ? '-ciw' : '-ci', '--write-info-json', '--print-json'];

    let qualityPath = null;
    if (sub.type && sub.type === 'audio') {
        qualityPath = ['-f', 'bestaudio']
        qualityPath.push('-x');
        qualityPath.push('--audio-format', 'mp3');
    } else {
        if (!sub.maxQuality || sub.maxQuality === 'best') qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4'];
        else qualityPath = ['-f', `bestvideo[height<=${sub.maxQuality}]+bestaudio/best[height<=${sub.maxQuality}]`, '--merge-output-format', 'mp4'];
    }

    downloadConfig.push(...qualityPath)

    if (sub.custom_args) {
        customArgsArray = sub.custom_args.split(',,');
        if (customArgsArray.indexOf('-f') !== -1) {
            // if custom args has a custom quality, replce the original quality with that of custom args
            const original_output_index = downloadConfig.indexOf('-f');
            downloadConfig.splice(original_output_index, 2);
        }
        downloadConfig.push(...customArgsArray);
    }

    let archive_dir = null;
    let archive_path = null;

    if (useArchive && !redownload) {
        if (sub.archive) {
            archive_dir = sub.archive;
            archive_path = path.join(archive_dir, 'archive.txt')
        }
        downloadConfig.push('--download-archive', archive_path);
    }

    // if streaming only mode, just get the list of videos
    if (sub.streamingOnly) {
        downloadConfig = ['-f', 'best', '--dump-json'];
    }

    if (sub.timerange && !redownload) {
        downloadConfig.push('--dateafter', sub.timerange);
    }

    let useCookies = config_api.getConfigItem('ytdl_use_cookies');
    if (useCookies) {
        if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
            downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
        } else {
            logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
        }
    }

    if (config_api.getConfigItem('ytdl_include_thumbnail')) {
        downloadConfig.push('--write-thumbnail');
    }

    const download_delay = config_api.getConfigItem('ytdl_subscriptions_download_delay');
    if (download_delay && downloadConfig.indexOf('--sleep-interval') === -1) {
        if (!(+download_delay)) {
            logger.warn(`Invalid download delay of ${download_delay}, please remember to use non-zero numbers.`);
        } else {
            downloadConfig.push('--sleep-interval', +download_delay);
        }
    }

    const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
    if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
        downloadConfig.push('-r', rate_limit);
    }

    const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
    if (default_downloader === 'yt-dlp') {
        downloadConfig.push('--no-clean-infojson');
    }

    return downloadConfig;
}

async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_videos = false) {
    // TODO: remove streaming only mode
    if (false && sub.streamingOnly) {
        if (reset_videos) {
            sub_db.assign({videos: []}).write();
        }

        // remove unnecessary info
        output_json.formats = null;

        // add to db
        sub_db.get('videos').push(output_json).write();
    } else {
        const file_url = output_json['webpage_url']

        const file_exists = await db_api.getRecord('files', {url: file_url, sub_id: sub.id});
        if (file_exists) {
            // file already exists in DB, return early to avoid reseting the download date
            return;
        }

        await db_api.registerFileDB2(output_json['_filename'], sub.type, sub.user_uid, null, sub.id);

        const url = output_json['webpage_url'];
        if (sub.type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
            && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) {
                const file_name = path.basename(output_json['_filename']);
                const id = file_name.substring(0, file_name.length-4);
                let vodId = url.split('twitch.tv/videos/')[1];
                vodId = vodId.split('?')[0];
                twitch_api.downloadTwitchChatByVODID(vodId, id, sub.type, multiUserMode.user, sub);
        }
    }
}

async function getSubscriptions(user_uid = null) {
    return await db_api.getRecords('subscriptions', {user_uid: user_uid});
}

async function getAllSubscriptions() {
    const all_subs = await db_api.getRecords('subscriptions');
    const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode');
    return all_subs.filter(sub => !!(sub.user_uid) === !!multiUserMode);
}

async function getSubscription(subID) {
    return await db_api.getRecord('subscriptions', {id: subID});
}

async function getSubscriptionByName(subName, user_uid = null) {
    return await db_api.getRecord('subscriptions', {name: subName, user_uid: user_uid});
}

async function updateSubscription(sub, user_uid = null) {
    await db_api.updateRecord('subscriptions', {id: sub.id}, sub);
    return true;
}

async function updateSubscriptionPropertyMultiple(subs, assignment_obj) {
    subs.forEach(async sub => {
        await updateSubscriptionProperty(sub, assignment_obj, sub.user_uid);
    });
}

async function updateSubscriptionProperty(sub, assignment_obj, user_uid = null) {
    // TODO: combine with updateSubscription
    await db_api.updateRecord('subscriptions', {id: sub.id}, assignment_obj);
    return true;
}

async function setFreshUploads(sub, user_uid) {
    const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
    sub.videos.forEach(async video => {
        if (current_date === video['upload_date'].replace(/-/g, '')) {
            // set upload as fresh
            const video_uid = video['uid'];
            await db_api.setVideoProperty(video_uid, {'fresh_upload': true}, user_uid, sub['id']);
        }
    });
}

async function checkVideosForFreshUploads(sub, user_uid) {
    const current_date = new Date().toISOString().split('T')[0].replace(/-/g, '');
    sub.videos.forEach(async video => {
        if (video['fresh_upload'] && current_date > video['upload_date'].replace(/-/g, '')) {
            await checkVideoIfBetterExists(video, sub, user_uid)
        }
    });
}

async function checkVideoIfBetterExists(file_obj, sub, user_uid) {
    const new_path = file_obj['path'].substring(0, file_obj['path'].length - 4);
    const downloadConfig = await generateArgsForSubscription(sub, user_uid, true, new_path);
    logger.verbose(`Checking if a better version of the fresh upload ${file_obj['id']} exists.`);
    // simulate a download to verify that a better version exists
    youtubedl.getInfo(file_obj['url'], downloadConfig, async (err, output) => {
        if (err) {
            // video is not available anymore for whatever reason
        } else if (output) {
            const metric_to_compare = sub.type === 'audio' ? 'abr' : 'height';
            if (output[metric_to_compare] > file_obj[metric_to_compare]) {
                // download new video as the simulated one is better
                youtubedl.exec(file_obj['url'], downloadConfig, {maxBuffer: Infinity}, async (err, output) => {
                    if (err) {
                        logger.verbose(`Failed to download better version of video ${file_obj['id']}`);
                    } else if (output) {
                        logger.verbose(`Successfully upgraded video ${file_obj['id']}'s ${metric_to_compare} from ${file_obj[metric_to_compare]} to ${output[metric_to_compare]}`);
                        await db_api.setVideoProperty(file_obj['uid'], {[metric_to_compare]: output[metric_to_compare]}, user_uid, sub['id']);
                    }
                });
            } 
        }
    });
    await db_api.setVideoProperty(file_obj['uid'], {'fresh_upload': false}, user_uid, sub['id']);
}

// helper functions

function getAppendedBasePath(sub, base_path) {

    return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}

module.exports = {
    getSubscription        : getSubscription,
    getSubscriptionByName  : getSubscriptionByName,
    getSubscriptions       : getSubscriptions,
    getAllSubscriptions    : getAllSubscriptions,
    updateSubscription     : updateSubscription,
    subscribe              : subscribe,
    unsubscribe            : unsubscribe,
    deleteSubscriptionFile : deleteSubscriptionFile,
    getVideosForSub        : getVideosForSub,
    setLogger              : setLogger,
    initialize             : initialize,
    updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple
}