Merge pull request #216 from NotWoods/await

Use async versions of filesystem methods
locale-based-dates
Tzahi12345 5 years ago committed by GitHub
commit fb72dee26f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

File diff suppressed because it is too large Load Diff

@ -139,12 +139,12 @@ exports.registerUser = function(req, res) {
exports.passport.use(new LocalStrategy({ exports.passport.use(new LocalStrategy({
usernameField: 'username', usernameField: 'username',
passwordField: 'password'}, passwordField: 'password'},
function(username, password, done) { async function(username, password, done) {
const user = users_db.get('users').find({name: username}).value(); const user = users_db.get('users').find({name: username}).value();
if (!user) { logger.error(`User ${username} not found`); return done(null, false); } if (!user) { logger.error(`User ${username} not found`); return done(null, false); }
if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); } if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); }
if (user) { if (user) {
return done(null, bcrypt.compareSync(password, user.passhash) ? user : false); return done(null, (await bcrypt.compare(password, user.passhash)) ? user : false);
} }
} }
)); ));
@ -226,15 +226,13 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
// change password // change password
exports.changeUserPassword = async function(user_uid, new_pass) { exports.changeUserPassword = async function(user_uid, new_pass) {
return new Promise(resolve => { try {
bcrypt.hash(new_pass, saltRounds) const hash = await bcrypt.hash(new_pass, saltRounds);
.then(function(hash) { users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write(); return true;
resolve(true); } catch (err) {
}).catch(err => { return false;
resolve(false); }
});
});
} }
// change user permissions // change user permissions
@ -352,7 +350,7 @@ exports.registerUserFile = function(user_uid, file_object, type) {
.write(); .write();
} }
exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = false) { exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode = false) {
let success = false; let success = false;
const file_obj = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value(); const file_obj = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value();
if (file_obj) { if (file_obj) {
@ -375,20 +373,20 @@ exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = fals
.remove({ .remove({
uid: file_uid uid: file_uid
}).write(); }).write();
if (fs.existsSync(full_path)) { if (await fs.pathExists(full_path)) {
// remove json and file // remove json and file
const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json'); const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json');
const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json'); const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json');
let youtube_id = null; let youtube_id = null;
if (fs.existsSync(json_path)) { if (await fs.pathExists(json_path)) {
youtube_id = fs.readJSONSync(json_path).id; youtube_id = await fs.readJSON(json_path).id;
fs.unlinkSync(json_path); await fs.unlink(json_path);
} else if (fs.existsSync(alternate_json_path)) { } else if (await fs.pathExists(alternate_json_path)) {
youtube_id = fs.readJSONSync(alternate_json_path).id; youtube_id = await fs.readJSON(alternate_json_path).id;
fs.unlinkSync(alternate_json_path); await fs.unlink(alternate_json_path);
} }
fs.unlinkSync(full_path); await fs.unlink(full_path);
// do archive stuff // do archive stuff
@ -397,17 +395,17 @@ exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = fals
const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`); const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`);
// use subscriptions API to remove video from the archive file, and write it to the blacklist // use subscriptions API to remove video from the archive file, and write it to the blacklist
if (fs.existsSync(archive_path)) { if (await fs.pathExists(archive_path)) {
const line = youtube_id ? subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null; const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
if (blacklistMode && line) { if (blacklistMode && line) {
let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`); let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`);
// adds newline to the beginning of the line // adds newline to the beginning of the line
line = '\n' + line; line = '\n' + line;
fs.appendFileSync(blacklistPath, line); await fs.appendFile(blacklistPath, line);
} }
} else { } else {
logger.info(`Could not find archive file for ${type} files. Creating...`); logger.info(`Could not find archive file for ${type} files. Creating...`);
fs.ensureFileSync(archive_path); await fs.ensureFile(archive_path);
} }
} }
} }

@ -182,9 +182,9 @@ async function importUnregisteredFiles() {
} }
// run through check list and check each file to see if it's missing from the db // run through check list and check each file to see if it's missing from the db
dirs_to_check.forEach(dir_to_check => { for (const dir_to_check of dirs_to_check) {
// recursively get all files in dir's path // recursively get all files in dir's path
const files = utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
files.forEach(file => { files.forEach(file => {
// check if file exists in db, if not add it // check if file exists in db, if not add it
@ -195,7 +195,7 @@ async function importUnregisteredFiles() {
logger.verbose(`Added discovered file to the database: ${file.id}`); logger.verbose(`Added discovered file to the database: ${file.id}`);
} }
}); });
}); }
} }

@ -79,17 +79,18 @@ async function getSubscriptionInfo(sub, user_uid = null) {
else else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
return new Promise(resolve => { // get videos
// get videos let downloadConfig = ['--dump-json', '--playlist-end', '1'];
let downloadConfig = ['--dump-json', '--playlist-end', '1']; let useCookies = config_api.getConfigItem('ytdl_use_cookies');
let useCookies = config_api.getConfigItem('ytdl_use_cookies'); if (useCookies) {
if (useCookies) { if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) { downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); } else {
} else { logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
} }
}
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
if (debugMode) { if (debugMode) {
logger.info('Subscribe: got info for subscription ' + sub.id); logger.info('Subscribe: got info for subscription ' + sub.id);
@ -152,39 +153,36 @@ async function getSubscriptionInfo(sub, user_uid = null) {
} }
async function unsubscribe(sub, deleteMode, user_uid = null) { async function unsubscribe(sub, deleteMode, user_uid = null) {
return new Promise(async resolve => { let basePath = null;
let basePath = null; if (user_uid)
if (user_uid) basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); else
else basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); let result_obj = { success: false, error: '' };
let result_obj = { success: false, error: '' };
let id = sub.id; let id = sub.id;
if (user_uid) if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write(); users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write();
else else
db.get('subscriptions').remove({id: id}).write(); db.get('subscriptions').remove({id: id}).write();
// failed subs have no name, on unsubscribe they shouldn't error // failed subs have no name, on unsubscribe they shouldn't error
if (!sub.name) { if (!sub.name) {
return; return;
} }
const appendedBasePath = getAppendedBasePath(sub, basePath); const appendedBasePath = getAppendedBasePath(sub, basePath);
if (deleteMode && fs.existsSync(appendedBasePath)) { if (deleteMode && (await fs.pathExists(appendedBasePath))) {
if (sub.archive && fs.existsSync(sub.archive)) { if (sub.archive && (await fs.pathExists(sub.archive))) {
const archive_file_path = path.join(sub.archive, 'archive.txt'); const archive_file_path = path.join(sub.archive, 'archive.txt');
// deletes archive if it exists // deletes archive if it exists
if (fs.existsSync(archive_file_path)) { if (await fs.pathExists(archive_file_path)) {
fs.unlinkSync(archive_file_path); await fs.unlink(archive_file_path);
}
fs.rmdirSync(sub.archive);
} }
deleteFolderRecursive(appendedBasePath); await fs.rmdir(sub.archive);
} }
}); await fs.remove(appendedBasePath);
}
} }
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) { async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
@ -202,155 +200,154 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
const name = file; const name = file;
let retrievedID = null; let retrievedID = null;
sub_db.get('videos').remove({uid: file_uid}).write(); sub_db.get('videos').remove({uid: file_uid}).write();
return new Promise(resolve => {
let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
var jsonPath = path.join(__dirname,filePath,name+'.info.json');
var videoFilePath = path.join(__dirname,filePath,name+ext);
var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
var altImageFilePath = path.join(__dirname,filePath,name+'.jpg');
jsonExists = fs.existsSync(jsonPath);
videoFileExists = fs.existsSync(videoFilePath);
imageFileExists = fs.existsSync(imageFilePath);
altImageFileExists = fs.existsSync(altImageFilePath);
if (jsonExists) {
retrievedID = JSON.parse(fs.readFileSync(jsonPath, 'utf8'))['id'];
fs.unlinkSync(jsonPath);
}
if (imageFileExists) { let filePath = appendedBasePath;
fs.unlinkSync(imageFilePath); const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
} var jsonPath = path.join(__dirname,filePath,name+'.info.json');
var videoFilePath = path.join(__dirname,filePath,name+ext);
var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
var altImageFilePath = path.join(__dirname,filePath,name+'.jpg');
const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
fs.pathExists(jsonPath),
fs.pathExists(videoFilePath),
fs.pathExists(imageFilePath),
fs.pathExists(altImageFilePath),
]);
if (jsonExists) {
retrievedID = JSON.parse(await fs.readFile(jsonPath, 'utf8'))['id'];
await fs.unlink(jsonPath);
}
if (altImageFileExists) { if (imageFileExists) {
fs.unlinkSync(altImageFilePath); await fs.unlink(imageFilePath);
} }
if (videoFileExists) { if (altImageFileExists) {
fs.unlink(videoFilePath, function(err) { await fs.unlink(altImageFilePath);
if (fs.existsSync(jsonPath) || fs.existsSync(videoFilePath)) { }
resolve(false);
} else { if (videoFileExists) {
// check if the user wants the video to be redownloaded (deleteForever === false) await fs.unlink(videoFilePath);
if (!deleteForever && useArchive && sub.archive && retrievedID) { if ((await fs.pathExists(jsonPath)) || (await fs.pathExists(videoFilePath))) {
const archive_path = path.join(sub.archive, 'archive.txt') return false;
// if archive exists, remove line with video ID
if (fs.existsSync(archive_path)) {
removeIDFromArchive(archive_path, retrievedID);
}
}
resolve(true);
}
});
} else { } else {
// TODO: tell user that the file didn't exist // check if the user wants the video to be redownloaded (deleteForever === false)
resolve(true); if (!deleteForever && useArchive && sub.archive && retrievedID) {
const archive_path = path.join(sub.archive, 'archive.txt')
// if archive exists, remove line with video ID
if (await fs.pathExists(archive_path)) {
await removeIDFromArchive(archive_path, retrievedID);
}
}
return true;
} }
} else {
}); // TODO: tell user that the file didn't exist
return true;
}
} }
async function getVideosForSub(sub, user_uid = null) { async function getVideosForSub(sub, user_uid = null) {
return new Promise(resolve => { if (!subExists(sub.id, user_uid)) {
if (!subExists(sub.id, user_uid)) { return false;
resolve(false); }
return;
}
// get sub_db // get sub_db
let sub_db = null; let sub_db = null;
if (user_uid) if (user_uid)
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
else else
sub_db = db.get('subscriptions').find({id: sub.id}); sub_db = db.get('subscriptions').find({id: sub.id});
// get basePath // get basePath
let basePath = null; let basePath = null;
if (user_uid) if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = null let appendedBasePath = null
appendedBasePath = getAppendedBasePath(sub, basePath); appendedBasePath = getAppendedBasePath(sub, basePath);
let multiUserMode = null; let multiUserMode = null;
if (user_uid) { if (user_uid) {
multiUserMode = { multiUserMode = {
user: user_uid, user: user_uid,
file_path: appendedBasePath file_path: appendedBasePath
}
} }
}
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4' const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`; let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`;
if (sub.custom_output) { if (sub.custom_output) {
fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`; fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
} }
let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json']; let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json'];
let qualityPath = null; let qualityPath = null;
if (sub.type && sub.type === 'audio') { if (sub.type && sub.type === 'audio') {
qualityPath = ['-f', 'bestaudio'] qualityPath = ['-f', 'bestaudio']
qualityPath.push('-x'); qualityPath.push('-x');
qualityPath.push('--audio-format', 'mp3'); qualityPath.push('--audio-format', 'mp3');
} else { } else {
qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4'] qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4']
} }
downloadConfig.push(...qualityPath) downloadConfig.push(...qualityPath)
if (sub.custom_args) { if (sub.custom_args) {
customArgsArray = sub.custom_args.split(',,'); customArgsArray = sub.custom_args.split(',,');
if (customArgsArray.indexOf('-f') !== -1) { if (customArgsArray.indexOf('-f') !== -1) {
// if custom args has a custom quality, replce the original quality with that of custom args // if custom args has a custom quality, replce the original quality with that of custom args
const original_output_index = downloadConfig.indexOf('-f'); const original_output_index = downloadConfig.indexOf('-f');
downloadConfig.splice(original_output_index, 2); downloadConfig.splice(original_output_index, 2);
}
downloadConfig.push(...customArgsArray);
} }
downloadConfig.push(...customArgsArray);
}
let archive_dir = null; let archive_dir = null;
let archive_path = null; let archive_path = null;
if (useArchive) { if (useArchive) {
if (sub.archive) { if (sub.archive) {
archive_dir = sub.archive; archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt') archive_path = path.join(archive_dir, 'archive.txt')
}
downloadConfig.push('--download-archive', archive_path);
} }
downloadConfig.push('--download-archive', archive_path);
}
// if streaming only mode, just get the list of videos // if streaming only mode, just get the list of videos
if (sub.streamingOnly) { if (sub.streamingOnly) {
downloadConfig = ['-f', 'best', '--dump-json']; downloadConfig = ['-f', 'best', '--dump-json'];
} }
if (sub.timerange) { if (sub.timerange) {
downloadConfig.push('--dateafter', sub.timerange); downloadConfig.push('--dateafter', sub.timerange);
} }
let useCookies = config_api.getConfigItem('ytdl_use_cookies'); let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) { if (useCookies) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) { if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt')); downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else { } else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.'); logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
} }
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) { if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail'); downloadConfig.push('--write-thumbnail');
} }
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
// get videos return new Promise(resolve => {
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
logger.verbose('Subscription: finished check for ' + sub.name); logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) { if (err && !output) {
@ -456,23 +453,8 @@ function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
} }
// https://stackoverflow.com/a/32197381/8088021 async function removeIDFromArchive(archive_path, id) {
const deleteFolderRecursive = function(folder_to_delete) { let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (fs.existsSync(folder_to_delete)) {
fs.readdirSync(folder_to_delete).forEach((file, index) => {
const curPath = path.join(folder_to_delete, file);
if (fs.lstatSync(curPath).isDirectory()) { // recurse
deleteFolderRecursive(curPath);
} else { // delete file
fs.unlinkSync(curPath);
}
});
fs.rmdirSync(folder_to_delete);
}
};
function removeIDFromArchive(archive_path, id) {
let data = fs.readFileSync(archive_path, {encoding: 'utf-8'});
if (!data) { if (!data) {
logger.error('Archive could not be found.'); logger.error('Archive could not be found.');
return; return;
@ -493,7 +475,7 @@ function removeIDFromArchive(archive_path, id) {
// UPDATE FILE WITH NEW DATA // UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n'); const updatedData = dataArray.join('\n');
fs.writeFileSync(archive_path, updatedData); await fs.writeFile(archive_path, updatedData);
if (line) return line; if (line) return line;
if (err) throw err; if (err) throw err;
} }

@ -4,6 +4,7 @@ const config_api = require('./config');
const is_windows = process.platform === 'win32'; const is_windows = process.platform === 'win32';
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) { function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path; let fixed_path = unfixed_path;
@ -19,21 +20,21 @@ function getTrueFileName(unfixed_path, type) {
return fixed_path; return fixed_path;
} }
function getDownloadedFilesByType(basePath, type) { async function getDownloadedFilesByType(basePath, type) {
// return empty array if the path doesn't exist // return empty array if the path doesn't exist
if (!fs.existsSync(basePath)) return []; if (!(await fs.pathExists(basePath))) return [];
let files = []; let files = [];
const ext = type === 'audio' ? 'mp3' : 'mp4'; const ext = type === 'audio' ? 'mp3' : 'mp4';
var located_files = recFindByExt(basePath, ext); var located_files = await recFindByExt(basePath, ext);
for (let i = 0; i < located_files.length; i++) { for (let i = 0; i < located_files.length; i++) {
let file = located_files[i]; let file = located_files[i];
var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length); var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
var stats = fs.statSync(file); var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4); var id = file_path.substring(0, file_path.length-4);
var jsonobj = getJSONByType(type, id, basePath); var jsonobj = await getJSONByType(type, id, basePath);
if (!jsonobj) continue; if (!jsonobj) continue;
var title = jsonobj.title; var title = jsonobj.title;
var url = jsonobj.webpage_url; var url = jsonobj.webpage_url;
@ -158,27 +159,25 @@ function deleteJSONFile(name, type, customPath = null) {
} }
function recFindByExt(base,ext,files,result) async function recFindByExt(base,ext,files,result)
{ {
files = files || fs.readdirSync(base) files = files || (await fs.readdir(base))
result = result || [] result = result || []
files.forEach( for (const file of files) {
function (file) { var newbase = path.join(base,file)
var newbase = path.join(base,file) if ( (await fs.stat(newbase)).isDirectory() )
if ( fs.statSync(newbase).isDirectory() ) {
{ result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result) }
} else
else {
if ( file.substr(-1*(ext.length+1)) == '.' + ext )
{ {
if ( file.substr(-1*(ext.length+1)) == '.' + ext ) result.push(newbase)
{
result.push(newbase)
}
} }
} }
) }
return result return result
} }

Loading…
Cancel
Save