aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #216 from NotWoods/await

Browse Source 
Use async versions of filesystem methods
locale-based-dates
Tzahi12345 5 years ago committed by GitHub
parent b8280e8646 3e4e7edd90
commit fb72dee26f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 660 additions and 729 deletions
Show Stats Download Patch File Download Diff File

960
backend/app.js
View File

File diff suppressed because it is too large Load Diff

50
backend/authentication/auth.js
Unescape Escape View File

@ -139,12 +139,12 @@ exports.registerUser = function(req, res) {
exports.passport.use(new LocalStrategy({
usernameField: 'username',
passwordField: 'password'},
function(username, password, done) {
async function(username, password, done) {
const user = users_db.get('users').find({name: username}).value();
if (!user) { logger.error(`User ${username} not found`); return done(null, false); }
if (user.auth_method && user.auth_method !== 'internal') { return done(null, false); }
if (user) {
return done(null, bcrypt.compareSync(password, user.passhash) ? user : false);
return done(null, (await bcrypt.compare(password, user.passhash)) ? user : false);
}
}
));
@ -160,7 +160,7 @@ exports.passport.use(new LdapStrategy(getLDAPConfiguration,
// check if ldap auth is enabled
const ldap_enabled = config_api.getConfigItem('ytdl_auth_method') === 'ldap';
if (!ldap_enabled) return done(null, false);
const user_uid = user.uid;
let db_user = users_db.get('users').find({uid: user_uid}).value();
if (!db_user) {
@ -226,15 +226,13 @@ exports.ensureAuthenticatedElseError = function(req, res, next) {
// change password
exports.changeUserPassword = async function(user_uid, new_pass) {
return new Promise(resolve => {
bcrypt.hash(new_pass, saltRounds)
.then(function(hash) {
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
resolve(true);
}).catch(err => {
resolve(false);
});
});
try {
const hash = await bcrypt.hash(new_pass, saltRounds);
users_db.get('users').find({uid: user_uid}).assign({passhash: hash}).write();
return true;
} catch (err) {
return false;
}
}
// change user permissions
@ -352,7 +350,7 @@ exports.registerUserFile = function(user_uid, file_object, type) {
.write();
}
exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = false) {
exports.deleteUserFile = async function(user_uid, file_uid, type, blacklistMode = false) {
let success = false;
const file_obj = users_db.get('users').find({uid: user_uid}).get(`files.${type}`).find({uid: file_uid}).value();
if (file_obj) {
@ -375,20 +373,20 @@ exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = fals
.remove({
uid: file_uid
}).write();
if (fs.existsSync(full_path)) {
if (await fs.pathExists(full_path)) {
// remove json and file
const json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + '.info.json');
const alternate_json_path = path.join(usersFileFolder, user_uid, type, file_obj.id + ext + '.info.json');
let youtube_id = null;
if (fs.existsSync(json_path)) {
youtube_id = fs.readJSONSync(json_path).id;
fs.unlinkSync(json_path);
} else if (fs.existsSync(alternate_json_path)) {
youtube_id = fs.readJSONSync(alternate_json_path).id;
fs.unlinkSync(alternate_json_path);
if (await fs.pathExists(json_path)) {
youtube_id = await fs.readJSON(json_path).id;
await fs.unlink(json_path);
} else if (await fs.pathExists(alternate_json_path)) {
youtube_id = await fs.readJSON(alternate_json_path).id;
await fs.unlink(alternate_json_path);
}
fs.unlinkSync(full_path);
await fs.unlink(full_path);
// do archive stuff
@ -397,17 +395,17 @@ exports.deleteUserFile = function(user_uid, file_uid, type, blacklistMode = fals
const archive_path = path.join(usersFileFolder, user_uid, 'archives', `archive_${type}.txt`);
// use subscriptions API to remove video from the archive file, and write it to the blacklist
if (fs.existsSync(archive_path)) {
const line = youtube_id ? subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
if (await fs.pathExists(archive_path)) {
const line = youtube_id ? await subscriptions_api.removeIDFromArchive(archive_path, youtube_id) : null;
if (blacklistMode && line) {
let blacklistPath = path.join(usersFileFolder, user_uid, 'archives', `blacklist_${type}.txt`);
// adds newline to the beginning of the line
line = '\n' + line;
fs.appendFileSync(blacklistPath, line);
await fs.appendFile(blacklistPath, line);
}
} else {
logger.info(`Could not find archive file for ${type} files. Creating...`);
fs.ensureFileSync(archive_path);
await fs.ensureFile(archive_path);
}
}
}
@ -533,7 +531,7 @@ function generateUserObject(userid, username, hash, auth_method = 'internal') {
role: userid === 'admin' && auth_method === 'internal' ? 'admin' : 'user',
permissions: [],
permission_overrides: [],
auth_method: auth_method
auth_method: auth_method
};
return new_user;
}

6
backend/db.js
Unescape Escape View File

@ -182,9 +182,9 @@ async function importUnregisteredFiles() {
}
// run through check list and check each file to see if it's missing from the db
dirs_to_check.forEach(dir_to_check => {
for (const dir_to_check of dirs_to_check) {
// recursively get all files in dir's path
const files = utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
files.forEach(file => {
// check if file exists in db, if not add it
@ -195,7 +195,7 @@ async function importUnregisteredFiles() {
logger.verbose(`Added discovered file to the database: ${file.id}`);
}
});
});
}
}

332
backend/subscriptions.js
Unescape Escape View File

@ -79,17 +79,18 @@ async function getSubscriptionInfo(sub, user_uid = null) {
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
return new Promise(resolve => {
// get videos
let downloadConfig = ['--dump-json', '--playlist-end', '1'];
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
// get videos
let downloadConfig = ['--dump-json', '--playlist-end', '1'];
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) {
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
}
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
if (debugMode) {
logger.info('Subscribe: got info for subscription ' + sub.id);
@ -152,39 +153,36 @@ async function getSubscriptionInfo(sub, user_uid = null) {
}
async function unsubscribe(sub, deleteMode, user_uid = null) {
return new Promise(async resolve => {
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let result_obj = { success: false, error: '' };
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
let result_obj = { success: false, error: '' };
let id = sub.id;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write();
else
db.get('subscriptions').remove({id: id}).write();
let id = sub.id;
if (user_uid)
users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write();
else
db.get('subscriptions').remove({id: id}).write();
// failed subs have no name, on unsubscribe they shouldn't error
if (!sub.name) {
return;
}
// failed subs have no name, on unsubscribe they shouldn't error
if (!sub.name) {
return;
}
const appendedBasePath = getAppendedBasePath(sub, basePath);
if (deleteMode && fs.existsSync(appendedBasePath)) {
if (sub.archive && fs.existsSync(sub.archive)) {
const archive_file_path = path.join(sub.archive, 'archive.txt');
// deletes archive if it exists
if (fs.existsSync(archive_file_path)) {
fs.unlinkSync(archive_file_path);
}
fs.rmdirSync(sub.archive);
const appendedBasePath = getAppendedBasePath(sub, basePath);
if (deleteMode && (await fs.pathExists(appendedBasePath))) {
if (sub.archive && (await fs.pathExists(sub.archive))) {
const archive_file_path = path.join(sub.archive, 'archive.txt');
// deletes archive if it exists
if (await fs.pathExists(archive_file_path)) {
await fs.unlink(archive_file_path);
}
deleteFolderRecursive(appendedBasePath);
await fs.rmdir(sub.archive);
}
});
await fs.remove(appendedBasePath);
}
}
async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null, user_uid = null) {
@ -202,155 +200,154 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
const name = file;
let retrievedID = null;
sub_db.get('videos').remove({uid: file_uid}).write();
return new Promise(resolve => {
let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
var jsonPath = path.join(__dirname,filePath,name+'.info.json');
var videoFilePath = path.join(__dirname,filePath,name+ext);
var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
var altImageFilePath = path.join(__dirname,filePath,name+'.jpg');
jsonExists = fs.existsSync(jsonPath);
videoFileExists = fs.existsSync(videoFilePath);
imageFileExists = fs.existsSync(imageFilePath);
altImageFileExists = fs.existsSync(altImageFilePath);
if (jsonExists) {
retrievedID = JSON.parse(fs.readFileSync(jsonPath, 'utf8'))['id'];
fs.unlinkSync(jsonPath);
}
if (imageFileExists) {
fs.unlinkSync(imageFilePath);
}
let filePath = appendedBasePath;
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
var jsonPath = path.join(__dirname,filePath,name+'.info.json');
var videoFilePath = path.join(__dirname,filePath,name+ext);
var imageFilePath = path.join(__dirname,filePath,name+'.jpg');
var altImageFilePath = path.join(__dirname,filePath,name+'.jpg');
const [jsonExists, videoFileExists, imageFileExists, altImageFileExists] = await Promise.all([
fs.pathExists(jsonPath),
fs.pathExists(videoFilePath),
fs.pathExists(imageFilePath),
fs.pathExists(altImageFilePath),
]);
if (jsonExists) {
retrievedID = JSON.parse(await fs.readFile(jsonPath, 'utf8'))['id'];
await fs.unlink(jsonPath);
}
if (altImageFileExists) {
fs.unlinkSync(altImageFilePath);
}
if (imageFileExists) {
await fs.unlink(imageFilePath);
}
if (videoFileExists) {
fs.unlink(videoFilePath, function(err) {
if (fs.existsSync(jsonPath) || fs.existsSync(videoFilePath)) {
resolve(false);
} else {
// check if the user wants the video to be redownloaded (deleteForever === false)
if (!deleteForever && useArchive && sub.archive && retrievedID) {
const archive_path = path.join(sub.archive, 'archive.txt')
// if archive exists, remove line with video ID
if (fs.existsSync(archive_path)) {
removeIDFromArchive(archive_path, retrievedID);
}
}
resolve(true);
}
});
if (altImageFileExists) {
await fs.unlink(altImageFilePath);
}
if (videoFileExists) {
await fs.unlink(videoFilePath);
if ((await fs.pathExists(jsonPath)) || (await fs.pathExists(videoFilePath))) {
return false;
} else {
// TODO: tell user that the file didn't exist
resolve(true);
// check if the user wants the video to be redownloaded (deleteForever === false)
if (!deleteForever && useArchive && sub.archive && retrievedID) {
const archive_path = path.join(sub.archive, 'archive.txt')
// if archive exists, remove line with video ID
if (await fs.pathExists(archive_path)) {
await removeIDFromArchive(archive_path, retrievedID);
}
}
return true;
}
});
} else {
// TODO: tell user that the file didn't exist
return true;
}
}
async function getVideosForSub(sub, user_uid = null) {
return new Promise(resolve => {
if (!subExists(sub.id, user_uid)) {
resolve(false);
return;
}
if (!subExists(sub.id, user_uid)) {
return false;
}
// get sub_db
let sub_db = null;
if (user_uid)
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
else
sub_db = db.get('subscriptions').find({id: sub.id});
// get sub_db
let sub_db = null;
if (user_uid)
sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id});
else
sub_db = db.get('subscriptions').find({id: sub.id});
// get basePath
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
// get basePath
let basePath = null;
if (user_uid)
basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = null
appendedBasePath = getAppendedBasePath(sub, basePath);
let appendedBasePath = null
appendedBasePath = getAppendedBasePath(sub, basePath);
let multiUserMode = null;
if (user_uid) {
multiUserMode = {
user: user_uid,
file_path: appendedBasePath
}
let multiUserMode = null;
if (user_uid) {
multiUserMode = {
user: user_uid,
file_path: appendedBasePath
}
}
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
const ext = (sub.type && sub.type === 'audio') ? '.mp3' : '.mp4'
let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`;
if (sub.custom_output) {
fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
}
let fullOutput = `${appendedBasePath}/%(title)s.%(ext)s`;
if (sub.custom_output) {
fullOutput = `${appendedBasePath}/${sub.custom_output}.%(ext)s`;
}
let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json'];
let downloadConfig = ['-o', fullOutput, '-ciw', '--write-info-json', '--print-json'];
let qualityPath = null;
if (sub.type && sub.type === 'audio') {
qualityPath = ['-f', 'bestaudio']
qualityPath.push('-x');
qualityPath.push('--audio-format', 'mp3');
} else {
qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4']
}
let qualityPath = null;
if (sub.type && sub.type === 'audio') {
qualityPath = ['-f', 'bestaudio']
qualityPath.push('-x');
qualityPath.push('--audio-format', 'mp3');
} else {
qualityPath = ['-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4']
}
downloadConfig.push(...qualityPath)
downloadConfig.push(...qualityPath)
if (sub.custom_args) {
customArgsArray = sub.custom_args.split(',,');
if (customArgsArray.indexOf('-f') !== -1) {
// if custom args has a custom quality, replce the original quality with that of custom args
const original_output_index = downloadConfig.indexOf('-f');
downloadConfig.splice(original_output_index, 2);
}
downloadConfig.push(...customArgsArray);
if (sub.custom_args) {
customArgsArray = sub.custom_args.split(',,');
if (customArgsArray.indexOf('-f') !== -1) {
// if custom args has a custom quality, replce the original quality with that of custom args
const original_output_index = downloadConfig.indexOf('-f');
downloadConfig.splice(original_output_index, 2);
}
downloadConfig.push(...customArgsArray);
}
let archive_dir = null;
let archive_path = null;
let archive_dir = null;
let archive_path = null;
if (useArchive) {
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
}
downloadConfig.push('--download-archive', archive_path);
if (useArchive) {
if (sub.archive) {
archive_dir = sub.archive;
archive_path = path.join(archive_dir, 'archive.txt')
}
downloadConfig.push('--download-archive', archive_path);
}
// if streaming only mode, just get the list of videos
if (sub.streamingOnly) {
downloadConfig = ['-f', 'best', '--dump-json'];
}
// if streaming only mode, just get the list of videos
if (sub.streamingOnly) {
downloadConfig = ['-f', 'best', '--dump-json'];
}
if (sub.timerange) {
downloadConfig.push('--dateafter', sub.timerange);
}
if (sub.timerange) {
downloadConfig.push('--dateafter', sub.timerange);
}
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) {
if (fs.existsSync(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
if (useCookies) {
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
// get videos
logger.verbose('Subscription: getting videos for subscription ' + sub.name);
return new Promise(resolve => {
youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {
logger.verbose('Subscription: finished check for ' + sub.name);
if (err && !output) {
@ -456,23 +453,8 @@ function getAppendedBasePath(sub, base_path) {
return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name);
}
// https://stackoverflow.com/a/32197381/8088021
const deleteFolderRecursive = function(folder_to_delete) {
if (fs.existsSync(folder_to_delete)) {
fs.readdirSync(folder_to_delete).forEach((file, index) => {
const curPath = path.join(folder_to_delete, file);
if (fs.lstatSync(curPath).isDirectory()) { // recurse
deleteFolderRecursive(curPath);
} else { // delete file
fs.unlinkSync(curPath);
}
});
fs.rmdirSync(folder_to_delete);
}
};
function removeIDFromArchive(archive_path, id) {
let data = fs.readFileSync(archive_path, {encoding: 'utf-8'});
async function removeIDFromArchive(archive_path, id) {
let data = await fs.readFile(archive_path, {encoding: 'utf-8'});
if (!data) {
logger.error('Archive could not be found.');
return;
@ -493,7 +475,7 @@ function removeIDFromArchive(archive_path, id) {
// UPDATE FILE WITH NEW DATA
const updatedData = dataArray.join('\n');
fs.writeFileSync(archive_path, updatedData);
await fs.writeFile(archive_path, updatedData);
if (line) return line;
if (err) throw err;
}

41
backend/utils.js
Unescape Escape View File

@ -4,6 +4,7 @@ const config_api = require('./config');
const is_windows = process.platform === 'win32';
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path;
@ -19,21 +20,21 @@ function getTrueFileName(unfixed_path, type) {
return fixed_path;
}
function getDownloadedFilesByType(basePath, type) {
async function getDownloadedFilesByType(basePath, type) {
// return empty array if the path doesn't exist
if (!fs.existsSync(basePath)) return [];
if (!(await fs.pathExists(basePath))) return [];
let files = [];
const ext = type === 'audio' ? 'mp3' : 'mp4';
var located_files = recFindByExt(basePath, ext);
var located_files = await recFindByExt(basePath, ext);
for (let i = 0; i < located_files.length; i++) {
let file = located_files[i];
var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
var stats = fs.statSync(file);
var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = getJSONByType(type, id, basePath);
var jsonobj = await getJSONByType(type, id, basePath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
@ -129,7 +130,7 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
const files_to_fix = [
// JSONs
path.join(customPath, name + '.info.json'),
@ -158,27 +159,25 @@ function deleteJSONFile(name, type, customPath = null) {
}
function recFindByExt(base,ext,files,result)
async function recFindByExt(base,ext,files,result)
{
files = files || fs.readdirSync(base)
files = files || (await fs.readdir(base))
result = result || []
files.forEach(
function (file) {
var newbase = path.join(base,file)
if ( fs.statSync(newbase).isDirectory() )
{
result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result)
}
else
for (const file of files) {
var newbase = path.join(base,file)
if ( (await fs.stat(newbase)).isDirectory() )
{
result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
}
else
{
if ( file.substr(-1*(ext.length+1)) == '.' + ext )
{
if ( file.substr(-1*(ext.length+1)) == '.' + ext )
{
result.push(newbase)
}
result.push(newbase)
}
}
)
}
return result
}

Write Preview
Loading…
Cancel
Save