From 31f581c6420517d80439807a20260a2dedd4e7cc Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Thu, 30 Apr 2020 04:54:41 -0400 Subject: [PATCH] Subscriptions now support multi-user-mode Fixed bug where playlist subscription downloads would fail due to a mislabeled parameter Components that are routes now make sure auth is finished before sending requests to the backend --- backend/app.js | 72 ++++++++---- backend/authentication/auth.js | 1 + backend/subscriptions.js | 108 +++++++++++++----- src/app/main/main.component.ts | 12 +- src/app/player/player.component.ts | 86 +++++++------- src/app/posts.services.ts | 5 +- .../subscription/subscription.component.ts | 4 +- .../subscriptions/subscriptions.component.ts | 15 ++- 8 files changed, 207 insertions(+), 96 deletions(-) diff --git a/backend/app.js b/backend/app.js index 5cf71a0..aaa4550 100644 --- a/backend/app.js +++ b/backend/app.js @@ -588,7 +588,18 @@ function calculateSubcriptionRetrievalDelay(amount) { } function watchSubscriptions() { - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = null; + + const multiUserMode = config_api.getConfigItem('ytdl_multi_user_mode'); + if (multiUserMode) { + subscriptions = []; + let users = users_db.get('users').value(); + for (let i = 0; i < users.length; i++) { + if (users[i]['subscriptions']) subscriptions = subscriptions.concat(users[i]['subscriptions']); + } + } else { + subscriptions = subscriptions_api.getAllSubscriptions(); + } if (!subscriptions) return; @@ -600,7 +611,7 @@ function watchSubscriptions() { let sub = subscriptions[i]; logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval); setTimeout(() => { - subscriptions_api.getVideosForSub(sub); + subscriptions_api.getVideosForSub(sub, sub.user_uid); }, current_delay); current_delay += delay_interval; if (current_delay >= subscriptionsCheckInterval * 1000) current_delay = 0; @@ -2022,17 +2033,19 @@ app.post('/api/disableSharing', optionalJwt, function(req, res) { }); }); -app.post('/api/subscribe', async (req, res) => { +app.post('/api/subscribe', optionalJwt, async (req, res) => { let name = req.body.name; let url = req.body.url; let timerange = req.body.timerange; let streamingOnly = req.body.streamingOnly; + let user_uid = req.isAuthenticated() ? req.user.uid : null; const new_sub = { name: name, url: url, id: uuid(), - streamingOnly: streamingOnly + streamingOnly: streamingOnly, + user_uid: user_uid }; // adds timerange if it exists, otherwise all videos will be downloaded @@ -2040,7 +2053,7 @@ app.post('/api/subscribe', async (req, res) => { new_sub.timerange = timerange; } - const result_obj = await subscriptions_api.subscribe(new_sub); + const result_obj = await subscriptions_api.subscribe(new_sub, user_uid); if (result_obj.success) { res.send({ @@ -2054,11 +2067,12 @@ app.post('/api/subscribe', async (req, res) => { } }); -app.post('/api/unsubscribe', async (req, res) => { +app.post('/api/unsubscribe', optionalJwt, async (req, res) => { let deleteMode = req.body.deleteMode let sub = req.body.sub; + let user_uid = req.isAuthenticated() ? req.user.uid : null; - let result_obj = subscriptions_api.unsubscribe(sub, deleteMode); + let result_obj = subscriptions_api.unsubscribe(sub, deleteMode, user_uid); if (result_obj.success) { res.send({ success: result_obj.success @@ -2071,12 +2085,13 @@ app.post('/api/unsubscribe', async (req, res) => { } }); -app.post('/api/deleteSubscriptionFile', async (req, res) => { +app.post('/api/deleteSubscriptionFile', optionalJwt, async (req, res) => { let deleteForever = req.body.deleteForever; let file = req.body.file; let sub = req.body.sub; + let user_uid = req.isAuthenticated() ? req.user.uid : null; - let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever); + let success = await subscriptions_api.deleteSubscriptionFile(sub, file, deleteForever, user_uid); if (success) { res.send({ @@ -2088,11 +2103,12 @@ app.post('/api/deleteSubscriptionFile', async (req, res) => { }); -app.post('/api/getSubscription', async (req, res) => { +app.post('/api/getSubscription', optionalJwt, async (req, res) => { let subID = req.body.id; + let user_uid = req.isAuthenticated() ? req.user.uid : null; // get sub from db - let subscription = subscriptions_api.getSubscription(subID); + let subscription = subscriptions_api.getSubscription(subID, user_uid); if (!subscription) { // failed to get subscription from db, send 400 error @@ -2102,7 +2118,12 @@ app.post('/api/getSubscription', async (req, res) => { // get sub videos if (subscription.name && !subscription.streamingOnly) { - let base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); + let base_path = null; + if (user_uid) + base_path = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); + let appended_base_path = path.join(base_path, subscription.isPlaylist ? 'playlists' : 'channels', subscription.name, '/'); let files; try { @@ -2159,18 +2180,22 @@ app.post('/api/getSubscription', async (req, res) => { } }); -app.post('/api/downloadVideosForSubscription', async (req, res) => { +app.post('/api/downloadVideosForSubscription', optionalJwt, async (req, res) => { let subID = req.body.subID; - let sub = subscriptions_api.getSubscription(subID); - subscriptions_api.getVideosForSub(sub); + let user_uid = req.isAuthenticated() ? req.user.uid : null; + + let sub = subscriptions_api.getSubscription(subID, user_uid); + subscriptions_api.getVideosForSub(sub, user_uid); res.send({ success: true }); }); -app.post('/api/getAllSubscriptions', async (req, res) => { +app.post('/api/getAllSubscriptions', optionalJwt, async (req, res) => { + let user_uid = req.isAuthenticated() ? req.user.uid : null; + // get subs from api - let subscriptions = subscriptions_api.getAllSubscriptions(); + let subscriptions = subscriptions_api.getAllSubscriptions(user_uid); res.send({ subscriptions: subscriptions @@ -2360,7 +2385,7 @@ app.post('/api/downloadFile', optionalJwt, async (req, res) => { let outputName = req.body.outputName; let fullPathProvided = req.body.fullPathProvided; let subscriptionName = req.body.subscriptionName; - let subscriptionPlaylist = req.body.subscriptionPlaylist; + let subscriptionPlaylist = req.body.subPlaylist; let file = null; if (!zip_mode) { fileNames = decodeURIComponent(fileNames); @@ -2369,14 +2394,19 @@ app.post('/api/downloadFile', optionalJwt, async (req, res) => { const ext = is_audio ? '.mp3' : '.mp4'; let base_path = fileFolderPath; + let usersFileFolder = null; if (req.isAuthenticated()) { - const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); base_path = path.join(usersFileFolder, req.user.uid, type); } if (!subscriptionName) { file = path.join(__dirname, base_path, fileNames + ext); } else { - let basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + let basePath = null; + if (usersFileFolder) + basePath = path.join(usersFileFolder, req.user.uid, 'subscriptions'); + else + basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); file = path.join(__dirname, basePath, (subscriptionPlaylist ? 'playlists' : 'channels'), subscriptionName, fileNames + '.mp4') } } else { @@ -2509,7 +2539,7 @@ app.get('/api/video/:id', optionalJwt, function(req , res){ let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); if (optionalParams['subName']) { const isPlaylist = optionalParams['subPlaylist']; - file_path = path.join(usersFileFolder, req.user.uid, (isPlaylist === 'true' ? 'playlists/' : 'channels/'), id + '.mp4') + file_path = path.join(usersFileFolder, req.user.uid, 'subscriptions', (isPlaylist === 'true' ? 'playlists/' : 'channels/'),optionalParams['subName'], id + '.mp4') } else { file_path = path.join(usersFileFolder, req.user.uid, 'video', id + '.mp4'); } diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index 701b47f..5b5d0a6 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -95,6 +95,7 @@ exports.registerUser = function(req, res) { audio: [], video: [] }, + subscriptions: [], created: Date.now() }; // check if user exists diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 2ae810f..cccd224 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -20,7 +20,7 @@ function initialize(input_db, input_users_db, input_logger) { setLogger(input_logger); } -async function subscribe(sub) { +async function subscribe(sub, user_uid = null) { const result_obj = { success: false, error: '' @@ -29,7 +29,14 @@ async function subscribe(sub) { // sub should just have url and name. here we will get isPlaylist and path sub.isPlaylist = sub.url.includes('playlist'); - if (db.get('subscriptions').find({url: sub.url}).value()) { + let url_exists = false; + + if (user_uid) + url_exists = !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({url: sub.url}).value() + else + url_exists = !!db.get('subscriptions').find({url: sub.url}).value(); + + if (url_exists) { logger.info('Sub already exists'); result_obj.error = 'Subcription with URL ' + sub.url + ' already exists!'; resolve(result_obj); @@ -37,19 +44,27 @@ async function subscribe(sub) { } // add sub to db - db.get('subscriptions').push(sub).write(); + if (user_uid) + users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write(); + else + db.get('subscriptions').push(sub).write(); let success = await getSubscriptionInfo(sub); result_obj.success = success; result_obj.sub = sub; - getVideosForSub(sub); + getVideosForSub(sub, user_uid); resolve(result_obj); }); } -async function getSubscriptionInfo(sub) { - const basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); +async function getSubscriptionInfo(sub, user_uid = null) { + let basePath = null; + if (user_uid) + basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + return new Promise(resolve => { // get videos let downloadConfig = ['--dump-json', '--playlist-end', '1'] @@ -75,16 +90,19 @@ async function getSubscriptionInfo(sub) { if (!output_json) { continue; } - if (!sub.name) { sub.name = sub.isPlaylist ? output_json.playlist_title : output_json.uploader; // if it's now valid, update if (sub.name) { - db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write(); + if (user_uid) + users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write(); + else + db.get('subscriptions').find({id: sub.id}).assign({name: sub.name}).write(); } } - if (!sub.archive) { + const useArchive = config_api.getConfigItem('ytdl_subscriptions_use_youtubedl_archive'); + if (useArchive && !sub.archive) { // must create the archive const archive_dir = path.join(__dirname, basePath, 'archives', sub.name); const archive_path = path.join(archive_dir, 'archive.txt'); @@ -95,7 +113,10 @@ async function getSubscriptionInfo(sub) { // updates subscription sub.archive = archive_dir; - db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write(); + if (user_uid) + users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write(); + else + db.get('subscriptions').find({id: sub.id}).assign({archive: archive_dir}).write(); } // TODO: get even more info @@ -108,13 +129,20 @@ async function getSubscriptionInfo(sub) { }); } -async function unsubscribe(sub, deleteMode) { +async function unsubscribe(sub, deleteMode, user_uid = null) { return new Promise(async resolve => { - const basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + let basePath = null; + if (user_uid) + basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); let result_obj = { success: false, error: '' }; let id = sub.id; - db.get('subscriptions').remove({id: id}).write(); + if (user_uid) + users_db.get('users').find({uid: user_uid}).get('subscriptions').remove({id: id}).write(); + else + db.get('subscriptions').remove({id: id}).write(); const appendedBasePath = getAppendedBasePath(sub, basePath); if (deleteMode && fs.existsSync(appendedBasePath)) { @@ -132,8 +160,12 @@ async function unsubscribe(sub, deleteMode) { } -async function deleteSubscriptionFile(sub, file, deleteForever) { - const basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); +async function deleteSubscriptionFile(sub, file, deleteForever, user_uid = null) { + let basePath = null; + if (user_uid) + basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); const useArchive = config_api.getConfigItem('ytdl_subscriptions_use_youtubedl_archive'); const appendedBasePath = getAppendedBasePath(sub, basePath); const name = file; @@ -181,14 +213,27 @@ async function deleteSubscriptionFile(sub, file, deleteForever) { }); } -async function getVideosForSub(sub) { +async function getVideosForSub(sub, user_uid = null) { return new Promise(resolve => { - if (!subExists(sub.id)) { + if (!subExists(sub.id, user_uid)) { resolve(false); return; } - const sub_db = db.get('subscriptions').find({id: sub.id}); - const basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + + // get sub_db + let sub_db = null; + if (user_uid) + sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); + else + sub_db = db.get('subscriptions').find({id: sub.id}); + + // get basePath + let basePath = null; + if (user_uid) + basePath = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); + const useArchive = config_api.getConfigItem('ytdl_subscriptions_use_youtubedl_archive'); let appendedBasePath = null @@ -263,23 +308,32 @@ async function getVideosForSub(sub) { }); } -function getAllSubscriptions() { - const subscriptions = db.get('subscriptions').value(); - return subscriptions; +function getAllSubscriptions(user_uid = null) { + if (user_uid) + return users_db.get('users').find({uid: user_uid}).get('subscriptions').value(); + else + return db.get('subscriptions').value(); } -function getSubscription(subID) { - return db.get('subscriptions').find({id: subID}).value(); +function getSubscription(subID, user_uid = null) { + if (user_uid) + return users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); + else + return db.get('subscriptions').find({id: subID}).value(); } -function subExists(subID) { - return !!db.get('subscriptions').find({id: subID}).value(); +function subExists(subID, user_uid = null) { + if (user_uid) + return !!users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: subID}).value(); + else + return !!db.get('subscriptions').find({id: subID}).value(); } // helper functions function getAppendedBasePath(sub, base_path) { - return base_path + (sub.isPlaylist ? 'playlists/' : 'channels/') + sub.name; + + return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); } // https://stackoverflow.com/a/32197381/8088021 diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index bc365d0..0645c9b 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -196,6 +196,7 @@ export class MainComponent implements OnInit { selectedQuality = ''; formats_loading = false; + config_loaded = false; @ViewChild('urlinput', { read: ElementRef }) urlInput: ElementRef; @ViewChildren('audiofilecard') audioFileCards: QueryList; @@ -294,7 +295,16 @@ export class MainComponent implements OnInit { // app initialization. ngOnInit() { - this.configLoad(); + if (this.postsService.config) { + this.configLoad(); + } else { + this.postsService.config_reloaded.subscribe(changed => { + if (changed && !this.config_loaded) { + this.config_loaded = true; + this.configLoad(); + } + }); + } this.postsService.config_reloaded.subscribe(changed => { if (changed) { diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 06d3f53..0377063 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -79,45 +79,15 @@ export class PlayerComponent implements OnInit { this.uuid = this.route.snapshot.paramMap.get('uuid'); // loading config - this.postsService.loadNavItems().subscribe(res => { // loads settings - const result = !this.postsService.debugMode ? res['config_file'] : res; - this.baseStreamPath = this.postsService.path; - this.audioFolderPath = result['YoutubeDLMaterial']['Downloader']['path-audio']; - this.videoFolderPath = result['YoutubeDLMaterial']['Downloader']['path-video']; - this.subscriptionFolderPath = result['YoutubeDLMaterial']['Subscriptions']['subscriptions_base_path']; - this.fileNames = this.route.snapshot.paramMap.get('fileNames') ? this.route.snapshot.paramMap.get('fileNames').split('|nvr|') : null; - - if (!this.fileNames) { - this.is_shared = true; - } - - if (this.uid && !this.id) { - this.getFile(); - } else if (this.id) { - this.getPlaylistFiles(); - } - - if (this.url) { - // if a url is given, just stream the URL - this.playlist = []; - const imedia: IMedia = { - title: this.name, - label: this.name, - src: this.url, - type: 'video/mp4' + if (this.postsService.config) { + this.processConfig(); + } else { + this.postsService.config_reloaded.subscribe(changed => { // loads settings + if (changed) { + this.processConfig(); } - this.playlist.push(imedia); - this.currentItem = this.playlist[0]; - this.currentIndex = 0; - this.show_player = true; - } else if (this.type === 'subscription' || this.fileNames) { - this.show_player = true; - this.parseFileNames(); - } - }); - - // this.getFileInfos(); - + }); + } } constructor(private postsService: PostsService, private route: ActivatedRoute, private dialog: MatDialog, private router: Router, @@ -125,6 +95,42 @@ export class PlayerComponent implements OnInit { } + processConfig() { + this.baseStreamPath = this.postsService.path; + this.audioFolderPath = this.postsService.config['Downloader']['path-audio']; + this.videoFolderPath = this.postsService.config['Downloader']['path-video']; + this.subscriptionFolderPath = this.postsService.config['Subscriptions']['subscriptions_base_path']; + this.fileNames = this.route.snapshot.paramMap.get('fileNames') ? this.route.snapshot.paramMap.get('fileNames').split('|nvr|') : null; + + if (!this.fileNames) { + this.is_shared = true; + } + + if (this.uid && !this.id) { + this.getFile(); + } else if (this.id) { + this.getPlaylistFiles(); + } + + if (this.url) { + // if a url is given, just stream the URL + this.playlist = []; + const imedia: IMedia = { + title: this.name, + label: this.name, + src: this.url, + type: 'video/mp4' + } + this.playlist.push(imedia); + this.currentItem = this.playlist[0]; + this.currentIndex = 0; + this.show_player = true; + } else if (this.type === 'subscription' || this.fileNames) { + this.show_player = true; + this.parseFileNames(); + } + } + getFile() { const already_has_filenames = !!this.fileNames; this.postsService.getFile(this.uid, null, this.uuid).subscribe(res => { @@ -191,10 +197,10 @@ export class PlayerComponent implements OnInit { // adds user token if in multi-user-mode if (this.postsService.isLoggedIn) { - fullLocation += `?jwt=${this.postsService.token}`; + fullLocation += (this.subscriptionName ? '&' : '?') + `jwt=${this.postsService.token}`; if (this.is_shared) { fullLocation += `&uuid=${this.uuid}&uid=${this.db_file.uid}&type=${this.db_file.type}`; } } else if (this.is_shared) { - fullLocation += `?uuid=${this.uuid}&uid=${this.db_file.uid}&type=${this.db_file.type}`; + fullLocation += (this.subscriptionName ? '&' : '?') + `uuid=${this.uuid}&uid=${this.db_file.uid}&type=${this.db_file.type}`; } // if it has a slash (meaning it's in a directory), only get the file name for the label let label = null; diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 98c16ba..8fb2a2f 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -74,6 +74,8 @@ export class PostsService implements CanActivate { }; this.jwtAuth(); } + } else { + this.config_reloaded.next(true); } } }); @@ -107,7 +109,7 @@ export class PostsService implements CanActivate { const result = !this.debugMode ? res['config_file'] : res; if (result) { this.config = result['YoutubeDLMaterial']; - this.config_reloaded = true; + this.config_reloaded.next(true); } }); } @@ -348,6 +350,7 @@ export class PostsService implements CanActivate { call.subscribe(res => { if (res['token']) { this.afterLogin(res['user'], res['token']); + this.config_reloaded.next(true); } }, err => { if (err.status === 401) { diff --git a/src/app/subscription/subscription/subscription.component.ts b/src/app/subscription/subscription/subscription.component.ts index fce7761..cd7037c 100644 --- a/src/app/subscription/subscription/subscription.component.ts +++ b/src/app/subscription/subscription/subscription.component.ts @@ -49,10 +49,10 @@ export class SubscriptionComponent implements OnInit { if (this.route.snapshot.paramMap.get('id')) { this.id = this.route.snapshot.paramMap.get('id'); - this.getSubscription(); this.postsService.config_reloaded.subscribe(changed => { if (changed) { this.getConfig(); + this.getSubscription(); } }); } @@ -93,7 +93,7 @@ export class SubscriptionComponent implements OnInit { this.router.navigate(['/player', {name: name, url: url}]); } else { this.router.navigate(['/player', {fileNames: name, type: 'subscription', subscriptionName: this.subscription.name, - subPlaylist: this.subscription.isPlaylist}]); + subPlaylist: this.subscription.isPlaylist, uuid: this.postsService.user ? this.postsService.user.uid : null}]); } } diff --git a/src/app/subscriptions/subscriptions.component.ts b/src/app/subscriptions/subscriptions.component.ts index 9776047..01ab0ea 100644 --- a/src/app/subscriptions/subscriptions.component.ts +++ b/src/app/subscriptions/subscriptions.component.ts @@ -22,16 +22,23 @@ export class SubscriptionsComponent implements OnInit { constructor(private dialog: MatDialog, public postsService: PostsService, private router: Router, private snackBar: MatSnackBar) { } ngOnInit() { - this.getSubscriptions(); + if (this.postsService.config) { + this.getSubscriptions(); + } + this.postsService.config_reloaded.subscribe(changed => { + if (changed) { + this.getSubscriptions(); + } + }); } getSubscriptions() { this.subscriptions_loading = true; this.subscriptions = null; - this.channel_subscriptions = []; - this.playlist_subscriptions = []; this.postsService.getAllSubscriptions().subscribe(res => { - this.subscriptions_loading = false; + this.channel_subscriptions = []; + this.playlist_subscriptions = []; + this.subscriptions_loading = false; this.subscriptions = res['subscriptions']; if (!this.subscriptions) { // set it to an empty array so it can notify the user there are no subscriptions