From ebfa49240c96129afe5fb45fabd5b82902bd9de6 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 10 Aug 2021 21:32:13 -0600 Subject: [PATCH] Added methods to modify download state Added missing optionalJwt calls in several routes --- backend/app.js | 57 +++++++++--- backend/downloader.js | 91 +++++++++++++++++-- src/app/app-routing.module.ts | 2 +- .../downloads/downloads.component.html | 23 +++-- .../downloads/downloads.component.scss | 6 ++ .../downloads/downloads.component.ts | 87 +++++++++++++++--- src/app/posts.services.ts | 30 ++++-- 7 files changed, 245 insertions(+), 51 deletions(-) diff --git a/backend/app.js b/backend/app.js index 13513b6..02daddf 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1608,7 +1608,7 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => { }); }); -app.post('/api/downloadArchive', async (req, res) => { +app.post('/api/downloadArchive', optionalJwt, async (req, res) => { let sub = req.body.sub; let archive_dir = sub.archive; @@ -1643,7 +1643,7 @@ app.post('/api/uploadCookies', upload_multer.single('cookies'), async (req, res) // Updater API calls -app.get('/api/updaterStatus', async (req, res) => { +app.get('/api/updaterStatus', optionalJwt, async (req, res) => { let status = updaterStatus; if (status) { @@ -1654,7 +1654,7 @@ app.get('/api/updaterStatus', async (req, res) => { }); -app.post('/api/updateServer', async (req, res) => { +app.post('/api/updateServer', optionalJwt, async (req, res) => { let tag = req.body.tag; updateServer(tag); @@ -1667,7 +1667,7 @@ app.post('/api/updateServer', async (req, res) => { // API Key API calls -app.post('/api/generateNewAPIKey', function (req, res) { +app.post('/api/generateNewAPIKey', optionalJwt, function (req, res) { const new_api_key = uuid(); config_api.setConfigItem('ytdl_api_key', new_api_key); res.send({new_api_key: new_api_key}); @@ -1739,12 +1739,12 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => { // Downloads management - app.get('/api/downloads', async (req, res) => { +app.get('/api/downloads', optionalJwt, async (req, res) => { const downloads = await db_api.getRecords('download_queue'); res.send({downloads: downloads}); - }); +}); - app.post('/api/download', async (req, res) => { +app.post('/api/download', optionalJwt, async (req, res) => { const download_uid = req.body.download_uid; const download = await db_api.getRecord('download_queue', {uid: download_uid}); @@ -1754,15 +1754,46 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => { } else { res.send({download: null}); } - }); +}); + +app.post('/api/clearFinishedDownloads', optionalJwt, async (req, res) => { + const success = db_api.removeAllRecords('download_queue', {finished: true}); + res.send({success: success}); +}); + +app.post('/api/clearDownload', optionalJwt, async (req, res) => { + const download_uid = req.body.download_uid; + const success = await downloader_api.clearDownload(download_uid); + res.send({success: success}); +}); - app.post('/api/clearFinishedDownloads', async (req, res) => { +app.post('/api/pauseDownload', optionalJwt, async (req, res) => { + const download_uid = req.body.download_uid; + const success = await downloader_api.pauseDownload(download_uid); + res.send({success: success}); +}); - }); +app.post('/api/resumeDownload', optionalJwt, async (req, res) => { + const download_uid = req.body.download_uid; + const success = await downloader_api.resumeDownload(download_uid); + res.send({success: success}); +}); + +app.post('/api/restartDownload', optionalJwt, async (req, res) => { + const download_uid = req.body.download_uid; + const success = await downloader_api.restartDownload(download_uid); + res.send({success: success}); +}); + +app.post('/api/cancelDownload', optionalJwt, async (req, res) => { + const download_uid = req.body.download_uid; + const success = await downloader_api.cancelDownload(download_uid); + res.send({success: success}); +}); // logs management -app.post('/api/logs', async function(req, res) { +app.post('/api/logs', optionalJwt, async function(req, res) { let logs = null; let lines = req.body.lines; const logs_path = path.join('appdata', 'logs', 'combined.log') @@ -1779,7 +1810,7 @@ app.post('/api/logs', async function(req, res) { }); }); -app.post('/api/clearAllLogs', async function(req, res) { +app.post('/api/clearAllLogs', optionalJwt, async function(req, res) { const logs_path = path.join('appdata', 'logs', 'combined.log'); const logs_err_path = path.join('appdata', 'logs', 'error.log'); let success = false; @@ -1798,7 +1829,7 @@ app.post('/api/clearAllLogs', async function(req, res) { }); }); - app.post('/api/getFileFormats', async (req, res) => { + app.post('/api/getFileFormats', optionalJwt, async (req, res) => { let url = req.body.url; let result = await getUrlInfos(url); res.send({ diff --git a/backend/downloader.js b/backend/downloader.js index 0843a78..e7a9239 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -15,22 +15,23 @@ const utils = require('./utils'); let db_api = null; +let downloads_setup_done = false; + const archivePath = path.join(__dirname, 'appdata', 'archives'); function setDB(input_db_api) { db_api = input_db_api } exports.initialize = (input_db_api) => { setDB(input_db_api); - setInterval(checkDownloads, 10000); categories_api.initialize(db_api); - // temporary - db_api.removeAllRecords('download_queue'); + setupDownloads(); } exports.createDownload = async (url, type, options) => { const download = { url: url, type: type, + title: '', options: options, uid: uuid(), step_index: 0, @@ -45,15 +46,80 @@ exports.createDownload = async (url, type, options) => { return download; } -exports.pauseDownload = () => { +exports.pauseDownload = async (download_uid) => { + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (download['paused']) { + logger.warn(`Download ${download_uid} is already paused!`); + return false; + } else if (download['finished']) { + logger.info(`Download ${download_uid} could not be paused before completing.`); + return false; + } + + return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: true}); +} + +exports.resumeDownload = async (download_uid) => { + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (!download['paused']) { + logger.warn(`Download ${download_uid} is not paused!`); + return false; + } + return await db_api.updateRecord('download_queue', {uid: download_uid}, {paused: false}); +} + +exports.restartDownload = async (download_uid) => { + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + await exports.clearDownload(download_uid); + const success = !!(await exports.createDownload(download['url'], download['type'], download['options'])); + return success; +} + +exports.cancelDownload = async (download_uid) => { + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (download['cancelled']) { + logger.warn(`Download ${download_uid} is already cancelled!`); + return false; + } else if (download['finished']) { + logger.info(`Download ${download_uid} could not be cancelled before completing.`); + return false; + } + return await db_api.updateRecord('download_queue', {uid: download_uid}, {cancelled: true}); +} + +exports.clearDownload = async (download_uid) => { + return await db_api.removeRecord('download_queue', {uid: download_uid}); } // questions // how do we want to manage queued downloads that errored in any step? do we set the index back and finished_step to true or let the manager do it? +async function setupDownloads() { + await fixDownloadState(); + setInterval(checkDownloads, 10000); +} + +async function fixDownloadState() { + const downloads = await db_api.getRecords('download_queue'); + downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start); + const running_downloads = downloads.filter(download => !download['finished_step']); + for (let i = 0; i < running_downloads.length; i++) { + const running_download = running_downloads[i]; + const update_obj = {finished_step: true, paused: true}; + if (running_download['step_index'] > 0) { + update_obj['step_index'] = running_download['step_index'] - 1; + } + await db_api.updateRecord('download_queue', {uid: running_download['uid']}, update_obj); + } +} + async function checkDownloads() { - logger.verbose('Checking downloads'); + if (!downloads_setup_done) { + await setupDownloads(); + downloads_setup_done = true; + } + const downloads = await db_api.getRecords('download_queue'); downloads.sort((download1, download2) => download1.timestamp_start - download2.timestamp_start); const running_downloads = downloads.filter(download => !download['paused'] && download['finished_step']); @@ -65,7 +131,6 @@ async function checkDownloads() { // move to next step if (running_download['step_index'] === 0) { - collectInfo(running_download['uid']); } else if (running_download['step_index'] === 1) { downloadQueuedFile(running_download['uid']); @@ -75,9 +140,12 @@ async function checkDownloads() { } async function collectInfo(download_uid) { + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (download['paused']) { + return; + } logger.verbose(`Collecting info for download ${download_uid}`); await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 1, finished_step: false}); - const download = await db_api.getRecord('download_queue', {uid: download_uid}); const url = download['url']; const type = download['type']; @@ -127,21 +195,26 @@ async function collectInfo(download_uid) { files_to_check_for_progress.push(utils.removeFileExtension(info['_filename'])); } + const playlist_title = Array.isArray(info) ? info[0]['playlist_title'] || info[0]['playlist'] : null; await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args, finished_step: true, options: options, files_to_check_for_progress: files_to_check_for_progress, - expected_file_size: expected_file_size + expected_file_size: expected_file_size, + title: playlist_title ? playlist_title : info['title'] }); } async function downloadQueuedFile(download_uid) { + const download = await db_api.getRecord('download_queue', {uid: download_uid}); + if (download['paused']) { + return; + } logger.verbose(`Downloading ${download_uid}`); return new Promise(async resolve => { const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); await db_api.updateRecord('download_queue', {uid: download_uid}, {step_index: 2, finished_step: false}); - const download = await db_api.getRecord('download_queue', {uid: download_uid}); const url = download['url']; const type = download['type']; diff --git a/src/app/app-routing.module.ts b/src/app/app-routing.module.ts index 276e990..8620fcb 100644 --- a/src/app/app-routing.module.ts +++ b/src/app/app-routing.module.ts @@ -16,7 +16,7 @@ const routes: Routes = [ { path: 'subscription', component: SubscriptionComponent, canActivate: [PostsService] }, { path: 'settings', component: SettingsComponent, canActivate: [PostsService] }, { path: 'login', component: LoginComponent }, - { path: 'downloads', component: DownloadsComponent }, + { path: 'downloads', component: DownloadsComponent, canActivate: [PostsService] }, { path: '', redirectTo: '/home', pathMatch: 'full' } ]; diff --git a/src/app/components/downloads/downloads.component.html b/src/app/components/downloads/downloads.component.html index 41fdb9d..bbb953c 100644 --- a/src/app/components/downloads/downloads.component.html +++ b/src/app/components/downloads/downloads.component.html @@ -12,8 +12,8 @@ Title - - {{element.container.title ? element?.container.title : element.container.name}} + + {{element.title}} @@ -27,7 +27,14 @@ Progress - {{+(element.percent_complete) > 100 ? '100' : element.percent_complete}}% + + + {{+(element.percent_complete) > 100 ? '100' : element.percent_complete}}% + + + N/A + + @@ -35,12 +42,14 @@ Actions
- - + + +
- - + + +
diff --git a/src/app/components/downloads/downloads.component.scss b/src/app/components/downloads/downloads.component.scss index a6df223..e81f2d7 100644 --- a/src/app/components/downloads/downloads.component.scss +++ b/src/app/components/downloads/downloads.component.scss @@ -6,4 +6,10 @@ mat-header-cell, mat-cell { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; +} + +.icon-button-spinner { + position: absolute; + top: 7px; + left: 6px; } \ No newline at end of file diff --git a/src/app/components/downloads/downloads.component.ts b/src/app/components/downloads/downloads.component.ts index 643c21b..ba0eee5 100644 --- a/src/app/components/downloads/downloads.component.ts +++ b/src/app/components/downloads/downloads.component.ts @@ -48,7 +48,8 @@ export class DownloadsComponent implements OnInit, OnDestroy { STEP_INDEX_TO_LABEL = { 0: 'Creating download', 1: 'Getting info', - 2: 'Downloading file' + 2: 'Downloading file', + 3: 'Complete' } displayedColumns: string[] = ['date', 'title', 'stage', 'progress', 'actions']; @@ -57,25 +58,33 @@ export class DownloadsComponent implements OnInit, OnDestroy { @ViewChild(MatPaginator) paginator: MatPaginator; sort_downloads = (a, b) => { - const result = b.value.timestamp_start - a.value.timestamp_start; + const result = b.timestamp_start - a.timestamp_start; return result; } constructor(public postsService: PostsService, private router: Router) { } ngOnInit(): void { + if (this.postsService.initialized) { + this.getCurrentDownloadsRecurring(); + } else { + this.postsService.service_initialized.subscribe(init => { + if (init) { + this.getCurrentDownloadsRecurring(); + } + }); + } + } + + getCurrentDownloadsRecurring() { + if (!this.postsService.config['Extra']['enable_downloads_manager']) { + this.router.navigate(['/home']); + return; + } this.getCurrentDownloads(); this.interval_id = setInterval(() => { this.getCurrentDownloads(); }, this.downloads_check_interval); - - this.postsService.service_initialized.subscribe(init => { - if (init) { - if (!this.postsService.config['Extra']['enable_downloads_manager']) { - this.router.navigate(['/home']); - } - } - }); } ngOnDestroy(): void { @@ -88,6 +97,7 @@ export class DownloadsComponent implements OnInit, OnDestroy { && res['downloads'] !== undefined && JSON.stringify(this.downloads) !== JSON.stringify(res['downloads'])) { this.downloads = res['downloads']; + this.downloads.sort(this.sort_downloads); this.dataSource = new MatTableDataSource(this.downloads); this.dataSource.paginator = this.paginator; } else { @@ -97,13 +107,64 @@ export class DownloadsComponent implements OnInit, OnDestroy { } clearFinishedDownloads(): void { - this.postsService.clearDownloads(false).subscribe(res => { - if (res['success']) { - this.downloads = res['downloads']; + this.postsService.clearFinishedDownloads().subscribe(res => { + if (!res['success']) { + this.postsService.openSnackBar('Failed to clear finished downloads!'); + } + }); + } + + pauseDownload(download_uid) { + this.postsService.pauseDownload(download_uid).subscribe(res => { + if (!res['success']) { + this.postsService.openSnackBar('Failed to pause download! See server logs for more info.'); + } + }); + } + + resumeDownload(download_uid) { + this.postsService.resumeDownload(download_uid).subscribe(res => { + if (!res['success']) { + this.postsService.openSnackBar('Failed to resume download! See server logs for more info.'); } }); } + restartDownload(download_uid) { + this.postsService.restartDownload(download_uid).subscribe(res => { + if (!res['success']) { + this.postsService.openSnackBar('Failed to restart download! See server logs for more info.'); + } + }); + } + + cancelDownload(download_uid) { + this.postsService.cancelDownload(download_uid).subscribe(res => { + if (!res['success']) { + this.postsService.openSnackBar('Failed to cancel download! See server logs for more info.'); + } + }); + } + + clearDownload(download_uid) { + this.postsService.clearDownload(download_uid).subscribe(res => { + if (!res['success']) { + this.postsService.openSnackBar('Failed to pause download! See server logs for more info.'); + } + }); + } + + watchContent(download) { + const container = download['container']; + localStorage.setItem('player_navigator', this.router.url.split(';')[0]); + const is_playlist = container['uids']; // hacky, TODO: fix + if (is_playlist) { + this.router.navigate(['/player', {playlist_id: container['id'], type: download['type']}]); + } else { + this.router.navigate(['/player', {type: download['type'], uid: container['uid']}]); + } + } + } export interface Download { diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index d3a21a3..626c839 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -413,24 +413,38 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'getSubscriptions', {}, this.httpOptions); } - // current downloads getCurrentDownloads() { return this.http.get(this.path + 'downloads', this.httpOptions); } - // current download getCurrentDownload(download_uid) { return this.http.post(this.path + 'download', {download_uid: download_uid}, this.httpOptions); } - // clear downloads. download_id is optional, if it exists only 1 download will be cleared - clearDownloads(delete_all = false, session_id = null, download_id = null) { - return this.http.post(this.path + 'clearDownloads', {delete_all: delete_all, - download_id: download_id, - session_id: session_id ? session_id : this.session_id}, this.httpOptions); + pauseDownload(download_uid) { + return this.http.post(this.path + 'pauseDownload', {download_uid: download_uid}, this.httpOptions); + } + + resumeDownload(download_uid) { + return this.http.post(this.path + 'resumeDownload', {download_uid: download_uid}, this.httpOptions); + } + + restartDownload(download_uid) { + return this.http.post(this.path + 'restartDownload', {download_uid: download_uid}, this.httpOptions); + } + + cancelDownload(download_uid) { + return this.http.post(this.path + 'cancelDownload', {download_uid: download_uid}, this.httpOptions); + } + + clearDownload(download_uid) { + return this.http.post(this.path + 'clearDownload', {download_uid: download_uid}, this.httpOptions); + } + + clearFinishedDownloads() { + return this.http.post(this.path + 'clearFinishedDownloads', {}, this.httpOptions); } - // updates the server to the latest version updateServer(tag) { return this.http.post(this.path + 'updateServer', {tag: tag}, this.httpOptions); }