From 2b1771d30d7562945208e1df5ef12b8cc9dc5252 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 17 Apr 2022 23:37:47 -0400 Subject: [PATCH 01/10] Began work on tasks --- backend/db.js | 72 +++++++++++++++++++++++++-- backend/tasks.js | 112 ++++++++++++++++++++++++++++++++++++++++++ backend/test/tests.js | 91 +++++++++++++++++++++++++++++++++- backend/utils.js | 7 +++ 4 files changed, 278 insertions(+), 4 deletions(-) create mode 100644 backend/tasks.js diff --git a/backend/db.js b/backend/db.js index 18f9064..af51a4e 100644 --- a/backend/db.js +++ b/backend/db.js @@ -300,6 +300,7 @@ exports.getFileDirectoriesAndDBs = async () => { } exports.importUnregisteredFiles = async () => { + const imported_files = []; const dirs_to_check = await exports.getFileDirectoriesAndDBs(); // run through check list and check each file to see if it's missing from the db @@ -316,12 +317,17 @@ exports.importUnregisteredFiles = async () => { const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path))); if (!file_is_registered) { // add additional info - await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); - logger.verbose(`Added discovered file to the database: ${file.id}`); + const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); + if (file_obj) { + imported_files.push(file_obj['uid']); + logger.verbose(`Added discovered file to the database: ${file.id}`); + } else { + logger.error(`Failed to import ${file['path']} automatically.`); + } } } } - + return imported_files; } exports.addMetadataPropertyToDB = async (property_key) => { @@ -744,6 +750,66 @@ exports.removeRecord = async (table, filter_obj) => { return !!(output['result']['ok']); } +// exports.removeRecordsByUIDBulk = async (table, uids) => { +// // local db override +// if (using_local_db) { +// applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write(); +// return true; +// } + +// const table_collection = database.collection(table); + +// let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch + +// const item_ids_to_remove = + +// for (let i = 0; i < item_ids_to_update.length; i++) { +// const item_id_to_update = item_ids_to_update[i]; +// bulk.find({[key_label]: item_id_to_update }).updateOne({ +// "$set": update_obj[item_id_to_update] +// }); +// } + +// const output = await bulk.execute(); +// return !!(output['result']['ok']); +// } + + +exports.findDuplicatesByKey = async (table, key) => { + let duplicates = []; + if (using_local_db) { + // this can probably be optimized + const all_records = await exports.getRecords(table); + const existing_records = {}; + for (let i = 0; i < all_records.length; i++) { + const record = all_records[i]; + const value = record[key]; + + if (existing_records[value]) { + duplicates.push(record); + } + + existing_records[value] = true; + } + return duplicates; + } + + const duplicated_values = await database.collection(table).aggregate([ + {"$group" : { "_id": `$${key}`, "count": { "$sum": 1 } } }, + {"$match": {"_id" :{ "$ne" : null } , "count" : {"$gt": 1} } }, + {"$project": {[key] : "$_id", "_id" : 0} } + ]).toArray(); + + for (let i = 0; i < duplicated_values.length; i++) { + const duplicated_value = duplicated_values[i]; + const duplicated_records = await exports.getRecords(table, duplicated_value, false); + if (duplicated_records.length > 1) { + duplicates = duplicates.concat(duplicated_records.slice(1, duplicated_records.length)); + } + } + return duplicates; +} + exports.removeAllRecords = async (table = null, filter_obj = null) => { // local db override const tables_to_remove = table ? [table] : tables_list; diff --git a/backend/tasks.js b/backend/tasks.js new file mode 100644 index 0000000..e928725 --- /dev/null +++ b/backend/tasks.js @@ -0,0 +1,112 @@ +const utils = require('./utils'); +const db_api = require('./db'); + +const fs = require('fs-extra'); +const logger = require('./logger'); + +const TASKS = { + backup_local_db: { + run: utils.backupLocalDB, + title: 'Backup Local DB', + }, + missing_files_check: { + run: checkForMissingFiles, + confirm: deleteMissingFiles, + title: 'Missing files check' + }, + missing_db_records: { + run: db_api.importUnregisteredFiles, + title: 'Import missing DB records' + }, + duplicate_files_check: { + run: checkForDuplicateFiles, + confirm: removeDuplicates, + title: 'Find duplicate files in DB' + } +} + +exports.initialize = async () => { + const tasks_keys = Object.keys(TASKS); + for (let i = 0; i < tasks_keys.length; i++) { + const task_key = tasks_keys[i]; + const task_in_db = await db_api.getRecord('tasks', {key: task_key}); + if (!task_in_db) { + await db_api.insertRecordIntoTable('tasks', { + key: task_key, + last_ran: null, + last_confirmed: null, + running: false, + confirming: false, + data: null, + error: null + }); + } + } +} + +exports.executeTask = async (task_key) => { + if (!TASKS[task_key]) { + logger.error(`Task ${task_key} does not exist!`); + return; + } + logger.verbose(`Executing task ${task_key}`); + await exports.executeRun(task_key); + if (!TASKS[task_key]['confirm']) return; + await exports.executeConfirm(task_key); + logger.verbose(`Finished executing ${task_key}`); +} + +exports.executeRun = async (task_key) => { + await db_api.updateRecord('tasks', {key: task_key}, {running: true}); + const data = await TASKS[task_key].run(); + await db_api.updateRecord('tasks', {key: task_key}, {data: data, last_ran: Date.now()/1000, running: false}); +} + +exports.executeConfirm = async (task_key) => { + if (!TASKS[task_key]['confirm']) { + return null; + } + await db_api.updateRecord('tasks', {key: task_key}, {confirming: true}); + const task_obj = await db_api.getRecord('tasks', {key: task_key}); + const data = task_obj['data']; + await TASKS[task_key].confirm(data); + await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000}); +} + +// missing files check + +async function checkForMissingFiles() { + const missing_files = []; + const all_files = await db_api.getRecords('files'); + for (let i = 0; i < all_files.length; i++) { + const file_to_check = all_files[i]; + const file_exists = fs.existsSync(file_to_check['path']); + if (!file_exists) missing_files.push(file_to_check['uid']); + } + return {uids: missing_files}; +} + +async function deleteMissingFiles(data) { + const uids = data['uids']; + for (let i = 0; i < uids.length; i++) { + const uid = uids[i]; + await db_api.removeRecord('files', {uid: uid}); + } +} + +// duplicate files check + +async function checkForDuplicateFiles() { + const duplicate_files = await db_api.findDuplicatesByKey('files', 'path'); + const duplicate_uids = duplicate_files.map(duplicate_file => duplicate_file['uid']); + if (duplicate_uids && duplicate_uids.length > 0) { + return {uids: duplicate_uids}; + } + return {uids: []}; +} + +async function removeDuplicates(data) { + for (let i = 0; i < data['uids'].length; i++) { + await db_api.removeRecord('files', {uid: data['uids'][i]}); + } +} \ No newline at end of file diff --git a/backend/test/tests.js b/backend/test/tests.js index c52fa13..7856f28 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -83,12 +83,37 @@ describe('Database', async function() { await db_api.removeAllRecords('test'); }); it('Add and read record', async function() { + this.timeout(120000); await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined}); const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null}); assert(added_record['test_add'] === 'test'); await db_api.removeRecord('test', {test_add: 'test'}); }); + it('Find duplicates by key', async function() { + const test_duplicates = [ + { + test: 'testing', + key: '1' + }, + { + test: 'testing', + key: '2' + }, + { + test: 'testing_missing', + key: '3' + }, + { + test: 'testing', + key: '4' + } + ]; + await db_api.insertRecordsIntoTable('test', test_duplicates); + const duplicates = await db_api.findDuplicatesByKey('test', 'test'); + console.log(duplicates); + }); + it('Update record', async function() { await db_api.insertRecordIntoTable('test', {test_update: 'test'}); await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true}); @@ -122,6 +147,7 @@ describe('Database', async function() { }); it('Bulk add', async function() { + this.timeout(120000); const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000 const test_records = []; for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { @@ -291,7 +317,6 @@ describe('Multi User', async function() { describe('Downloader', function() { const downloader_api = require('../downloader'); - downloader_api.initialize(db_api); const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'; const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c'; const options = { @@ -348,5 +373,69 @@ describe('Downloader', function() { const sample_json = fs.readJSONSync('./test/sample.info.json'); downloader_api.generateNFOFile(sample_json, nfo_file_path); assert(fs.existsSync(nfo_file_path), true); + fs.unlinkSync(nfo_file_path); }); }); + +describe('Tasks', function() { + const tasks_api = require('../tasks'); + beforeEach(async function() { + await db_api.connectToDB(); + await db_api.removeAllRecords('tasks'); + await tasks_api.initialize(); + }); + it('Backup local db', async function() { + const backups_original = await utils.recFindByExt('appdata', 'bak'); + const original_length = backups_original.length; + await tasks_api.executeTask('backup_local_db'); + const backups_new = await utils.recFindByExt('appdata', 'bak'); + const new_length = backups_new.length; + assert(original_length, new_length-1); + }); + + it('Check for missing files', async function() { + await db_api.removeAllRecords('files', {uid: 'test'}); + const test_missing_file = {uid: 'test', path: 'test/missing_file.mp4'}; + await db_api.insertRecordIntoTable('files', test_missing_file); + await tasks_api.executeTask('missing_files_check'); + const task_obj = await db_api.getRecord('tasks', {key: 'missing_files_check'}); + assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true); + }); + + it('Check for duplicate files', async function() { + this.timeout(300000); + await db_api.removeAllRecords('files', {uid: 'test1'}); + await db_api.removeAllRecords('files', {uid: 'test2'}); + const test_duplicate_file1 = {uid: 'test1', path: 'test/missing_file.mp4'}; + const test_duplicate_file2 = {uid: 'test2', path: 'test/missing_file.mp4'}; + const test_duplicate_file3 = {uid: 'test3', path: 'test/missing_file.mp4'}; + await db_api.insertRecordIntoTable('files', test_duplicate_file1); + await db_api.insertRecordIntoTable('files', test_duplicate_file2); + await db_api.insertRecordIntoTable('files', test_duplicate_file3); + await tasks_api.executeTask('duplicate_files_check'); + const task_obj = await db_api.getRecord('tasks', {key: 'duplicate_files_check'}); + const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true); + assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true); + assert(duplicated_record_count == 1, true); + }); + + it('Import unregistered files', async function() { + this.timeout(300000); + + // pre-test cleanup + await db_api.removeAllRecords('files', {title: 'Sample File'}); + if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json'); + if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4'); + + // copies in files + fs.copyFileSync('test/sample.info.json', 'video/sample.info.json'); + fs.copyFileSync('test/sample.mp4', 'video/sample.mp4'); + await tasks_api.executeTask('missing_db_records'); + const imported_file = await db_api.getRecord('files', {title: 'Sample File'}); + assert(!!imported_file, true); + + // post-test cleanup + if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json'); + if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4'); + }); +}); \ No newline at end of file diff --git a/backend/utils.js b/backend/utils.js index 137274d..340a214 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -266,6 +266,12 @@ function getCurrentDownloader() { return details_json['downloader']; } +async function backupLocalDB() { + const path_to_backups = path.join('appdata', 'db_backup'); + fs.ensureDir(path_to_backups); + await fs.copyFile('appdata/local_db.json', path.join(path_to_backups, `local_db.json.${Date.now()/1000}.bak`)); +} + async function recFindByExt(base,ext,files,result) { files = files || (await fs.readdir(base)) @@ -390,6 +396,7 @@ module.exports = { getMatchingCategoryFiles: getMatchingCategoryFiles, addUIDsToCategory: addUIDsToCategory, getCurrentDownloader: getCurrentDownloader, + backupLocalDB: backupLocalDB, recFindByExt: recFindByExt, removeFileExtension: removeFileExtension, formatDateString: formatDateString, From 5b4d4d5f813a78bb254986ea39560a54534260d9 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 19 Apr 2022 22:29:41 -0400 Subject: [PATCH 02/10] Added scheduler for tasks --- backend/db.js | 4 ++ backend/package-lock.json | 102 ++++++++++++++++++++++++++++++++++++++ backend/package.json | 1 + backend/tasks.js | 50 +++++++++++++++++-- backend/test/tests.js | 29 +++++++++++ 5 files changed, 181 insertions(+), 5 deletions(-) diff --git a/backend/db.js b/backend/db.js index af51a4e..07dc95d 100644 --- a/backend/db.js +++ b/backend/db.js @@ -54,6 +54,10 @@ const tables = { name: 'download_queue', primary_key: 'uid' }, + tasks: { + name: 'tasks', + primary_key: 'key' + }, test: { name: 'test' } diff --git a/backend/package-lock.json b/backend/package-lock.json index d8d0e37..76daecd 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -599,6 +599,15 @@ } } }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -909,6 +918,15 @@ "readable-stream": "^3.4.0" } }, + "cron-parser": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-3.5.0.tgz", + "integrity": "sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==", + "requires": { + "is-nan": "^1.3.2", + "luxon": "^1.26.0" + } + }, "cross-spawn": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz", @@ -968,6 +986,15 @@ "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==" }, + "define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "requires": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -1358,11 +1385,26 @@ "rimraf": "2" } }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, "get-stream": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", @@ -1460,11 +1502,32 @@ "har-schema": "^2.0.0" } }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "requires": { + "get-intrinsic": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, "has-yarn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", @@ -1635,6 +1698,15 @@ "is-path-inside": "^3.0.1" } }, + "is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + } + }, "is-npm": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", @@ -2034,6 +2106,11 @@ } } }, + "long-timeout": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz", + "integrity": "sha1-lyHXiLR+C8taJMLivuGg2lXatRQ=" + }, "lowdb": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-1.0.0.tgz", @@ -2051,6 +2128,11 @@ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" }, + "luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==" + }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -2407,6 +2489,16 @@ "iconv-lite": "^0.4.15" } }, + "node-schedule": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz", + "integrity": "sha512-nl4JTiZ7ZQDc97MmpTq9BQjYhq7gOtoh7SiPH069gBFBj0PzD8HI7zyFs6rzqL8Y5tTiEEYLxgtbx034YPrbyQ==", + "requires": { + "cron-parser": "^3.5.0", + "long-timeout": "0.1.1", + "sorted-array-functions": "^1.3.0" + } + }, "nodemon": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz", @@ -2475,6 +2567,11 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -3053,6 +3150,11 @@ "is-arrayish": "^0.3.1" } }, + "sorted-array-functions": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz", + "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==" + }, "sparse-bitfield": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", diff --git a/backend/package.json b/backend/package.json index d30ded2..76db3a5 100644 --- a/backend/package.json +++ b/backend/package.json @@ -52,6 +52,7 @@ "multer": "^1.4.2", "node-fetch": "^2.6.7", "node-id3": "^0.1.14", + "node-schedule": "^2.1.0", "nodemon": "^2.0.7", "passport": "^0.4.1", "passport-http": "^0.3.0", diff --git a/backend/tasks.js b/backend/tasks.js index e928725..701cbff 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -3,34 +3,53 @@ const db_api = require('./db'); const fs = require('fs-extra'); const logger = require('./logger'); +const scheduler = require('node-schedule'); const TASKS = { backup_local_db: { run: utils.backupLocalDB, title: 'Backup Local DB', + job: null }, missing_files_check: { run: checkForMissingFiles, confirm: deleteMissingFiles, - title: 'Missing files check' + title: 'Missing files check', + job: null }, missing_db_records: { run: db_api.importUnregisteredFiles, - title: 'Import missing DB records' + title: 'Import missing DB records', + job: null }, duplicate_files_check: { run: checkForDuplicateFiles, confirm: removeDuplicates, - title: 'Find duplicate files in DB' + title: 'Find duplicate files in DB', + job: null } } +function scheduleJob(task_key, schedule) { + return scheduler.scheduleJob(schedule, async () => { + const task_state = await db_api.getRecord('tasks', {key: task_key}); + if (task_state['running'] || task_state['confirming']) { + logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`); + return; + } + + // we're just "running" the task, any confirmation should be user-initiated + exports.executeRun(task_key); + }); +} + exports.initialize = async () => { const tasks_keys = Object.keys(TASKS); for (let i = 0; i < tasks_keys.length; i++) { const task_key = tasks_keys[i]; const task_in_db = await db_api.getRecord('tasks', {key: task_key}); if (!task_in_db) { + // insert task into table if missing await db_api.insertRecordIntoTable('tasks', { key: task_key, last_ran: null, @@ -38,8 +57,17 @@ exports.initialize = async () => { running: false, confirming: false, data: null, - error: null + error: null, + schedule: null }); + } else { + // reset task if necessary + await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false}); + + // schedule task and save job + if (task_in_db['schedule']) { + TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']); + } } } } @@ -73,6 +101,16 @@ exports.executeConfirm = async (task_key) => { await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000}); } +exports.updateTaskSchedule = async (task_key, schedule) => { + await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule}); + if (TASKS[task_key]['job']) { + TASKS[task_key]['job'].cancel(); + } + if (schedule) { + TASKS[task_key]['job'] = scheduleJob(task_key, schedule); + } +} + // missing files check async function checkForMissingFiles() { @@ -109,4 +147,6 @@ async function removeDuplicates(data) { for (let i = 0; i < data['uids'].length; i++) { await db_api.removeRecord('files', {uid: data['uids'][i]}); } -} \ No newline at end of file +} + +exports.TASKS = TASKS; \ No newline at end of file diff --git a/backend/test/tests.js b/backend/test/tests.js index 7856f28..0c26fdc 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -382,6 +382,15 @@ describe('Tasks', function() { beforeEach(async function() { await db_api.connectToDB(); await db_api.removeAllRecords('tasks'); + + const dummy_task = { + run: async () => { await utils.wait(500); return true; }, + confirm: async () => { await utils.wait(500); return true; }, + title: 'Dummy task', + job: null + }; + tasks_api.TASKS['dummy_task'] = dummy_task; + await tasks_api.initialize(); }); it('Backup local db', async function() { @@ -438,4 +447,24 @@ describe('Tasks', function() { if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json'); if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4'); }); + + it('Schedule and cancel task', async function() { + const today_4_hours = new Date(); + today_4_hours.setHours(today_4_hours.getHours() + 4); + await tasks_api.updateTaskSchedule('dummy_task', today_4_hours); + assert(!!tasks_api.TASKS['dummy_task']['job'], true); + await tasks_api.updateTaskSchedule('dummy_task', null); + assert(!!tasks_api.TASKS['dummy_task']['job'], false); + }); + + it('Schedule and run task', async function() { + this.timeout(5000); + const today_1_second = new Date(); + today_1_second.setSeconds(today_1_second.getSeconds() + 1); + await tasks_api.updateTaskSchedule('dummy_task', today_1_second); + assert(!!tasks_api.TASKS['dummy_task']['job'], true); + await utils.wait(2000); + const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'}); + assert(dummy_task_obj['data'], true); + }); }); \ No newline at end of file From 091f81bb388a7c5e02e60fa17dd782a1bea331b2 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 21 Apr 2022 03:01:49 -0400 Subject: [PATCH 03/10] Added UI for managing tasks Added ability to schedule tasks based on timestamp Fixed mismatched types between frontend and openapi yaml Simplified imports for several backend components --- Public API v1.yaml | 363 +++++++++++++++++- backend/app.js | 52 ++- backend/authentication/auth.js | 10 +- backend/categories.js | 11 +- backend/downloader.js | 21 +- backend/subscriptions.js | 12 +- backend/tasks.js | 36 +- src/api-types/index.ts | 6 + src/api-types/models/DownloadFileRequest.ts | 3 +- src/api-types/models/GetAllTasksResponse.ts | 9 + .../models/GetFullTwitchChatRequest.ts | 3 - src/api-types/models/GetTaskRequest.ts | 8 + src/api-types/models/GetTaskResponse.ts | 9 + src/api-types/models/Schedule.ts | 24 ++ src/api-types/models/Task.ts | 15 + .../models/UpdateTaskScheduleRequest.ts | 10 + src/app/app-routing.module.ts | 2 + src/app/app.component.html | 1 + src/app/app.module.ts | 8 +- src/app/components/tasks/tasks.component.html | 77 ++++ src/app/components/tasks/tasks.component.scss | 32 ++ .../components/tasks/tasks.component.spec.ts | 25 ++ src/app/components/tasks/tasks.component.ts | 109 ++++++ ...update-task-schedule-dialog.component.html | 53 +++ ...update-task-schedule-dialog.component.scss | 0 ...ate-task-schedule-dialog.component.spec.ts | 25 ++ .../update-task-schedule-dialog.component.ts | 83 ++++ src/app/player/player.component.ts | 2 +- src/app/posts.services.ts | 54 ++- 29 files changed, 996 insertions(+), 67 deletions(-) create mode 100644 src/api-types/models/GetAllTasksResponse.ts create mode 100644 src/api-types/models/GetTaskRequest.ts create mode 100644 src/api-types/models/GetTaskResponse.ts create mode 100644 src/api-types/models/Schedule.ts create mode 100644 src/api-types/models/Task.ts create mode 100644 src/api-types/models/UpdateTaskScheduleRequest.ts create mode 100644 src/app/components/tasks/tasks.component.html create mode 100644 src/app/components/tasks/tasks.component.scss create mode 100644 src/app/components/tasks/tasks.component.spec.ts create mode 100644 src/app/components/tasks/tasks.component.ts create mode 100644 src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.html create mode 100644 src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.scss create mode 100644 src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.spec.ts create mode 100644 src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts diff --git a/Public API v1.yaml b/Public API v1.yaml index ad43d48..fc63ef8 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -666,11 +666,287 @@ paths: schema: $ref: '#/components/schemas/GetDownloadRequest' description: '' - description: "Gets a single download using its download_id and session_id. session_id is the device fingerprint. If none was provided at the time of download, then set session_id is 'undeclared'." + description: "Gets a single download using its download_id." security: - Auth query parameter: [] tags: - downloader + /api/pauseDownload: + post: + summary: Pauses one download + operationId: post-api-pause-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Pause a single download using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/pauseAllDownloads: + post: + tags: + - downloader + summary: Pauses all downloads + operationId: post-api-pause-all-downloads + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/resumeDownload: + post: + summary: Resume one download + operationId: post-api-resume-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Resume a single download using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/resumeAllDownloads: + post: + tags: + - downloader + summary: Resumes all downloads + operationId: post-api-resume-all-downloads + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/restartDownload: + post: + summary: Restart one download + operationId: post-api-restart-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Restart a single download using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/cancelDownload: + post: + summary: Cancel one download + operationId: post-api-cancel-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Cancel a single download using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/clearDownload: + post: + summary: Clear one download + operationId: post-api-clear-download + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetDownloadRequest' + description: '' + description: "Clears a single download from the downloaded list using its download_id." + security: + - Auth query parameter: [] + tags: + - downloader + /api/clearFinishedDownloads: + post: + tags: + - downloader + summary: Clear finished downloads + operationId: post-api-clear-finished-downloads + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/getTask: + post: + summary: Get info for one task + operationId: post-api-get-task + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskResponse' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskRequest' + description: '' + description: "Gets a single task using its key." + security: + - Auth query parameter: [] + tags: + - tasks + /api/getTasks: + post: + tags: + - tasks + summary: Get tasks + operationId: post-api-get-tasks + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetAllTasksResponse' + security: + - Auth query parameter: [] + /api/runTask: + post: + summary: Runs one task + operationId: post-api-run-task + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskRequest' + /api/confirmTask: + post: + summary: Confirms a task + operationId: post-api-confirm-task + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskRequest' + /api/cancelTask: + post: + summary: Cancels a task + operationId: post-api-cancel-task + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/GetTaskRequest' + /api/updateTaskSchedule: + post: + summary: Updates task schedule + operationId: post-api-update-task-schedule + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateTaskScheduleRequest' /api/auth/login: post: summary: Login @@ -1231,6 +1507,35 @@ components: type: array items: $ref: '#/components/schemas/Download' + GetTaskRequest: + type: object + properties: + task_key: + type: string + required: + - task_key + UpdateTaskScheduleRequest: + type: object + properties: + task_key: + type: string + new_schedule: + $ref: '#/components/schemas/Schedule' + required: + - task_key + - new_schedule + GetTaskResponse: + type: object + properties: + task: + $ref: '#/components/schemas/Task' + GetAllTasksResponse: + type: object + properties: + tasks: + type: array + items: + $ref: '#/components/schemas/Task' GetMp3sResponse: required: - mp3s @@ -1506,6 +1811,10 @@ components: type: string playlist_id: type: string + url: + type: string + type: + $ref: '#/components/schemas/FileType' DownloadArchiveRequest: required: - sub @@ -1967,6 +2276,58 @@ components: type: string sub_name: type: string + Task: + required: + - key + - last_ran + - last_confirmed + - running + - confirming + - data + - error + - schedule + type: object + properties: + key: + type: string + last_ran: + type: number + last_confirmed: + type: number + running: + type: boolean + confirming: + type: boolean + data: + type: object + error: + type: string + schedule: + type: object + Schedule: + required: + - type + - data + type: object + properties: + type: + type: string + enum: + - timestamp + - recurring + data: + type: object + properties: + dayOfWeek: + type: array + items: + type: number + hour: + type: number + minute: + type: number + timestamp: + type: number SubscriptionRequestData: required: - id diff --git a/backend/app.js b/backend/app.js index b12d502..8aa2696 100644 --- a/backend/app.js +++ b/backend/app.js @@ -28,6 +28,7 @@ const youtubedl = require('youtube-dl'); const logger = require('./logger'); const config_api = require('./config.js'); const downloader_api = require('./downloader'); +const tasks_api = require('./tasks'); const subscriptions_api = require('./subscriptions'); const categories_api = require('./categories'); const twitch_api = require('./twitch'); @@ -60,9 +61,6 @@ const admin_token = '4241b401-7236-493e-92b5-b72696b9d853'; config_api.initialize(); db_api.initialize(db, users_db); auth_api.initialize(db_api); -downloader_api.initialize(db_api); -subscriptions_api.initialize(db_api, downloader_api); -categories_api.initialize(db_api); // Set some defaults db.defaults( @@ -1878,6 +1876,54 @@ app.post('/api/cancelDownload', optionalJwt, async (req, res) => { res.send({success: success}); }); +// tasks + +app.post('/api/getTasks', optionalJwt, async (req, res) => { + const tasks = await db_api.getRecords('tasks'); + for (let task of tasks) { + if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task['key']]['job'].nextInvocation().getTime(); + } + res.send({tasks: tasks}); +}); + +app.post('/api/getTask', optionalJwt, async (req, res) => { + const task_key = req.body.task_key; + const task = await db_api.getRecord('tasks', {key: task_key}); + if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task_key]['job'].nextInvocation().getTime(); + res.send({task: task}); +}); + +app.post('/api/runTask', optionalJwt, async (req, res) => { + const task_key = req.body.task_key; + const task = await db_api.getRecord('tasks', {key: task_key}); + + let success = true; + if (task['running'] || task['confirming']) success = false; + else await tasks_api.executeRun(task_key); + + res.send({success: success}); +}); + +app.post('/api/confirmTask', optionalJwt, async (req, res) => { + const task_key = req.body.task_key; + const task = await db_api.getRecord('tasks', {key: task_key}); + + let success = true; + if (task['running'] || task['confirming'] || !task['data']) success = false; + else await tasks_api.executeConfirm(task_key); + + res.send({success: success}); +}); + +app.post('/api/updateTaskSchedule', optionalJwt, async (req, res) => { + const task_key = req.body.task_key; + const new_schedule = req.body.new_schedule; + + await tasks_api.updateTaskSchedule(task_key, new_schedule); + + res.send({success: true}); +}); + // logs management app.post('/api/logs', optionalJwt, async function(req, res) { diff --git a/backend/authentication/auth.js b/backend/authentication/auth.js index 7aad070..de54a0b 100644 --- a/backend/authentication/auth.js +++ b/backend/authentication/auth.js @@ -1,6 +1,7 @@ const config_api = require('../config'); const consts = require('../consts'); const logger = require('../logger'); +const db_api = require('../db'); const jwt = require('jsonwebtoken'); const { uuid } = require('uuidv4'); @@ -12,15 +13,12 @@ var JwtStrategy = require('passport-jwt').Strategy, ExtractJwt = require('passport-jwt').ExtractJwt; // other required vars -let db_api = null; let SERVER_SECRET = null; let JWT_EXPIRATION = null; let opts = null; let saltRounds = null; -exports.initialize = function(db_api) { - setDB(db_api); - +exports.initialize = function() { /************************* * Authentication module ************************/ @@ -51,10 +49,6 @@ exports.initialize = function(db_api) { })); } -function setDB(input_db_api) { - db_api = input_db_api; -} - exports.passport = require('passport'); exports.passport.serializeUser(function(user, done) { diff --git a/backend/categories.js b/backend/categories.js index 269ae9c..2236d9f 100644 --- a/backend/categories.js +++ b/backend/categories.js @@ -1,14 +1,6 @@ const utils = require('./utils'); const logger = require('./logger'); - -var db_api = null; - -function setDB(input_db_api) { db_api = input_db_api } - -function initialize(input_db_api) { - setDB(input_db_api); -} - +const db_api = require('./db'); /* Categories: @@ -137,7 +129,6 @@ function applyCategoryRules(file_json, rules, category_name) { // } module.exports = { - initialize: initialize, categorize: categorize, getCategories: getCategories, getCategoriesAsPlaylists: getCategoriesAsPlaylists diff --git a/backend/downloader.js b/backend/downloader.js index e151f87..f139a1f 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -14,26 +14,19 @@ const twitch_api = require('./twitch'); const { create } = require('xmlbuilder2'); const categories_api = require('./categories'); const utils = require('./utils'); - -let db_api = null; +const db_api = require('./db'); const mutex = new Mutex(); let should_check_downloads = true; const archivePath = path.join(__dirname, 'appdata', 'archives'); -function setDB(input_db_api) { db_api = input_db_api } - -exports.initialize = (input_db_api) => { - setDB(input_db_api); - categories_api.initialize(db_api); - if (db_api.database_initialized) { - setupDownloads(); - } else { - db_api.database_initialized_bs.subscribe(init => { - if (init) setupDownloads(); - }); - } +if (db_api.database_initialized) { + setupDownloads(); +} else { + db_api.database_initialized_bs.subscribe(init => { + if (init) setupDownloads(); + }); } exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => { diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 869cf76..1b35a4f 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -8,15 +8,8 @@ const logger = require('./logger'); const debugMode = process.env.YTDL_MODE === 'debug'; -let db_api = null; -let downloader_api = null; - -function setDB(input_db_api) { db_api = input_db_api } - -function initialize(input_db_api, input_downloader_api) { - setDB(input_db_api); - downloader_api = input_downloader_api; -} +const db_api = require('./db'); +const downloader_api = require('./downloader'); async function subscribe(sub, user_uid = null) { const result_obj = { @@ -542,7 +535,6 @@ module.exports = { unsubscribe : unsubscribe, deleteSubscriptionFile : deleteSubscriptionFile, getVideosForSub : getVideosForSub, - initialize : initialize, updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple, generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload } diff --git a/backend/tasks.js b/backend/tasks.js index 701cbff..c66b23a 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -31,7 +31,21 @@ const TASKS = { } function scheduleJob(task_key, schedule) { - return scheduler.scheduleJob(schedule, async () => { + // schedule has to be converted from our format to one node-schedule can consume + let converted_schedule = null; + if (schedule['type'] === 'timestamp') { + converted_schedule = new Date(schedule['data']['timestamp']); + } else if (schedule['type'] === 'recurring') { + const dayOfWeek = schedule['data']['dayOfWeek'] ? schedule['data']['dayOfWeek'] : null; + const hour = schedule['data']['hour'] ? schedule['data']['hour'] : null; + const minute = schedule['data']['minute'] ? schedule['data']['minute'] : null; + converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute); + } else { + logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`) + return null; + } + + return scheduler.scheduleJob(converted_schedule, async () => { const task_state = await db_api.getRecord('tasks', {key: task_key}); if (task_state['running'] || task_state['confirming']) { logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`); @@ -43,15 +57,24 @@ function scheduleJob(task_key, schedule) { }); } -exports.initialize = async () => { +if (db_api.database_initialized) { + setupTasks(); +} else { + db_api.database_initialized_bs.subscribe(init => { + if (init) setupTasks(); + }); +} + +const setupTasks = async () => { const tasks_keys = Object.keys(TASKS); for (let i = 0; i < tasks_keys.length; i++) { const task_key = tasks_keys[i]; const task_in_db = await db_api.getRecord('tasks', {key: task_key}); if (!task_in_db) { - // insert task into table if missing + // insert task metadata into table if missing await db_api.insertRecordIntoTable('tasks', { key: task_key, + title: TASKS[task_key]['title'], last_ran: null, last_confirmed: null, running: false, @@ -85,12 +108,15 @@ exports.executeTask = async (task_key) => { } exports.executeRun = async (task_key) => { + logger.verbose(`Running task ${task_key}`); await db_api.updateRecord('tasks', {key: task_key}, {running: true}); const data = await TASKS[task_key].run(); await db_api.updateRecord('tasks', {key: task_key}, {data: data, last_ran: Date.now()/1000, running: false}); + logger.verbose(`Finished running task ${task_key}`); } exports.executeConfirm = async (task_key) => { + logger.verbose(`Confirming task ${task_key}`); if (!TASKS[task_key]['confirm']) { return null; } @@ -98,10 +124,12 @@ exports.executeConfirm = async (task_key) => { const task_obj = await db_api.getRecord('tasks', {key: task_key}); const data = task_obj['data']; await TASKS[task_key].confirm(data); - await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000}); + await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null}); + logger.verbose(`Finished confirming task ${task_key}`); } exports.updateTaskSchedule = async (task_key, schedule) => { + logger.verbose(`Updating schedule for task ${task_key}`); await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule}); if (TASKS[task_key]['job']) { TASKS[task_key]['job'].cancel(); diff --git a/src/api-types/index.ts b/src/api-types/index.ts index c08044a..93d611f 100644 --- a/src/api-types/index.ts +++ b/src/api-types/index.ts @@ -44,6 +44,7 @@ export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest'; export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse'; export type { GetAllFilesResponse } from './models/GetAllFilesResponse'; export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse'; +export type { GetAllTasksResponse } from './models/GetAllTasksResponse'; export type { GetDownloadRequest } from './models/GetDownloadRequest'; export type { GetDownloadResponse } from './models/GetDownloadResponse'; export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest'; @@ -63,6 +64,8 @@ export type { GetPlaylistsResponse } from './models/GetPlaylistsResponse'; export type { GetRolesResponse } from './models/GetRolesResponse'; export type { GetSubscriptionRequest } from './models/GetSubscriptionRequest'; export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse'; +export type { GetTaskRequest } from './models/GetTaskRequest'; +export type { GetTaskResponse } from './models/GetTaskResponse'; export type { GetUsersResponse } from './models/GetUsersResponse'; export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest'; export type { inline_response_200_15 } from './models/inline_response_200_15'; @@ -71,6 +74,7 @@ export type { LoginResponse } from './models/LoginResponse'; export type { Playlist } from './models/Playlist'; export type { RegisterRequest } from './models/RegisterRequest'; export type { RegisterResponse } from './models/RegisterResponse'; +export { Schedule } from './models/Schedule'; export type { SetConfigRequest } from './models/SetConfigRequest'; export type { SharingToggle } from './models/SharingToggle'; export type { SubscribeRequest } from './models/SubscribeRequest'; @@ -79,6 +83,7 @@ export type { Subscription } from './models/Subscription'; export type { SubscriptionRequestData } from './models/SubscriptionRequestData'; export type { SuccessObject } from './models/SuccessObject'; export type { TableInfo } from './models/TableInfo'; +export type { Task } from './models/Task'; export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest'; export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse'; export type { TransferDBRequest } from './models/TransferDBRequest'; @@ -93,6 +98,7 @@ export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentSt export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest'; export type { UpdaterStatus } from './models/UpdaterStatus'; export type { UpdateServerRequest } from './models/UpdateServerRequest'; +export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest'; export type { UpdateUserRequest } from './models/UpdateUserRequest'; export type { User } from './models/User'; export { UserPermission } from './models/UserPermission'; diff --git a/src/api-types/models/DownloadFileRequest.ts b/src/api-types/models/DownloadFileRequest.ts index 31ba393..a874a8c 100644 --- a/src/api-types/models/DownloadFileRequest.ts +++ b/src/api-types/models/DownloadFileRequest.ts @@ -2,6 +2,7 @@ /* tslint:disable */ /* eslint-disable */ +import { FileType } from './FileType'; export interface DownloadFileRequest { uid?: string; @@ -9,5 +10,5 @@ export interface DownloadFileRequest { sub_id?: string; playlist_id?: string; url?: string; - type?: string; + type?: FileType; } \ No newline at end of file diff --git a/src/api-types/models/GetAllTasksResponse.ts b/src/api-types/models/GetAllTasksResponse.ts new file mode 100644 index 0000000..221d44a --- /dev/null +++ b/src/api-types/models/GetAllTasksResponse.ts @@ -0,0 +1,9 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import { Task } from './Task'; + +export interface GetAllTasksResponse { + tasks?: Array; +} \ No newline at end of file diff --git a/src/api-types/models/GetFullTwitchChatRequest.ts b/src/api-types/models/GetFullTwitchChatRequest.ts index 6d6a3a6..64a043d 100644 --- a/src/api-types/models/GetFullTwitchChatRequest.ts +++ b/src/api-types/models/GetFullTwitchChatRequest.ts @@ -15,8 +15,5 @@ export interface GetFullTwitchChatRequest { * User UID */ uuid?: string; - /** - * Subscription - */ sub?: Subscription; } \ No newline at end of file diff --git a/src/api-types/models/GetTaskRequest.ts b/src/api-types/models/GetTaskRequest.ts new file mode 100644 index 0000000..655a69f --- /dev/null +++ b/src/api-types/models/GetTaskRequest.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + + +export interface GetTaskRequest { + task_key: string; +} \ No newline at end of file diff --git a/src/api-types/models/GetTaskResponse.ts b/src/api-types/models/GetTaskResponse.ts new file mode 100644 index 0000000..7f11c6e --- /dev/null +++ b/src/api-types/models/GetTaskResponse.ts @@ -0,0 +1,9 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import { Task } from './Task'; + +export interface GetTaskResponse { + task?: Task; +} \ No newline at end of file diff --git a/src/api-types/models/Schedule.ts b/src/api-types/models/Schedule.ts new file mode 100644 index 0000000..452202d --- /dev/null +++ b/src/api-types/models/Schedule.ts @@ -0,0 +1,24 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + + +export interface Schedule { + type: Schedule.type; + data: { +dayOfWeek?: Array, +hour?: number, +minute?: number, +timestamp?: number, +}; +} + +export namespace Schedule { + + export enum type { + TIMESTAMP = 'timestamp', + RECURRING = 'recurring', + } + + +} \ No newline at end of file diff --git a/src/api-types/models/Task.ts b/src/api-types/models/Task.ts new file mode 100644 index 0000000..95c864a --- /dev/null +++ b/src/api-types/models/Task.ts @@ -0,0 +1,15 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + + +export interface Task { + key: string; + last_ran: number; + last_confirmed: number; + running: boolean; + confirming: boolean; + data: any; + error: string; + schedule: any; +} \ No newline at end of file diff --git a/src/api-types/models/UpdateTaskScheduleRequest.ts b/src/api-types/models/UpdateTaskScheduleRequest.ts new file mode 100644 index 0000000..b9e61d6 --- /dev/null +++ b/src/api-types/models/UpdateTaskScheduleRequest.ts @@ -0,0 +1,10 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import { Schedule } from './Schedule'; + +export interface UpdateTaskScheduleRequest { + task_key: string; + new_schedule: Schedule; +} \ No newline at end of file diff --git a/src/app/app-routing.module.ts b/src/app/app-routing.module.ts index 8620fcb..ab9c609 100644 --- a/src/app/app-routing.module.ts +++ b/src/app/app-routing.module.ts @@ -8,6 +8,7 @@ import { PostsService } from './posts.services'; import { LoginComponent } from './components/login/login.component'; import { DownloadsComponent } from './components/downloads/downloads.component'; import { SettingsComponent } from './settings/settings.component'; +import { TasksComponent } from './components/tasks/tasks.component'; const routes: Routes = [ { path: 'home', component: MainComponent, canActivate: [PostsService] }, @@ -17,6 +18,7 @@ const routes: Routes = [ { path: 'settings', component: SettingsComponent, canActivate: [PostsService] }, { path: 'login', component: LoginComponent }, { path: 'downloads', component: DownloadsComponent, canActivate: [PostsService] }, + { path: 'tasks', component: TasksComponent, canActivate: [PostsService] }, { path: '', redirectTo: '/home', pathMatch: 'full' } ]; diff --git a/src/app/app.component.html b/src/app/app.component.html index df61ad2..f1270c7 100644 --- a/src/app/app.component.html +++ b/src/app/app.component.html @@ -44,6 +44,7 @@ Login Subscriptions Downloads + Tasks Settings diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 7b26e63..0da23ee 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -28,6 +28,7 @@ import { MatTabsModule } from '@angular/material/tabs'; import { MatPaginatorModule } from '@angular/material/paginator'; import { MatSortModule } from '@angular/material/sort'; import { MatTableModule } from '@angular/material/table'; +import { MatDatepickerModule } from '@angular/material/datepicker'; import { DragDropModule } from '@angular/cdk/drag-drop'; import { ClipboardModule } from '@angular/cdk/clipboard'; import { TextFieldModule } from '@angular/cdk/text-field'; @@ -87,6 +88,8 @@ import { LinkifyPipe, SeeMoreComponent } from './components/see-more/see-more.co import { H401Interceptor } from './http.interceptor'; import { ConcurrentStreamComponent } from './components/concurrent-stream/concurrent-stream.component'; import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-button.component'; +import { TasksComponent } from './components/tasks/tasks.component'; +import { UpdateTaskScheduleDialogComponent } from './dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component'; registerLocaleData(es, 'es'); @@ -135,7 +138,9 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible TwitchChatComponent, SeeMoreComponent, ConcurrentStreamComponent, - SkipAdButtonComponent + SkipAdButtonComponent, + TasksComponent, + UpdateTaskScheduleDialogComponent ], imports: [ CommonModule, @@ -171,6 +176,7 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible MatPaginatorModule, MatSortModule, MatTableModule, + MatDatepickerModule, MatChipsModule, DragDropModule, ClipboardModule, diff --git a/src/app/components/tasks/tasks.component.html b/src/app/components/tasks/tasks.component.html new file mode 100644 index 0000000..556ded4 --- /dev/null +++ b/src/app/components/tasks/tasks.component.html @@ -0,0 +1,77 @@ +
+
+ + + + Title + + + {{element.title}} + + + + + + + Last ran + + {{element.last_ran*1000 | date: 'short'}} + N/A + + + + + + Last confirmed + + {{element.last_confirmed*1000 | date: 'short'}} + N/A + + + + + + Status + + + + Scheduled for  + {{element.next_invocation | date: 'short'}}repeat + + + Not scheduled + + + + + + + Actions + +
+ + + + + +
+ + + + + + + + + +
+
+ +
+

No tasks available!

+
\ No newline at end of file diff --git a/src/app/components/tasks/tasks.component.scss b/src/app/components/tasks/tasks.component.scss new file mode 100644 index 0000000..ed84df9 --- /dev/null +++ b/src/app/components/tasks/tasks.component.scss @@ -0,0 +1,32 @@ +mat-header-cell, mat-cell { + justify-content: center; +} + +.one-line { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.icon-button-spinner { + position: absolute; + top: 7px; + left: 6px; +} + +.downloads-action-button-div { + margin-top: 10px; + margin-left: 5px; +} + +.rounded-top { + border-radius: 16px 16px 0px 0px !important; +} + +.rounded-bottom { + border-radius: 0px 0px 16px 16px !important; +} + +.rounded { + border-radius: 16px 16px 16px 16px !important; +} \ No newline at end of file diff --git a/src/app/components/tasks/tasks.component.spec.ts b/src/app/components/tasks/tasks.component.spec.ts new file mode 100644 index 0000000..d4ab7a5 --- /dev/null +++ b/src/app/components/tasks/tasks.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { TasksComponent } from './tasks.component'; + +describe('TasksComponent', () => { + let component: TasksComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ TasksComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(TasksComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/components/tasks/tasks.component.ts b/src/app/components/tasks/tasks.component.ts new file mode 100644 index 0000000..a294b5b --- /dev/null +++ b/src/app/components/tasks/tasks.component.ts @@ -0,0 +1,109 @@ +import { Component, OnInit, ViewChild } from '@angular/core'; +import { MatDialog } from '@angular/material/dialog'; +import { MatPaginator } from '@angular/material/paginator'; +import { MatSort } from '@angular/material/sort'; +import { MatTableDataSource } from '@angular/material/table'; +import { UpdateTaskScheduleDialogComponent } from 'app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component'; +import { PostsService } from 'app/posts.services'; + +@Component({ + selector: 'app-tasks', + templateUrl: './tasks.component.html', + styleUrls: ['./tasks.component.scss'] +}) +export class TasksComponent implements OnInit { + + interval_id = null; + tasks_check_interval = 1500; + tasks = null; + tasks_retrieved = false; + + displayedColumns: string[] = ['title', 'last_ran', 'last_confirmed', 'status', 'actions']; + dataSource = null; + + @ViewChild(MatPaginator) paginator: MatPaginator; + @ViewChild(MatSort) sort: MatSort; + + constructor(private postsService: PostsService, private dialog: MatDialog) { } + + ngOnInit(): void { + if (this.postsService.initialized) { + this.getTasksRecurring(); + } else { + this.postsService.service_initialized.subscribe(init => { + if (init) { + this.getTasksRecurring(); + } + }); + } + } + + ngOnDestroy(): void { + if (this.interval_id) { clearInterval(this.interval_id) } + } + + getTasksRecurring(): void { + this.getTasks(); + this.interval_id = setInterval(() => { + this.getTasks(); + }, this.tasks_check_interval); + } + + getTasks(): void { + this.postsService.getTasks().subscribe(res => { + if (this.tasks) { + if (JSON.stringify(this.tasks) === JSON.stringify(res['tasks'])) return; + for (const task of res['tasks']) { + const task_index = this.tasks.map(t => t.key).indexOf(task['key']); + this.tasks[task_index] = task; + } + this.dataSource = new MatTableDataSource(this.tasks); + } else { + this.tasks = res['tasks']; + this.dataSource = new MatTableDataSource(this.tasks); + this.dataSource.paginator = this.paginator; + this.dataSource.sort = this.sort; + } + }); + } + + runTask(task_key: string): void { + this.postsService.runTask(task_key).subscribe(res => { + this.getTasks(); + }); + } + + confirmTask(task_key: string): void { + this.postsService.confirmTask(task_key).subscribe(res => { + this.getTasks(); + }); + } + + scheduleTask(task: any): void { + // open dialog + const dialogRef = this.dialog.open(UpdateTaskScheduleDialogComponent, { + data: { + task: task + } + }); + dialogRef.afterClosed().subscribe(schedule => { + if (schedule || schedule === null) { + this.postsService.updateTaskSchedule(task['key'], schedule).subscribe(res => { + this.getTasks(); + console.log(res); + }); + } + }); + } + +} + +export interface Task { + key: string; + title: string; + last_ran: number; + last_confirmed: number; + running: boolean; + confirming: boolean; + data: unknown; +} \ No newline at end of file diff --git a/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.html b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.html new file mode 100644 index 0000000..7782e15 --- /dev/null +++ b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.html @@ -0,0 +1,53 @@ +

Update task schedule

+ + +
+
+
+ Enabled +
+
+ Recurring +
+
+ + + Weekly + Daily + + +
+
+ + Choose a date + + + + +
+
+ + + M + T + W + T + F + S + S + +
+
+ + Time + + +
+
+
+ + + + + + \ No newline at end of file diff --git a/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.scss b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.scss new file mode 100644 index 0000000..e69de29 diff --git a/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.spec.ts b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.spec.ts new file mode 100644 index 0000000..fe52670 --- /dev/null +++ b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { UpdateTaskScheduleDialogComponent } from './update-task-schedule-dialog.component'; + +describe('UpdateTaskScheduleDialogComponent', () => { + let component: UpdateTaskScheduleDialogComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ UpdateTaskScheduleDialogComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(UpdateTaskScheduleDialogComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts new file mode 100644 index 0000000..ef61cf4 --- /dev/null +++ b/src/app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component.ts @@ -0,0 +1,83 @@ +import { Component, Inject, OnInit } from '@angular/core'; +import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { Schedule } from 'api-types'; +import { PostsService } from 'app/posts.services'; + +@Component({ + selector: 'app-update-task-schedule-dialog', + templateUrl: './update-task-schedule-dialog.component.html', + styleUrls: ['./update-task-schedule-dialog.component.scss'] +}) +export class UpdateTaskScheduleDialogComponent implements OnInit { + + enabled = true; + recurring = false; + days_of_week = []; + interval = 'daily'; + time = null; + date = null; + today = new Date(); + + constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef, private postsService: PostsService) { + this.processTask(this.data.task); + this.postsService.getTask(this.data.task.key).subscribe(res => { + this.processTask(res['task']); + }); + } + + ngOnInit(): void { + } + + processTask(task) { + if (!task['schedule']) { + this.enabled = false; + return; + } + + const schedule: Schedule = task['schedule']; + + this.recurring = schedule['type'] === Schedule.type.RECURRING; + + if (this.recurring) { + this.time = `${schedule['data']['hour']}:${schedule['data']['minute']}`; + + if (schedule['data']['dayOfWeek']) { + this.days_of_week = schedule['data']['dayOfWeek']; + this.interval = 'weekly'; + } else { + this.interval = 'daily'; + } + } else { + const schedule_date = new Date(schedule['data']['timestamp']); + this.time = `${schedule_date.getHours()}:${schedule_date.getMinutes()}` + this.date = schedule_date; + } + } + + updateTaskSchedule(): void { + if (!this.enabled) { + this.dialogRef.close(null); + return; + } + + if (!this.time) { + // needs time! + } + + const hours = parseInt(this.time.split(':')[0]); + const minutes = parseInt(this.time.split(':')[1]); + + const schedule: Schedule = {type: this.recurring ? Schedule.type.RECURRING : Schedule.type.TIMESTAMP, data: null}; + if (this.recurring) { + schedule['data'] = {hour: hours, minute: minutes}; + if (this.interval === 'weekly') { + schedule['data']['dayOfWeek'] = this.days_of_week; + } + } else { + this.date.setHours(hours, minutes); + console.log(this.date); + schedule['data'] = {timestamp: this.date.getTime()}; + } + this.dialogRef.close(schedule); + } +} diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 5fe3936..1a4e0ca 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -318,7 +318,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { const type = this.playlist[0].type; const url = this.playlist[0].url; this.downloading = true; - this.postsService.downloadFileFromServer(this.uid, this.uuid, this.sub_id, url, type).subscribe(res => { + this.postsService.downloadFileFromServer(this.uid, this.uuid, this.sub_id, url, type as FileType).subscribe(res => { this.downloading = false; const blob: Blob = res; saveAs(blob, filename + ext); diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index f157d47..923210a 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -90,6 +90,9 @@ import { DBInfoResponse, GetFileFormatsRequest, GetFileFormatsResponse, + GetTaskRequest, + GetTaskResponse, + UpdateTaskScheduleRequest, } from '../api-types'; import { isoLangs } from './settings/locales_list'; import { Title } from '@angular/platform-browser'; @@ -351,7 +354,7 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'getAllFiles', {sort: sort, range: range, text_search: text_search, file_type_filter: file_type_filter}, this.httpOptions); } - downloadFileFromServer(uid: string, uuid: string = null, sub_id: string = null, url: string = null, type: string = null) { + downloadFileFromServer(uid: string, uuid: string = null, sub_id: string = null, url: string = null, type: FileType = null) { const body: DownloadFileRequest = { uid: uid, uuid: uuid, @@ -544,38 +547,67 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'download', body, this.httpOptions); } - pauseDownload(download_uid) { - return this.http.post(this.path + 'pauseDownload', {download_uid: download_uid}, this.httpOptions); + pauseDownload(download_uid: string) { + const body: GetDownloadRequest = {download_uid: download_uid}; + return this.http.post(this.path + 'pauseDownload', body, this.httpOptions); } pauseAllDownloads() { return this.http.post(this.path + 'pauseAllDownloads', {}, this.httpOptions); } - resumeDownload(download_uid) { - return this.http.post(this.path + 'resumeDownload', {download_uid: download_uid}, this.httpOptions); + resumeDownload(download_uid: string) { + const body: GetDownloadRequest = {download_uid: download_uid}; + return this.http.post(this.path + 'resumeDownload', body, this.httpOptions); } resumeAllDownloads() { return this.http.post(this.path + 'resumeAllDownloads', {}, this.httpOptions); } - restartDownload(download_uid) { - return this.http.post(this.path + 'restartDownload', {download_uid: download_uid}, this.httpOptions); + restartDownload(download_uid: string) { + const body: GetDownloadRequest = {download_uid: download_uid}; + return this.http.post(this.path + 'restartDownload', body, this.httpOptions); } - cancelDownload(download_uid) { - return this.http.post(this.path + 'cancelDownload', {download_uid: download_uid}, this.httpOptions); + cancelDownload(download_uid: string) { + const body: GetDownloadRequest = {download_uid: download_uid}; + return this.http.post(this.path + 'cancelDownload', body, this.httpOptions); } - clearDownload(download_uid) { - return this.http.post(this.path + 'clearDownload', {download_uid: download_uid}, this.httpOptions); + clearDownload(download_uid: string) { + const body: GetDownloadRequest = {download_uid: download_uid}; + return this.http.post(this.path + 'clearDownload', body, this.httpOptions); } clearFinishedDownloads() { return this.http.post(this.path + 'clearFinishedDownloads', {}, this.httpOptions); } + getTasks() { + return this.http.post(this.path + 'getTasks', {}, this.httpOptions); + } + + getTask(task_key) { + const body: GetTaskRequest = {task_key: task_key}; + return this.http.post(this.path + 'getTask', body, this.httpOptions); + } + + runTask(task_key) { + const body: GetTaskRequest = {task_key: task_key}; + return this.http.post(this.path + 'runTask', body, this.httpOptions); + } + + confirmTask(task_key) { + const body: GetTaskRequest = {task_key: task_key}; + return this.http.post(this.path + 'confirmTask', body, this.httpOptions); + } + + updateTaskSchedule(task_key, schedule) { + const body: UpdateTaskScheduleRequest = {task_key: task_key, new_schedule: schedule}; + return this.http.post(this.path + 'updateTaskSchedule', body, this.httpOptions); + } + getVersionInfo() { return this.http.get(this.path + 'versionInfo', this.httpOptions); } From a28816364413b44d3cc5ece25997b4ca9f9b9037 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 21 Apr 2022 19:29:50 -0400 Subject: [PATCH 04/10] Added ability to backup remote DB Added ability to restore DB --- Public API v1.yaml | 91 +++++++++++++++++++ backend/app.js | 39 ++++++++ backend/db.js | 51 ++++++++++- backend/tasks.js | 7 +- backend/test/tests.js | 13 ++- backend/utils.js | 10 +- src/api-types/index.ts | 4 + src/api-types/models/DBBackup.ts | 21 +++++ src/api-types/models/GetDBBackupsResponse.ts | 9 ++ .../models/RestoreDBBackupRequest.ts | 8 ++ src/api-types/models/UpdateTaskDataRequest.ts | 9 ++ src/app/app.module.ts | 4 +- src/app/components/tasks/tasks.component.html | 28 ++++-- src/app/components/tasks/tasks.component.ts | 29 ++++++ .../restore-db-dialog.component.html | 29 ++++++ .../restore-db-dialog.component.scss | 0 .../restore-db-dialog.component.spec.ts | 25 +++++ .../restore-db-dialog.component.ts | 47 ++++++++++ src/app/posts.services.ts | 25 ++++- 19 files changed, 420 insertions(+), 29 deletions(-) create mode 100644 src/api-types/models/DBBackup.ts create mode 100644 src/api-types/models/GetDBBackupsResponse.ts create mode 100644 src/api-types/models/RestoreDBBackupRequest.ts create mode 100644 src/api-types/models/UpdateTaskDataRequest.ts create mode 100644 src/app/dialogs/restore-db-dialog/restore-db-dialog.component.html create mode 100644 src/app/dialogs/restore-db-dialog/restore-db-dialog.component.scss create mode 100644 src/app/dialogs/restore-db-dialog/restore-db-dialog.component.spec.ts create mode 100644 src/app/dialogs/restore-db-dialog/restore-db-dialog.component.ts diff --git a/Public API v1.yaml b/Public API v1.yaml index fc63ef8..5be8869 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -947,6 +947,54 @@ paths: application/json: schema: $ref: '#/components/schemas/UpdateTaskScheduleRequest' + /api/updateTaskData: + post: + summary: Updates task data + operationId: post-api-update-task-data + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/UpdateTaskDataRequest' + /api/getDBBackups: + post: + summary: Get database backups + operationId: post-api-get-database-backups + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/GetDBBackupsResponse' + requestBody: + content: + application/json: + schema: + type: object + /api/restoreDBBackup: + post: + summary: Restore database backup + operationId: post-api-restore-database-backup + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/RestoreDBBackupRequest' /api/auth/login: post: summary: Login @@ -1524,6 +1572,16 @@ components: required: - task_key - new_schedule + UpdateTaskDataRequest: + type: object + properties: + task_key: + type: string + new_data: + type: object + required: + - task_key + - new_data GetTaskResponse: type: object properties: @@ -1536,6 +1594,20 @@ components: type: array items: $ref: '#/components/schemas/Task' + GetDBBackupsResponse: + type: object + properties: + tasks: + type: array + items: + $ref: '#/components/schemas/DBBackup' + RestoreDBBackupRequest: + type: object + required: + - file_name + properties: + file_name: + type: string GetMp3sResponse: required: - mp3s @@ -2328,6 +2400,25 @@ components: type: number timestamp: type: number + DBBackup: + required: + - name + - timestamp + - size + - source + type: object + properties: + name: + type: string + timestamp: + type: number + size: + type: number + source: + type: string + enum: + - local + - remote SubscriptionRequestData: required: - id diff --git a/backend/app.js b/backend/app.js index 8aa2696..f596b8e 100644 --- a/backend/app.js +++ b/backend/app.js @@ -1924,6 +1924,45 @@ app.post('/api/updateTaskSchedule', optionalJwt, async (req, res) => { res.send({success: true}); }); +app.post('/api/updateTaskData', optionalJwt, async (req, res) => { + const task_key = req.body.task_key; + const new_data = req.body.new_data; + + const success = await db_api.updateRecord('tasks', {key: task_key}, {data: new_data}); + + res.send({success: success}); +}); + +app.post('/api/getDBBackups', optionalJwt, async (req, res) => { + const backup_dir = path.join('appdata', 'db_backup'); + const db_backups = []; + + const candidate_backups = await utils.recFindByExt(backup_dir, 'bak', null, [], false); + for (let i = 0; i < candidate_backups.length; i++) { + const candidate_backup = candidate_backups[i]; + + // must have specific format + if (candidate_backup.split('.').length - 1 !== 4) continue; + + const candidate_backup_path = candidate_backup; + const stats = fs.statSync(candidate_backup_path); + + db_backups.push({ name: path.basename(candidate_backup), timestamp: parseInt(candidate_backup.split('.')[2]), size: stats.size, source: candidate_backup.includes('local') ? 'local' : 'remote' }); + } + + db_backups.sort((a,b) => b.timestamp - a.timestamp); + + res.send({db_backups: db_backups}); +}); + +app.post('/api/restoreDBBackup', optionalJwt, async (req, res) => { + const file_name = req.body.file_name; + + const success = await db_api.restoreDB(file_name); + + res.send({success: success}); +}); + // logs management app.post('/api/logs', optionalJwt, async function(req, res) { diff --git a/backend/db.js b/backend/db.js index 07dc95d..591b92b 100644 --- a/backend/db.js +++ b/backend/db.js @@ -987,6 +987,52 @@ const createDownloadsRecords = (downloads) => { return new_downloads; } +exports.backupDB = async () => { + const backup_dir = path.join('appdata', 'db_backup'); + fs.ensureDirSync(backup_dir); + const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`; + const path_to_backups = path.join(backup_dir, backup_file_name); + + logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`); + + const table_to_records = {}; + for (let i = 0; i < tables_list.length; i++) { + const table = tables_list[i]; + table_to_records[table] = await exports.getRecords(table); + } + + fs.writeJsonSync(path_to_backups, table_to_records); + + return backup_file_name; +} + +exports.restoreDB = async (file_name) => { + const path_to_backup = path.join('appdata', 'db_backup', file_name); + + logger.debug('Reading database backup file.'); + const table_to_records = fs.readJSONSync(path_to_backup); + + if (!table_to_records) { + logger.error(`Failed to restore DB! Backup file '${path_to_backup}' could not be read.`); + return false; + } + + logger.debug('Clearing database.'); + await exports.removeAllRecords(); + + logger.debug('Database cleared! Beginning restore.'); + let success = true; + for (let i = 0; i < tables_list.length; i++) { + const table = tables_list[i]; + if (!table_to_records[table] || table_to_records[table].length === 0) continue; + success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]); + } + + logger.debug('Restore finished!'); + + return success; +} + exports.transferDB = async (local_to_remote) => { const table_to_records = {}; for (let i = 0; i < tables_list.length; i++) { @@ -996,9 +1042,8 @@ exports.transferDB = async (local_to_remote) => { using_local_db = !local_to_remote; if (local_to_remote) { - // backup local DB - logger.debug('Backup up Local DB...'); - await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`); + logger.debug('Backup up DB...'); + await exports.backupDB(); const db_connected = await exports.connectToDB(5, true); if (!db_connected) { logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.'); diff --git a/backend/tasks.js b/backend/tasks.js index c66b23a..9fdbad3 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -7,8 +7,8 @@ const scheduler = require('node-schedule'); const TASKS = { backup_local_db: { - run: utils.backupLocalDB, - title: 'Backup Local DB', + run: db_api.backupDB, + title: 'Backup DB', job: null }, missing_files_check: { @@ -81,7 +81,8 @@ const setupTasks = async () => { confirming: false, data: null, error: null, - schedule: null + schedule: null, + options: {} }); } else { // reset task if necessary diff --git a/backend/test/tests.js b/backend/test/tests.js index 0c26fdc..9ae95a8 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -70,6 +70,17 @@ describe('Database', async function() { const success = await db_api.getRecord('test', {test: 'test'}); assert(success); }); + + it('Restore db', async function() { + const db_stats = await db_api.getDBStats(); + + const file_name = await db_api.backupDB(); + await db_api.restoreDB(file_name); + + const new_db_stats = await db_api.getDBStats(); + + assert(JSON.stringify(db_stats), JSON.stringify(new_db_stats)); + }); }); describe('Export', function() { @@ -393,7 +404,7 @@ describe('Tasks', function() { await tasks_api.initialize(); }); - it('Backup local db', async function() { + it('Backup db', async function() { const backups_original = await utils.recFindByExt('appdata', 'bak'); const original_length = backups_original.length; await tasks_api.executeTask('backup_local_db'); diff --git a/backend/utils.js b/backend/utils.js index 340a214..4f94388 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -266,13 +266,7 @@ function getCurrentDownloader() { return details_json['downloader']; } -async function backupLocalDB() { - const path_to_backups = path.join('appdata', 'db_backup'); - fs.ensureDir(path_to_backups); - await fs.copyFile('appdata/local_db.json', path.join(path_to_backups, `local_db.json.${Date.now()/1000}.bak`)); -} - -async function recFindByExt(base,ext,files,result) +async function recFindByExt(base, ext, files, result, recursive = true) { files = files || (await fs.readdir(base)) result = result || [] @@ -281,6 +275,7 @@ async function recFindByExt(base,ext,files,result) var newbase = path.join(base,file) if ( (await fs.stat(newbase)).isDirectory() ) { + if (!recursive) continue; result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result) } else @@ -396,7 +391,6 @@ module.exports = { getMatchingCategoryFiles: getMatchingCategoryFiles, addUIDsToCategory: addUIDsToCategory, getCurrentDownloader: getCurrentDownloader, - backupLocalDB: backupLocalDB, recFindByExt: recFindByExt, removeFileExtension: removeFileExtension, formatDateString: formatDateString, diff --git a/src/api-types/index.ts b/src/api-types/index.ts index 93d611f..43f0f06 100644 --- a/src/api-types/index.ts +++ b/src/api-types/index.ts @@ -21,6 +21,7 @@ export type { CreatePlaylistRequest } from './models/CreatePlaylistRequest'; export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse'; export type { CropFileSettings } from './models/CropFileSettings'; export type { DatabaseFile } from './models/DatabaseFile'; +export { DBBackup } from './models/DBBackup'; export type { DBInfoResponse } from './models/DBInfoResponse'; export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest'; export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request'; @@ -45,6 +46,7 @@ export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse'; export type { GetAllFilesResponse } from './models/GetAllFilesResponse'; export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse'; export type { GetAllTasksResponse } from './models/GetAllTasksResponse'; +export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse'; export type { GetDownloadRequest } from './models/GetDownloadRequest'; export type { GetDownloadResponse } from './models/GetDownloadResponse'; export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest'; @@ -74,6 +76,7 @@ export type { LoginResponse } from './models/LoginResponse'; export type { Playlist } from './models/Playlist'; export type { RegisterRequest } from './models/RegisterRequest'; export type { RegisterResponse } from './models/RegisterResponse'; +export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest'; export { Schedule } from './models/Schedule'; export type { SetConfigRequest } from './models/SetConfigRequest'; export type { SharingToggle } from './models/SharingToggle'; @@ -98,6 +101,7 @@ export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentSt export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest'; export type { UpdaterStatus } from './models/UpdaterStatus'; export type { UpdateServerRequest } from './models/UpdateServerRequest'; +export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest'; export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest'; export type { UpdateUserRequest } from './models/UpdateUserRequest'; export type { User } from './models/User'; diff --git a/src/api-types/models/DBBackup.ts b/src/api-types/models/DBBackup.ts new file mode 100644 index 0000000..710c591 --- /dev/null +++ b/src/api-types/models/DBBackup.ts @@ -0,0 +1,21 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + + +export interface DBBackup { + name: string; + timestamp: number; + size: number; + source: DBBackup.source; +} + +export namespace DBBackup { + + export enum source { + LOCAL = 'local', + REMOTE = 'remote', + } + + +} \ No newline at end of file diff --git a/src/api-types/models/GetDBBackupsResponse.ts b/src/api-types/models/GetDBBackupsResponse.ts new file mode 100644 index 0000000..b02ced9 --- /dev/null +++ b/src/api-types/models/GetDBBackupsResponse.ts @@ -0,0 +1,9 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import { DBBackup } from './DBBackup'; + +export interface GetDBBackupsResponse { + tasks?: Array; +} \ No newline at end of file diff --git a/src/api-types/models/RestoreDBBackupRequest.ts b/src/api-types/models/RestoreDBBackupRequest.ts new file mode 100644 index 0000000..b5fde8a --- /dev/null +++ b/src/api-types/models/RestoreDBBackupRequest.ts @@ -0,0 +1,8 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + + +export interface RestoreDBBackupRequest { + file_name: string; +} \ No newline at end of file diff --git a/src/api-types/models/UpdateTaskDataRequest.ts b/src/api-types/models/UpdateTaskDataRequest.ts new file mode 100644 index 0000000..7768eaa --- /dev/null +++ b/src/api-types/models/UpdateTaskDataRequest.ts @@ -0,0 +1,9 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + + +export interface UpdateTaskDataRequest { + task_key: string; + new_data: any; +} \ No newline at end of file diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 0da23ee..3d6d040 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -90,6 +90,7 @@ import { ConcurrentStreamComponent } from './components/concurrent-stream/concur import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-button.component'; import { TasksComponent } from './components/tasks/tasks.component'; import { UpdateTaskScheduleDialogComponent } from './dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component'; +import { RestoreDbDialogComponent } from './dialogs/restore-db-dialog/restore-db-dialog.component'; registerLocaleData(es, 'es'); @@ -140,7 +141,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible ConcurrentStreamComponent, SkipAdButtonComponent, TasksComponent, - UpdateTaskScheduleDialogComponent + UpdateTaskScheduleDialogComponent, + RestoreDbDialogComponent ], imports: [ CommonModule, diff --git a/src/app/components/tasks/tasks.component.html b/src/app/components/tasks/tasks.component.html index 556ded4..9baa503 100644 --- a/src/app/components/tasks/tasks.component.html +++ b/src/app/components/tasks/tasks.component.html @@ -48,15 +48,23 @@ Actions -
- - - - - +
+
+
+ + + +
+
+ +
+
+ +
+
@@ -70,6 +78,8 @@ aria-label="Select page of tasks">
+ +
diff --git a/src/app/components/tasks/tasks.component.ts b/src/app/components/tasks/tasks.component.ts index a294b5b..cc86909 100644 --- a/src/app/components/tasks/tasks.component.ts +++ b/src/app/components/tasks/tasks.component.ts @@ -3,6 +3,7 @@ import { MatDialog } from '@angular/material/dialog'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { MatTableDataSource } from '@angular/material/table'; +import { RestoreDbDialogComponent } from 'app/dialogs/restore-db-dialog/restore-db-dialog.component'; import { UpdateTaskScheduleDialogComponent } from 'app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component'; import { PostsService } from 'app/posts.services'; @@ -21,6 +22,8 @@ export class TasksComponent implements OnInit { displayedColumns: string[] = ['title', 'last_ran', 'last_confirmed', 'status', 'actions']; dataSource = null; + db_backups = []; + @ViewChild(MatPaginator) paginator: MatPaginator; @ViewChild(MatSort) sort: MatSort; @@ -70,12 +73,23 @@ export class TasksComponent implements OnInit { runTask(task_key: string): void { this.postsService.runTask(task_key).subscribe(res => { this.getTasks(); + this.getDBBackups(); + if (res['success']) this.postsService.openSnackBar($localize`Successfully ran task!`); + else this.postsService.openSnackBar($localize`Failed to run task!`); + }, err => { + this.postsService.openSnackBar($localize`Failed to run task!`); + console.error(err); }); } confirmTask(task_key: string): void { this.postsService.confirmTask(task_key).subscribe(res => { this.getTasks(); + if (res['success']) this.postsService.openSnackBar($localize`Successfully confirmed task!`); + else this.postsService.openSnackBar($localize`Failed to confirm task!`); + }, err => { + this.postsService.openSnackBar($localize`Failed to confirm task!`); + console.error(err); }); } @@ -96,6 +110,21 @@ export class TasksComponent implements OnInit { }); } + getDBBackups(): void { + this.postsService.getDBBackups().subscribe(res => { + this.db_backups = res['db_backups']; + }); + } + + openRestoreDBBackupDialog(): void { + this.dialog.open(RestoreDbDialogComponent, { + data: { + db_backups: this.db_backups + }, + width: '80vw' + }) + } + } export interface Task { diff --git a/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.html b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.html new file mode 100644 index 0000000..abcb1e7 --- /dev/null +++ b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.html @@ -0,0 +1,29 @@ +

Restore DB from backup

+ + + + +
+
+
+ {{db_backup.timestamp*1000 | date: 'short'}} +
+
+ {{(db_backup.size/1000).toFixed(2)}} kB +
+
+ {{db_backup.source}} +
+
+
+ + + + + + + + + \ No newline at end of file diff --git a/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.scss b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.scss new file mode 100644 index 0000000..e69de29 diff --git a/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.spec.ts b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.spec.ts new file mode 100644 index 0000000..422e482 --- /dev/null +++ b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { RestoreDbDialogComponent } from './restore-db-dialog.component'; + +describe('RestoreDbDialogComponent', () => { + let component: RestoreDbDialogComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ RestoreDbDialogComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(RestoreDbDialogComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.ts b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.ts new file mode 100644 index 0000000..77df326 --- /dev/null +++ b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.ts @@ -0,0 +1,47 @@ +import { Component, Inject, OnInit } from '@angular/core'; +import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog'; +import { PostsService } from 'app/posts.services'; + +@Component({ + selector: 'app-restore-db-dialog', + templateUrl: './restore-db-dialog.component.html', + styleUrls: ['./restore-db-dialog.component.scss'] +}) +export class RestoreDbDialogComponent implements OnInit { + + db_backups = []; + selected_backup = null; + restoring = false; + + constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef, private postsService: PostsService) { + if (this.data?.db_backups) { + this.db_backups = this.data.db_backups; + } + + this.getDBBackups(); + } + + ngOnInit(): void { + } + + getDBBackups(): void { + this.postsService.getDBBackups().subscribe(res => { + this.db_backups = res['db_backups']; + }); + } + + restoreClicked(): void { + if (this.selected_backup.length !== 1) return; + this.postsService.restoreDBBackup(this.selected_backup[0]).subscribe(res => { + if (res['success']) { + this.postsService.openSnackBar('Database successfully restored!'); + } else { + this.postsService.openSnackBar('Failed to restore database! See logs for more info.'); + } + }, err => { + this.postsService.openSnackBar('Failed to restore database! See browser console for more info.'); + console.error(err); + }); + } + +} diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 923210a..68f9890 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -93,6 +93,9 @@ import { GetTaskRequest, GetTaskResponse, UpdateTaskScheduleRequest, + UpdateTaskDataRequest, + RestoreDBBackupRequest, + Schedule, } from '../api-types'; import { isoLangs } from './settings/locales_list'; import { Title } from '@angular/platform-browser'; @@ -588,26 +591,40 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'getTasks', {}, this.httpOptions); } - getTask(task_key) { + getTask(task_key: string) { const body: GetTaskRequest = {task_key: task_key}; return this.http.post(this.path + 'getTask', body, this.httpOptions); } - runTask(task_key) { + runTask(task_key: string) { const body: GetTaskRequest = {task_key: task_key}; return this.http.post(this.path + 'runTask', body, this.httpOptions); } - confirmTask(task_key) { + confirmTask(task_key: string) { const body: GetTaskRequest = {task_key: task_key}; return this.http.post(this.path + 'confirmTask', body, this.httpOptions); } - updateTaskSchedule(task_key, schedule) { + updateTaskSchedule(task_key: string, schedule: Schedule) { const body: UpdateTaskScheduleRequest = {task_key: task_key, new_schedule: schedule}; return this.http.post(this.path + 'updateTaskSchedule', body, this.httpOptions); } + updateTaskData(task_key: string, data: any) { + const body: UpdateTaskDataRequest = {task_key: task_key, new_data: data}; + return this.http.post(this.path + 'updateTaskData', body, this.httpOptions); + } + + getDBBackups() { + return this.http.post(this.path + 'getDBBackups', {}, this.httpOptions); + } + + restoreDBBackup(file_name: string) { + const body: RestoreDBBackupRequest = {file_name: file_name}; + return this.http.post(this.path + 'restoreDBBackup', body, this.httpOptions); + } + getVersionInfo() { return this.http.get(this.path + 'versionInfo', this.httpOptions); } From d2d125743e56b242cd628b2b3f2c01c6c032ce16 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 21 Apr 2022 19:56:09 -0400 Subject: [PATCH 05/10] Fixed issue where restoring a DB backup would cause backup_local_db task to be stuck running Slightly updated tasks UI --- backend/tasks.js | 3 ++- src/app/components/tasks/tasks.component.html | 8 ++++---- .../restore-db-dialog/restore-db-dialog.component.ts | 4 ++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/tasks.js b/backend/tasks.js index 9fdbad3..423f85a 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -110,7 +110,8 @@ exports.executeTask = async (task_key) => { exports.executeRun = async (task_key) => { logger.verbose(`Running task ${task_key}`); - await db_api.updateRecord('tasks', {key: task_key}, {running: true}); + // don't set running to true when backup up DB as it will be stick "running" if restored + if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true}); const data = await TASKS[task_key].run(); await db_api.updateRecord('tasks', {key: task_key}, {data: data, last_ran: Date.now()/1000, running: false}); logger.verbose(`Finished running task ${task_key}`); diff --git a/src/app/components/tasks/tasks.component.html b/src/app/components/tasks/tasks.component.html index 9baa503..f883f54 100644 --- a/src/app/components/tasks/tasks.component.html +++ b/src/app/components/tasks/tasks.component.html @@ -34,9 +34,9 @@ Status - + Scheduled for  - {{element.next_invocation | date: 'short'}}repeat + {{element.next_invocation | date: 'short'}}repeat Not scheduled @@ -58,10 +58,10 @@
-
+
-
+
diff --git a/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.ts b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.ts index 77df326..204a8c1 100644 --- a/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.ts +++ b/src/app/dialogs/restore-db-dialog/restore-db-dialog.component.ts @@ -31,14 +31,18 @@ export class RestoreDbDialogComponent implements OnInit { } restoreClicked(): void { + this.restoring = true; if (this.selected_backup.length !== 1) return; this.postsService.restoreDBBackup(this.selected_backup[0]).subscribe(res => { + this.restoring = false; if (res['success']) { this.postsService.openSnackBar('Database successfully restored!'); + this.dialogRef.close(); } else { this.postsService.openSnackBar('Failed to restore database! See logs for more info.'); } }, err => { + this.restoring = false; this.postsService.openSnackBar('Failed to restore database! See browser console for more info.'); console.error(err); }); From b53d9c9710dc1d1d86ecb22e34ddaa5abba8cbb0 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 21 Apr 2022 22:04:45 -0400 Subject: [PATCH 06/10] Added ability to reset tasks Refactored youtube-dl updating and added youtube-dl update task --- Public API v1.yaml | 32 +++ backend/app.js | 197 ++---------------- backend/tasks.js | 23 +- backend/utils.js | 69 +++++- backend/youtube-dl.js | 127 +++++++++++ src/app/components/tasks/tasks.component.html | 14 +- src/app/components/tasks/tasks.component.ts | 26 +++ src/app/posts.services.ts | 4 + 8 files changed, 298 insertions(+), 194 deletions(-) create mode 100644 backend/youtube-dl.js diff --git a/Public API v1.yaml b/Public API v1.yaml index 5be8869..0ca76da 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -881,10 +881,30 @@ paths: application/json: schema: $ref: '#/components/schemas/GetAllTasksResponse' + /api/resetTasks: + post: + tags: + - tasks + summary: Resets all tasks + operationId: post-api-reset-tasks + requestBody: + content: + application/json: + schema: + type: object + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' security: - Auth query parameter: [] /api/runTask: post: + tags: + - tasks summary: Runs one task operationId: post-api-run-task responses: @@ -901,6 +921,8 @@ paths: $ref: '#/components/schemas/GetTaskRequest' /api/confirmTask: post: + tags: + - tasks summary: Confirms a task operationId: post-api-confirm-task responses: @@ -917,6 +939,8 @@ paths: $ref: '#/components/schemas/GetTaskRequest' /api/cancelTask: post: + tags: + - tasks summary: Cancels a task operationId: post-api-cancel-task responses: @@ -933,6 +957,8 @@ paths: $ref: '#/components/schemas/GetTaskRequest' /api/updateTaskSchedule: post: + tags: + - tasks summary: Updates task schedule operationId: post-api-update-task-schedule responses: @@ -949,6 +975,8 @@ paths: $ref: '#/components/schemas/UpdateTaskScheduleRequest' /api/updateTaskData: post: + tags: + - tasks summary: Updates task data operationId: post-api-update-task-data responses: @@ -965,6 +993,8 @@ paths: $ref: '#/components/schemas/UpdateTaskDataRequest' /api/getDBBackups: post: + tags: + - tasks summary: Get database backups operationId: post-api-get-database-backups responses: @@ -981,6 +1011,8 @@ paths: type: object /api/restoreDBBackup: post: + tags: + - tasks summary: Restore database backup operationId: post-api-restore-database-backup responses: diff --git a/backend/app.js b/backend/app.js index f596b8e..ae71c15 100644 --- a/backend/app.js +++ b/backend/app.js @@ -13,7 +13,6 @@ const unzipper = require('unzipper'); const db_api = require('./db'); const utils = require('./utils') const low = require('lowdb') -const ProgressBar = require('progress'); const fetch = require('node-fetch'); const URL = require('url').URL; const CONSTS = require('./consts') @@ -32,8 +31,7 @@ const tasks_api = require('./tasks'); const subscriptions_api = require('./subscriptions'); const categories_api = require('./categories'); const twitch_api = require('./twitch'); - -const is_windows = process.platform === 'win32'; +const youtubedl_api = require('./youtube-dl'); var app = express(); @@ -357,34 +355,6 @@ async function downloadReleaseFiles(tag) { }); } -// helper function to download file using fetch -async function fetchFile(url, path, file_label) { - var len = null; - const res = await fetch(url); - - len = parseInt(res.headers.get("Content-Length"), 10); - - var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, { - complete: '=', - incomplete: ' ', - width: 20, - total: len - }); - const fileStream = fs.createWriteStream(path); - await new Promise((resolve, reject) => { - res.body.pipe(fileStream); - res.body.on("error", (err) => { - reject(err); - }); - res.body.on('data', function (chunk) { - bar.tick(chunk.length); - }); - fileStream.on("finish", function() { - resolve(); - }); - }); - } - async function downloadReleaseZip(tag) { return new Promise(async resolve => { // get name of zip file, which depends on the version @@ -395,7 +365,7 @@ async function downloadReleaseZip(tag) { let output_path = path.join(__dirname, `youtubedl-material-release-${tag}.zip`); // download zip from release - await fetchFile(latest_zip_link, output_path, 'update ' + tag); + await utils.fetchFile(latest_zip_link, output_path, 'update ' + tag); resolve(true); }); @@ -708,156 +678,8 @@ async function getUrlInfos(url) { async function startYoutubeDL() { // auto update youtube-dl - await autoUpdateYoutubeDL(); -} - -// auto updates the underlying youtube-dl binary, not YoutubeDL-Material -async function autoUpdateYoutubeDL() { - const download_sources = { - 'youtube-dl': { - 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags', - 'func': downloadLatestYoutubeDLBinary - }, - 'youtube-dlc': { - 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags', - 'func': downloadLatestYoutubeDLCBinary - }, - 'yt-dlp': { - 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags', - 'func': downloadLatestYoutubeDLPBinary - } - } - return new Promise(async resolve => { - const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); - const tags_url = download_sources[default_downloader]['tags_url']; - // get current version - let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); - if (!current_app_details_exists) { - logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader}); - } - let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); - let current_version = current_app_details['version']; - let current_downloader = current_app_details['downloader']; - let stored_binary_path = current_app_details['path']; - if (!stored_binary_path || typeof stored_binary_path !== 'string') { - // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); - const guessed_base_path = 'node_modules/youtube-dl/bin/'; - const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); - if (fs.existsSync(guessed_file_path)) { - stored_binary_path = guessed_file_path; - // logger.info('INFO: Guess successful! Update process continuing...') - } else { - logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); - resolve(false); - return; - } - } - - // got version, now let's check the latest version from the youtube-dl API - - - fetch(tags_url, {method: 'Get'}) - .then(async res => res.json()) - .then(async (json) => { - // check if the versions are different - if (!json || !json[0]) { - logger.error(`Failed to check ${default_downloader} version for an update.`) - resolve(false); - return false; - } - const latest_update_version = json[0]['name']; - if (current_version !== latest_update_version || default_downloader !== current_downloader) { - // versions different or different downloader is being used, download new update - logger.info(`Found new update for ${default_downloader}. Updating binary...`); - try { - await checkExistsWithTimeout(stored_binary_path, 10000); - } catch(e) { - logger.error(`Failed to update ${default_downloader} - ${e}`); - } - - await download_sources[default_downloader]['func'](latest_update_version); - - resolve(true); - } else { - resolve(false); - } - }) - .catch(err => { - logger.error(`Failed to check ${default_downloader} version for an update.`) - logger.error(err); - resolve(false); - return false; - }); - }); -} - -async function downloadLatestYoutubeDLBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await fetchFile(download_url, output_path, `youtube-dl ${new_version}`); - - updateDetailsJSON(new_version, 'youtube-dl'); -} - -async function downloadLatestYoutubeDLCBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await fetchFile(download_url, output_path, `youtube-dlc ${new_version}`); - - updateDetailsJSON(new_version, 'youtube-dlc'); -} - -async function downloadLatestYoutubeDLPBinary(new_version) { - const file_ext = is_windows ? '.exe' : ''; - - const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`; - const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; - - await fetchFile(download_url, output_path, `yt-dlp ${new_version}`); - - updateDetailsJSON(new_version, 'yt-dlp'); -} - -function updateDetailsJSON(new_version, downloader) { - const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); - if (new_version) details_json['version'] = new_version; - details_json['downloader'] = downloader; - fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); -} - -async function checkExistsWithTimeout(filePath, timeout) { - return new Promise(function (resolve, reject) { - - var timer = setTimeout(function () { - if (watcher) watcher.close(); - reject(new Error('File did not exists and was not created during the timeout.')); - }, timeout); - - fs.access(filePath, fs.constants.R_OK, function (err) { - if (!err) { - clearTimeout(timer); - if (watcher) watcher.close(); - resolve(); - } - }); - - var dir = path.dirname(filePath); - var basename = path.basename(filePath); - var watcher = fs.watch(dir, function (eventType, filename) { - if (eventType === 'rename' && filename === basename) { - clearTimeout(timer); - if (watcher) watcher.close(); - resolve(); - } - }); - }); + const update_available = await youtubedl_api.checkForYoutubeDLUpdate(); + if (update_available) await youtubedl_api.updateYoutubeDL(update_available); } app.use(function(req, res, next) { @@ -1886,6 +1708,17 @@ app.post('/api/getTasks', optionalJwt, async (req, res) => { res.send({tasks: tasks}); }); +app.post('/api/resetTasks', optionalJwt, async (req, res) => { + const tasks_keys = Object.keys(tasks_api.TASKS); + for (let i = 0; i < tasks_keys.length; i++) { + const task_key = tasks_keys[i]; + tasks_api.TASKS[task_key]['job'] = null; + } + await db_api.removeAllRecords('tasks'); + await tasks_api.setupTasks(); + res.send({success: true}); +}); + app.post('/api/getTask', optionalJwt, async (req, res) => { const task_key = req.body.task_key; const task = await db_api.getRecord('tasks', {key: task_key}); diff --git a/backend/tasks.js b/backend/tasks.js index 423f85a..2014da3 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -1,5 +1,5 @@ -const utils = require('./utils'); const db_api = require('./db'); +const youtubedl_api = require('./youtube-dl'); const fs = require('fs-extra'); const logger = require('./logger'); @@ -27,6 +27,12 @@ const TASKS = { confirm: removeDuplicates, title: 'Find duplicate files in DB', job: null + }, + youtubedl_update_check: { + run: youtubedl_api.checkForYoutubeDLUpdate, + confirm: youtubedl_api.updateYoutubeDL, + title: 'Update youtube-dl', + job: null } } @@ -52,20 +58,22 @@ function scheduleJob(task_key, schedule) { return; } + // remove schedule if it's a one-time task + if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null}); // we're just "running" the task, any confirmation should be user-initiated exports.executeRun(task_key); }); } if (db_api.database_initialized) { - setupTasks(); + exports.setupTasks(); } else { db_api.database_initialized_bs.subscribe(init => { - if (init) setupTasks(); + if (init) exports.setupTasks(); }); } -const setupTasks = async () => { +exports.setupTasks = async () => { const tasks_keys = Object.keys(TASKS); for (let i = 0; i < tasks_keys.length; i++) { const task_key = tasks_keys[i]; @@ -90,6 +98,11 @@ const setupTasks = async () => { // schedule task and save job if (task_in_db['schedule']) { + // prevent timestamp schedules from being set to the past + if (task_in_db['schedule']['type'] === 'timestamp' && task_in_db['schedule']['data']['timestamp'] < Date.now()) { + await db_api.updateRecord('tasks', {key: task_key}, {schedule: null}); + continue; + } TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']); } } @@ -113,7 +126,7 @@ exports.executeRun = async (task_key) => { // don't set running to true when backup up DB as it will be stick "running" if restored if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true}); const data = await TASKS[task_key].run(); - await db_api.updateRecord('tasks', {key: task_key}, {data: data, last_ran: Date.now()/1000, running: false}); + await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false}); logger.verbose(`Finished running task ${task_key}`); } diff --git a/backend/utils.js b/backend/utils.js index 4f94388..0271ed0 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -1,10 +1,13 @@ -const fs = require('fs-extra') -const path = require('path') +const fs = require('fs-extra'); +const path = require('path'); const ffmpeg = require('fluent-ffmpeg'); +const archiver = require('archiver'); +const fetch = require('node-fetch'); +const ProgressBar = require('progress'); + const config_api = require('./config'); const logger = require('./logger'); -const CONSTS = require('./consts') -const archiver = require('archiver'); +const CONSTS = require('./consts'); const is_windows = process.platform === 'win32'; @@ -356,6 +359,62 @@ async function cropFile(file_path, start, end, ext) { }); } +async function checkExistsWithTimeout(filePath, timeout) { + return new Promise(function (resolve, reject) { + + var timer = setTimeout(function () { + if (watcher) watcher.close(); + reject(new Error('File did not exists and was not created during the timeout.')); + }, timeout); + + fs.access(filePath, fs.constants.R_OK, function (err) { + if (!err) { + clearTimeout(timer); + if (watcher) watcher.close(); + resolve(); + } + }); + + var dir = path.dirname(filePath); + var basename = path.basename(filePath); + var watcher = fs.watch(dir, function (eventType, filename) { + if (eventType === 'rename' && filename === basename) { + clearTimeout(timer); + if (watcher) watcher.close(); + resolve(); + } + }); + }); +} + +// helper function to download file using fetch +async function fetchFile(url, path, file_label) { + var len = null; + const res = await fetch(url); + + len = parseInt(res.headers.get("Content-Length"), 10); + + var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, { + complete: '=', + incomplete: ' ', + width: 20, + total: len + }); + const fileStream = fs.createWriteStream(path); + await new Promise((resolve, reject) => { + res.body.pipe(fileStream); + res.body.on("error", (err) => { + reject(err); + }); + res.body.on('data', function (chunk) { + bar.tick(chunk.length); + }); + fileStream.on("finish", function() { + resolve(); + }); + }); +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { @@ -397,5 +456,7 @@ module.exports = { cropFile: cropFile, createEdgeNGrams: createEdgeNGrams, wait: wait, + checkExistsWithTimeout: checkExistsWithTimeout, + fetchFile: fetchFile, File: File } diff --git a/backend/youtube-dl.js b/backend/youtube-dl.js new file mode 100644 index 0000000..80432fb --- /dev/null +++ b/backend/youtube-dl.js @@ -0,0 +1,127 @@ +const fs = require('fs-extra'); +const fetch = require('node-fetch'); + +const logger = require('./logger'); +const utils = require('./utils'); +const CONSTS = require('./consts'); +const config_api = require('./config.js'); + +const is_windows = process.platform === 'win32'; + +const download_sources = { + 'youtube-dl': { + 'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags', + 'func': downloadLatestYoutubeDLBinary + }, + 'youtube-dlc': { + 'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags', + 'func': downloadLatestYoutubeDLCBinary + }, + 'yt-dlp': { + 'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags', + 'func': downloadLatestYoutubeDLPBinary + } +} + +exports.checkForYoutubeDLUpdate = async () => { + return new Promise(async resolve => { + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); + const tags_url = download_sources[default_downloader]['tags_url']; + // get current version + let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH); + if (!current_app_details_exists) { + logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`); + fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader}); + } + let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH)); + let current_version = current_app_details['version']; + let current_downloader = current_app_details['downloader']; + let stored_binary_path = current_app_details['path']; + if (!stored_binary_path || typeof stored_binary_path !== 'string') { + // logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`); + const guessed_base_path = 'node_modules/youtube-dl/bin/'; + const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : ''); + if (fs.existsSync(guessed_file_path)) { + stored_binary_path = guessed_file_path; + // logger.info('INFO: Guess successful! Update process continuing...') + } else { + logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`); + resolve(null); + return; + } + } + + // got version, now let's check the latest version from the youtube-dl API + + + fetch(tags_url, {method: 'Get'}) + .then(async res => res.json()) + .then(async (json) => { + // check if the versions are different + if (!json || !json[0]) { + logger.error(`Failed to check ${default_downloader} version for an update.`) + resolve(null); + return; + } + const latest_update_version = json[0]['name']; + if (current_version !== latest_update_version || default_downloader !== current_downloader) { + // versions different or different downloader is being used, download new update + resolve(latest_update_version); + } else { + resolve(null); + } + return; + }) + .catch(err => { + logger.error(`Failed to check ${default_downloader} version for an update.`) + logger.error(err); + resolve(null); + return; + }); + }); +} + +exports.updateYoutubeDL = async (latest_update_version) => { + const default_downloader = config_api.getConfigItem('ytdl_default_downloader'); + await download_sources[default_downloader]['func'](latest_update_version); +} + +async function downloadLatestYoutubeDLBinary(new_version) { + const file_ext = is_windows ? '.exe' : ''; + + const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`; + const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; + + await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`); + + updateDetailsJSON(new_version, 'youtube-dl'); +} + +async function downloadLatestYoutubeDLCBinary(new_version) { + const file_ext = is_windows ? '.exe' : ''; + + const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`; + const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; + + await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`); + + updateDetailsJSON(new_version, 'youtube-dlc'); +} + +async function downloadLatestYoutubeDLPBinary(new_version) { + const file_ext = is_windows ? '.exe' : ''; + + const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`; + const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`; + + await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`); + + updateDetailsJSON(new_version, 'yt-dlp'); +} + +function updateDetailsJSON(new_version, downloader) { + const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH); + if (new_version) details_json['version'] = new_version; + details_json['downloader'] = downloader; + fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json); +} diff --git a/src/app/components/tasks/tasks.component.html b/src/app/components/tasks/tasks.component.html index f883f54..662e3be 100644 --- a/src/app/components/tasks/tasks.component.html +++ b/src/app/components/tasks/tasks.component.html @@ -50,11 +50,18 @@
-
+
@@ -80,6 +87,7 @@
+
diff --git a/src/app/components/tasks/tasks.component.ts b/src/app/components/tasks/tasks.component.ts index cc86909..8fec372 100644 --- a/src/app/components/tasks/tasks.component.ts +++ b/src/app/components/tasks/tasks.component.ts @@ -3,6 +3,7 @@ import { MatDialog } from '@angular/material/dialog'; import { MatPaginator } from '@angular/material/paginator'; import { MatSort } from '@angular/material/sort'; import { MatTableDataSource } from '@angular/material/table'; +import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component'; import { RestoreDbDialogComponent } from 'app/dialogs/restore-db-dialog/restore-db-dialog.component'; import { UpdateTaskScheduleDialogComponent } from 'app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component'; import { PostsService } from 'app/posts.services'; @@ -125,6 +126,31 @@ export class TasksComponent implements OnInit { }) } + resetTasks(): void { + const dialogRef = this.dialog.open(ConfirmDialogComponent, { + data: { + dialogTitle: $localize`Reset tasks`, + dialogText: $localize`Would you like to reset your tasks? All your schedules will be removed as well.`, + submitText: $localize`Reset`, + warnSubmitColor: true + } + }); + dialogRef.afterClosed().subscribe(confirmed => { + if (confirmed) { + this.postsService.resetTasks().subscribe(res => { + if (res['success']) { + this.postsService.openSnackBar($localize`Tasks successfully reset!`); + } else { + this.postsService.openSnackBar($localize`Failed to reset tasks!`); + } + }, err => { + this.postsService.openSnackBar($localize`Failed to reset tasks!`); + console.error(err); + }); + } + }); + } + } export interface Task { diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 68f9890..6df00e5 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -591,6 +591,10 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'getTasks', {}, this.httpOptions); } + resetTasks() { + return this.http.post(this.path + 'resetTasks', {}, this.httpOptions); + } + getTask(task_key: string) { const body: GetTaskRequest = {task_key: task_key}; return this.http.post(this.path + 'getTask', body, this.httpOptions); From 7a4d91cea0b04188993378e0e165e0137fd67112 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Thu, 21 Apr 2022 22:08:47 -0400 Subject: [PATCH 07/10] Removed import of unregistered files on startup as it's a task now --- backend/app.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/app.js b/backend/app.js index ae71c15..2882663 100644 --- a/backend/app.js +++ b/backend/app.js @@ -536,8 +536,6 @@ async function loadConfig() { watchSubscriptionsInterval(); } - db_api.importUnregisteredFiles(); - // start the server here startServer(); From df687263c52903893a9a97017ab411e56f3dca3b Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Fri, 22 Apr 2022 00:35:58 -0400 Subject: [PATCH 08/10] Fixed bug that prevented files from being downloaded from the server. Reverted some changes from #528 as they are not needed --- src/app/player/player.component.html | 3 +-- src/app/player/player.component.ts | 18 ------------------ src/app/posts.services.ts | 9 +++------ 3 files changed, 4 insertions(+), 26 deletions(-) diff --git a/src/app/player/player.component.html b/src/app/player/player.component.html index 74c0b1b..0bf510f 100644 --- a/src/app/player/player.component.html +++ b/src/app/player/player.component.html @@ -33,8 +33,7 @@ - - + diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 1a4e0ca..2bf0d35 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -328,24 +328,6 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { }); } - downloadVideo() { - const filename = this.currentItem.label; - const ext = (this.currentItem.type === 'audio/mp3') ? '.mp3' : '.mp4'; - // const type = this.currentItem.type; - const url = this.currentItem.src; - this.downloading = true; - this.http.get(url, { - responseType: 'blob' - }).subscribe(res => { - const blob: Blob = res; - this.downloading = false; - saveAs(blob, filename + ext); - }, err => { - console.log(err); - this.downloading = false; - }) - } - playlistPostCreationHandler(playlistID) { // changes the route without moving from the current view or // triggering a navigation event diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 6df00e5..5674fec 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -357,15 +357,12 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'getAllFiles', {sort: sort, range: range, text_search: text_search, file_type_filter: file_type_filter}, this.httpOptions); } - downloadFileFromServer(uid: string, uuid: string = null, sub_id: string = null, url: string = null, type: FileType = null) { + downloadFileFromServer(uid: string, uuid: string = null) { const body: DownloadFileRequest = { uid: uid, - uuid: uuid, - sub_id: sub_id, - url: url, - type: type + uuid: uuid }; - return this.http.post(this.path + 'downloadFile', body, {responseType: 'blob', params: this.httpOptions.params}); + return this.http.post(this.path + 'downloadFileFromServer', body, {responseType: 'blob', params: this.httpOptions.params}); } getFullTwitchChat(id, type, uuid = null, sub = null) { From 2c3813f302a6c876c257aa09bddfbaf6ec3166c3 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Fri, 22 Apr 2022 00:40:34 -0400 Subject: [PATCH 09/10] Removed httpclient import from player component --- src/app/player/player.component.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 2bf0d35..7ad108e 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -9,7 +9,6 @@ import { ShareMediaDialogComponent } from '../dialogs/share-media-dialog/share-m import { FileType } from '../../api-types'; import { TwitchChatComponent } from 'app/components/twitch-chat/twitch-chat.component'; import { VideoInfoDialogComponent } from 'app/dialogs/video-info-dialog/video-info-dialog.component'; -import { HttpClient, HttpParams } from '@angular/common/http'; export interface IMedia { title: string; @@ -110,7 +109,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { } constructor(public postsService: PostsService, private route: ActivatedRoute, private dialog: MatDialog, private router: Router, - public snackBar: MatSnackBar, private cdr: ChangeDetectorRef, private http: HttpClient) { + public snackBar: MatSnackBar, private cdr: ChangeDetectorRef) { } processConfig() { From d32df84e3a2b46fe2a18411387c52bc1ce8d350a Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Fri, 22 Apr 2022 15:25:26 -0400 Subject: [PATCH 10/10] Fixed build error --- src/app/player/player.component.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 7ad108e..9c82d1a 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -314,10 +314,8 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { downloadFile() { const filename = this.playlist[0].title; const ext = (this.playlist[0].type === 'audio/mp3') ? '.mp3' : '.mp4'; - const type = this.playlist[0].type; - const url = this.playlist[0].url; this.downloading = true; - this.postsService.downloadFileFromServer(this.uid, this.uuid, this.sub_id, url, type as FileType).subscribe(res => { + this.postsService.downloadFileFromServer(this.uid, this.uuid).subscribe(res => { this.downloading = false; const blob: Blob = res; saveAs(blob, filename + ext);