From 5b4d4d5f813a78bb254986ea39560a54534260d9 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Tue, 19 Apr 2022 22:29:41 -0400 Subject: [PATCH] Added scheduler for tasks --- backend/db.js | 4 ++ backend/package-lock.json | 102 ++++++++++++++++++++++++++++++++++++++ backend/package.json | 1 + backend/tasks.js | 50 +++++++++++++++++-- backend/test/tests.js | 29 +++++++++++ 5 files changed, 181 insertions(+), 5 deletions(-) diff --git a/backend/db.js b/backend/db.js index af51a4e..07dc95d 100644 --- a/backend/db.js +++ b/backend/db.js @@ -54,6 +54,10 @@ const tables = { name: 'download_queue', primary_key: 'uid' }, + tasks: { + name: 'tasks', + primary_key: 'key' + }, test: { name: 'test' } diff --git a/backend/package-lock.json b/backend/package-lock.json index d8d0e37..76daecd 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -599,6 +599,15 @@ } } }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -909,6 +918,15 @@ "readable-stream": "^3.4.0" } }, + "cron-parser": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-3.5.0.tgz", + "integrity": "sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==", + "requires": { + "is-nan": "^1.3.2", + "luxon": "^1.26.0" + } + }, "cross-spawn": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz", @@ -968,6 +986,15 @@ "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==" }, + "define-properties": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", + "requires": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -1358,11 +1385,26 @@ "rimraf": "2" } }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, + "get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, "get-stream": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", @@ -1460,11 +1502,32 @@ "har-schema": "^2.0.0" } }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "requires": { + "get-intrinsic": "^1.1.1" + } + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + }, "has-yarn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", @@ -1635,6 +1698,15 @@ "is-path-inside": "^3.0.1" } }, + "is-nan": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", + "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==", + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3" + } + }, "is-npm": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", @@ -2034,6 +2106,11 @@ } } }, + "long-timeout": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz", + "integrity": "sha1-lyHXiLR+C8taJMLivuGg2lXatRQ=" + }, "lowdb": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-1.0.0.tgz", @@ -2051,6 +2128,11 @@ "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" }, + "luxon": { + "version": "1.28.0", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", + "integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ==" + }, "make-dir": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", @@ -2407,6 +2489,16 @@ "iconv-lite": "^0.4.15" } }, + "node-schedule": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz", + "integrity": "sha512-nl4JTiZ7ZQDc97MmpTq9BQjYhq7gOtoh7SiPH069gBFBj0PzD8HI7zyFs6rzqL8Y5tTiEEYLxgtbx034YPrbyQ==", + "requires": { + "cron-parser": "^3.5.0", + "long-timeout": "0.1.1", + "sorted-array-functions": "^1.3.0" + } + }, "nodemon": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz", @@ -2475,6 +2567,11 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" }, + "object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -3053,6 +3150,11 @@ "is-arrayish": "^0.3.1" } }, + "sorted-array-functions": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz", + "integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA==" + }, "sparse-bitfield": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", diff --git a/backend/package.json b/backend/package.json index d30ded2..76db3a5 100644 --- a/backend/package.json +++ b/backend/package.json @@ -52,6 +52,7 @@ "multer": "^1.4.2", "node-fetch": "^2.6.7", "node-id3": "^0.1.14", + "node-schedule": "^2.1.0", "nodemon": "^2.0.7", "passport": "^0.4.1", "passport-http": "^0.3.0", diff --git a/backend/tasks.js b/backend/tasks.js index e928725..701cbff 100644 --- a/backend/tasks.js +++ b/backend/tasks.js @@ -3,34 +3,53 @@ const db_api = require('./db'); const fs = require('fs-extra'); const logger = require('./logger'); +const scheduler = require('node-schedule'); const TASKS = { backup_local_db: { run: utils.backupLocalDB, title: 'Backup Local DB', + job: null }, missing_files_check: { run: checkForMissingFiles, confirm: deleteMissingFiles, - title: 'Missing files check' + title: 'Missing files check', + job: null }, missing_db_records: { run: db_api.importUnregisteredFiles, - title: 'Import missing DB records' + title: 'Import missing DB records', + job: null }, duplicate_files_check: { run: checkForDuplicateFiles, confirm: removeDuplicates, - title: 'Find duplicate files in DB' + title: 'Find duplicate files in DB', + job: null } } +function scheduleJob(task_key, schedule) { + return scheduler.scheduleJob(schedule, async () => { + const task_state = await db_api.getRecord('tasks', {key: task_key}); + if (task_state['running'] || task_state['confirming']) { + logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`); + return; + } + + // we're just "running" the task, any confirmation should be user-initiated + exports.executeRun(task_key); + }); +} + exports.initialize = async () => { const tasks_keys = Object.keys(TASKS); for (let i = 0; i < tasks_keys.length; i++) { const task_key = tasks_keys[i]; const task_in_db = await db_api.getRecord('tasks', {key: task_key}); if (!task_in_db) { + // insert task into table if missing await db_api.insertRecordIntoTable('tasks', { key: task_key, last_ran: null, @@ -38,8 +57,17 @@ exports.initialize = async () => { running: false, confirming: false, data: null, - error: null + error: null, + schedule: null }); + } else { + // reset task if necessary + await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false}); + + // schedule task and save job + if (task_in_db['schedule']) { + TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']); + } } } } @@ -73,6 +101,16 @@ exports.executeConfirm = async (task_key) => { await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000}); } +exports.updateTaskSchedule = async (task_key, schedule) => { + await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule}); + if (TASKS[task_key]['job']) { + TASKS[task_key]['job'].cancel(); + } + if (schedule) { + TASKS[task_key]['job'] = scheduleJob(task_key, schedule); + } +} + // missing files check async function checkForMissingFiles() { @@ -109,4 +147,6 @@ async function removeDuplicates(data) { for (let i = 0; i < data['uids'].length; i++) { await db_api.removeRecord('files', {uid: data['uids'][i]}); } -} \ No newline at end of file +} + +exports.TASKS = TASKS; \ No newline at end of file diff --git a/backend/test/tests.js b/backend/test/tests.js index 7856f28..0c26fdc 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -382,6 +382,15 @@ describe('Tasks', function() { beforeEach(async function() { await db_api.connectToDB(); await db_api.removeAllRecords('tasks'); + + const dummy_task = { + run: async () => { await utils.wait(500); return true; }, + confirm: async () => { await utils.wait(500); return true; }, + title: 'Dummy task', + job: null + }; + tasks_api.TASKS['dummy_task'] = dummy_task; + await tasks_api.initialize(); }); it('Backup local db', async function() { @@ -438,4 +447,24 @@ describe('Tasks', function() { if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json'); if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4'); }); + + it('Schedule and cancel task', async function() { + const today_4_hours = new Date(); + today_4_hours.setHours(today_4_hours.getHours() + 4); + await tasks_api.updateTaskSchedule('dummy_task', today_4_hours); + assert(!!tasks_api.TASKS['dummy_task']['job'], true); + await tasks_api.updateTaskSchedule('dummy_task', null); + assert(!!tasks_api.TASKS['dummy_task']['job'], false); + }); + + it('Schedule and run task', async function() { + this.timeout(5000); + const today_1_second = new Date(); + today_1_second.setSeconds(today_1_second.getSeconds() + 1); + await tasks_api.updateTaskSchedule('dummy_task', today_1_second); + assert(!!tasks_api.TASKS['dummy_task']['job'], true); + await utils.wait(2000); + const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'}); + assert(dummy_task_obj['data'], true); + }); }); \ No newline at end of file