aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added scheduler for tasks

Browse Source
tasks-and-maintenence-page
Isaac Abadi 3 years ago
parent 2b1771d30d
commit 5b4d4d5f81
5 changed files with 181 additions and 5 deletions
Show Stats Download Patch File Download Diff File

4
backend/db.js
Unescape Escape View File

@ -54,6 +54,10 @@ const tables = {
name: 'download_queue',
primary_key: 'uid'
},
tasks: {
name: 'tasks',
primary_key: 'key'
},
test: {
name: 'test'
}

102
backend/package-lock.json generated
Unescape Escape View File

@ -599,6 +599,15 @@
}
}
},
"call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
"integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
"requires": {
"function-bind": "^1.1.1",
"get-intrinsic": "^1.0.2"
}
},
"camelcase": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
@ -909,6 +918,15 @@
"readable-stream": "^3.4.0"
}
},
"cron-parser": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-3.5.0.tgz",
"integrity": "sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==",
"requires": {
"is-nan": "^1.3.2",
"luxon": "^1.26.0"
}
},
"cross-spawn": {
"version": "7.0.1",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz",
@ -968,6 +986,15 @@
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz",
"integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ=="
},
"define-properties": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz",
"integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==",
"requires": {
"has-property-descriptors": "^1.0.0",
"object-keys": "^1.1.1"
}
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@ -1358,11 +1385,26 @@
"rimraf": "2"
}
},
"function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
},
"get-caller-file": {
"version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
},
"get-intrinsic": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
"integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==",
"requires": {
"function-bind": "^1.1.1",
"has": "^1.0.3",
"has-symbols": "^1.0.1"
}
},
"get-stream": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
@ -1460,11 +1502,32 @@
"har-schema": "^2.0.0"
}
},
"has": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
"integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
"requires": {
"function-bind": "^1.1.1"
}
},
"has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
},
"has-property-descriptors": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz",
"integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==",
"requires": {
"get-intrinsic": "^1.1.1"
}
},
"has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
},
"has-yarn": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz",
@ -1635,6 +1698,15 @@
"is-path-inside": "^3.0.1"
}
},
"is-nan": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz",
"integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==",
"requires": {
"call-bind": "^1.0.0",
"define-properties": "^1.1.3"
}
},
"is-npm": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz",
@ -2034,6 +2106,11 @@
}
}
},
"long-timeout": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz",
"integrity": "sha1-lyHXiLR+C8taJMLivuGg2lXatRQ="
},
"lowdb": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/lowdb/-/lowdb-1.0.0.tgz",
@ -2051,6 +2128,11 @@
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
"integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA=="
},
"luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
},
"make-dir": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
@ -2407,6 +2489,16 @@
"iconv-lite": "^0.4.15"
}
},
"node-schedule": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz",
"integrity": "sha512-nl4JTiZ7ZQDc97MmpTq9BQjYhq7gOtoh7SiPH069gBFBj0PzD8HI7zyFs6rzqL8Y5tTiEEYLxgtbx034YPrbyQ==",
"requires": {
"cron-parser": "^3.5.0",
"long-timeout": "0.1.1",
"sorted-array-functions": "^1.3.0"
}
},
"nodemon": {
"version": "2.0.7",
"resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz",
@ -2475,6 +2567,11 @@
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"object-keys": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
"integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
},
"on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@ -3053,6 +3150,11 @@
"is-arrayish": "^0.3.1"
}
},
"sorted-array-functions": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
"integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA=="
},
"sparse-bitfield": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",

1
backend/package.json
Unescape Escape View File

@ -52,6 +52,7 @@
"multer": "^1.4.2",
"node-fetch": "^2.6.7",
"node-id3": "^0.1.14",
"node-schedule": "^2.1.0",
"nodemon": "^2.0.7",
"passport": "^0.4.1",
"passport-http": "^0.3.0",

50
backend/tasks.js
Unescape Escape View File

@ -3,34 +3,53 @@ const db_api = require('./db');
const fs = require('fs-extra');
const logger = require('./logger');
const scheduler = require('node-schedule');
const TASKS = {
backup_local_db: {
run: utils.backupLocalDB,
title: 'Backup Local DB',
job: null
},
missing_files_check: {
run: checkForMissingFiles,
confirm: deleteMissingFiles,
title: 'Missing files check'
title: 'Missing files check',
job: null
},
missing_db_records: {
run: db_api.importUnregisteredFiles,
title: 'Import missing DB records'
title: 'Import missing DB records',
job: null
},
duplicate_files_check: {
run: checkForDuplicateFiles,
confirm: removeDuplicates,
title: 'Find duplicate files in DB'
title: 'Find duplicate files in DB',
job: null
}
}
function scheduleJob(task_key, schedule) {
return scheduler.scheduleJob(schedule, async () => {
const task_state = await db_api.getRecord('tasks', {key: task_key});
if (task_state['running'] || task_state['confirming']) {
logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
return;
}
// we're just "running" the task, any confirmation should be user-initiated
exports.executeRun(task_key);
});
}
exports.initialize = async () => {
const tasks_keys = Object.keys(TASKS);
for (let i = 0; i < tasks_keys.length; i++) {
const task_key = tasks_keys[i];
const task_in_db = await db_api.getRecord('tasks', {key: task_key});
if (!task_in_db) {
// insert task into table if missing
await db_api.insertRecordIntoTable('tasks', {
key: task_key,
last_ran: null,
@ -38,8 +57,17 @@ exports.initialize = async () => {
running: false,
confirming: false,
data: null,
error: null
error: null,
schedule: null
});
} else {
// reset task if necessary
await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
// schedule task and save job
if (task_in_db['schedule']) {
TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']);
}
}
}
}
@ -73,6 +101,16 @@ exports.executeConfirm = async (task_key) => {
await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000});
}
exports.updateTaskSchedule = async (task_key, schedule) => {
await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
if (TASKS[task_key]['job']) {
TASKS[task_key]['job'].cancel();
}
if (schedule) {
TASKS[task_key]['job'] = scheduleJob(task_key, schedule);
}
}
// missing files check
async function checkForMissingFiles() {
@ -109,4 +147,6 @@ async function removeDuplicates(data) {
for (let i = 0; i < data['uids'].length; i++) {
await db_api.removeRecord('files', {uid: data['uids'][i]});
}
}
}
exports.TASKS = TASKS;

29
backend/test/tests.js
Unescape Escape View File

@ -382,6 +382,15 @@ describe('Tasks', function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('tasks');
const dummy_task = {
run: async () => { await utils.wait(500); return true; },
confirm: async () => { await utils.wait(500); return true; },
title: 'Dummy task',
job: null
};
tasks_api.TASKS['dummy_task'] = dummy_task;
await tasks_api.initialize();
});
it('Backup local db', async function() {
@ -438,4 +447,24 @@ describe('Tasks', function() {
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
});
it('Schedule and cancel task', async function() {
const today_4_hours = new Date();
today_4_hours.setHours(today_4_hours.getHours() + 4);
await tasks_api.updateTaskSchedule('dummy_task', today_4_hours);
assert(!!tasks_api.TASKS['dummy_task']['job'], true);
await tasks_api.updateTaskSchedule('dummy_task', null);
assert(!!tasks_api.TASKS['dummy_task']['job'], false);
});
it('Schedule and run task', async function() {
this.timeout(5000);
const today_1_second = new Date();
today_1_second.setSeconds(today_1_second.getSeconds() + 1);
await tasks_api.updateTaskSchedule('dummy_task', today_1_second);
assert(!!tasks_api.TASKS['dummy_task']['job'], true);
await utils.wait(2000);
const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
assert(dummy_task_obj['data'], true);
});
});
Write Preview
Loading…
Cancel
Save