Merge pull request #554 from Tzahi12345/tasks-and-maintenence-page

Tasks and maintenence page
pull/555/head
Glassed Silver 3 years ago committed by GitHub
commit 8314ab8fce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -666,11 +666,367 @@ paths:
schema: schema:
$ref: '#/components/schemas/GetDownloadRequest' $ref: '#/components/schemas/GetDownloadRequest'
description: '' description: ''
description: "Gets a single download using its download_id and session_id. session_id is the device fingerprint. If none was provided at the time of download, then set session_id is 'undeclared'." description: "Gets a single download using its download_id."
security: security:
- Auth query parameter: [] - Auth query parameter: []
tags: tags:
- downloader - downloader
/api/pauseDownload:
post:
summary: Pauses one download
operationId: post-api-pause-download
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetDownloadRequest'
description: ''
description: "Pause a single download using its download_id."
security:
- Auth query parameter: []
tags:
- downloader
/api/pauseAllDownloads:
post:
tags:
- downloader
summary: Pauses all downloads
operationId: post-api-pause-all-downloads
requestBody:
content:
application/json:
schema:
type: object
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/resumeDownload:
post:
summary: Resume one download
operationId: post-api-resume-download
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetDownloadRequest'
description: ''
description: "Resume a single download using its download_id."
security:
- Auth query parameter: []
tags:
- downloader
/api/resumeAllDownloads:
post:
tags:
- downloader
summary: Resumes all downloads
operationId: post-api-resume-all-downloads
requestBody:
content:
application/json:
schema:
type: object
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/restartDownload:
post:
summary: Restart one download
operationId: post-api-restart-download
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetDownloadRequest'
description: ''
description: "Restart a single download using its download_id."
security:
- Auth query parameter: []
tags:
- downloader
/api/cancelDownload:
post:
summary: Cancel one download
operationId: post-api-cancel-download
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetDownloadRequest'
description: ''
description: "Cancel a single download using its download_id."
security:
- Auth query parameter: []
tags:
- downloader
/api/clearDownload:
post:
summary: Clear one download
operationId: post-api-clear-download
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetDownloadRequest'
description: ''
description: "Clears a single download from the downloaded list using its download_id."
security:
- Auth query parameter: []
tags:
- downloader
/api/clearFinishedDownloads:
post:
tags:
- downloader
summary: Clear finished downloads
operationId: post-api-clear-finished-downloads
requestBody:
content:
application/json:
schema:
type: object
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/getTask:
post:
summary: Get info for one task
operationId: post-api-get-task
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/GetTaskResponse'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetTaskRequest'
description: ''
description: "Gets a single task using its key."
security:
- Auth query parameter: []
tags:
- tasks
/api/getTasks:
post:
tags:
- tasks
summary: Get tasks
operationId: post-api-get-tasks
requestBody:
content:
application/json:
schema:
type: object
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/GetAllTasksResponse'
/api/resetTasks:
post:
tags:
- tasks
summary: Resets all tasks
operationId: post-api-reset-tasks
requestBody:
content:
application/json:
schema:
type: object
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
security:
- Auth query parameter: []
/api/runTask:
post:
tags:
- tasks
summary: Runs one task
operationId: post-api-run-task
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetTaskRequest'
/api/confirmTask:
post:
tags:
- tasks
summary: Confirms a task
operationId: post-api-confirm-task
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetTaskRequest'
/api/cancelTask:
post:
tags:
- tasks
summary: Cancels a task
operationId: post-api-cancel-task
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/GetTaskRequest'
/api/updateTaskSchedule:
post:
tags:
- tasks
summary: Updates task schedule
operationId: post-api-update-task-schedule
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateTaskScheduleRequest'
/api/updateTaskData:
post:
tags:
- tasks
summary: Updates task data
operationId: post-api-update-task-data
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/UpdateTaskDataRequest'
/api/getDBBackups:
post:
tags:
- tasks
summary: Get database backups
operationId: post-api-get-database-backups
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/GetDBBackupsResponse'
requestBody:
content:
application/json:
schema:
type: object
/api/restoreDBBackup:
post:
tags:
- tasks
summary: Restore database backup
operationId: post-api-restore-database-backup
responses:
'200':
description: OK
content:
application/json:
schema:
$ref: '#/components/schemas/SuccessObject'
requestBody:
content:
application/json:
schema:
$ref: '#/components/schemas/RestoreDBBackupRequest'
/api/auth/login: /api/auth/login:
post: post:
summary: Login summary: Login
@ -1231,6 +1587,59 @@ components:
type: array type: array
items: items:
$ref: '#/components/schemas/Download' $ref: '#/components/schemas/Download'
GetTaskRequest:
type: object
properties:
task_key:
type: string
required:
- task_key
UpdateTaskScheduleRequest:
type: object
properties:
task_key:
type: string
new_schedule:
$ref: '#/components/schemas/Schedule'
required:
- task_key
- new_schedule
UpdateTaskDataRequest:
type: object
properties:
task_key:
type: string
new_data:
type: object
required:
- task_key
- new_data
GetTaskResponse:
type: object
properties:
task:
$ref: '#/components/schemas/Task'
GetAllTasksResponse:
type: object
properties:
tasks:
type: array
items:
$ref: '#/components/schemas/Task'
GetDBBackupsResponse:
type: object
properties:
tasks:
type: array
items:
$ref: '#/components/schemas/DBBackup'
RestoreDBBackupRequest:
type: object
required:
- file_name
properties:
file_name:
type: string
GetMp3sResponse: GetMp3sResponse:
required: required:
- mp3s - mp3s
@ -1506,6 +1915,10 @@ components:
type: string type: string
playlist_id: playlist_id:
type: string type: string
url:
type: string
type:
$ref: '#/components/schemas/FileType'
DownloadArchiveRequest: DownloadArchiveRequest:
required: required:
- sub - sub
@ -1967,6 +2380,77 @@ components:
type: string type: string
sub_name: sub_name:
type: string type: string
Task:
required:
- key
- last_ran
- last_confirmed
- running
- confirming
- data
- error
- schedule
type: object
properties:
key:
type: string
last_ran:
type: number
last_confirmed:
type: number
running:
type: boolean
confirming:
type: boolean
data:
type: object
error:
type: string
schedule:
type: object
Schedule:
required:
- type
- data
type: object
properties:
type:
type: string
enum:
- timestamp
- recurring
data:
type: object
properties:
dayOfWeek:
type: array
items:
type: number
hour:
type: number
minute:
type: number
timestamp:
type: number
DBBackup:
required:
- name
- timestamp
- size
- source
type: object
properties:
name:
type: string
timestamp:
type: number
size:
type: number
source:
type: string
enum:
- local
- remote
SubscriptionRequestData: SubscriptionRequestData:
required: required:
- id - id

@ -13,7 +13,6 @@ const unzipper = require('unzipper');
const db_api = require('./db'); const db_api = require('./db');
const utils = require('./utils') const utils = require('./utils')
const low = require('lowdb') const low = require('lowdb')
const ProgressBar = require('progress');
const fetch = require('node-fetch'); const fetch = require('node-fetch');
const URL = require('url').URL; const URL = require('url').URL;
const CONSTS = require('./consts') const CONSTS = require('./consts')
@ -28,11 +27,11 @@ const youtubedl = require('youtube-dl');
const logger = require('./logger'); const logger = require('./logger');
const config_api = require('./config.js'); const config_api = require('./config.js');
const downloader_api = require('./downloader'); const downloader_api = require('./downloader');
const tasks_api = require('./tasks');
const subscriptions_api = require('./subscriptions'); const subscriptions_api = require('./subscriptions');
const categories_api = require('./categories'); const categories_api = require('./categories');
const twitch_api = require('./twitch'); const twitch_api = require('./twitch');
const youtubedl_api = require('./youtube-dl');
const is_windows = process.platform === 'win32';
var app = express(); var app = express();
@ -60,9 +59,6 @@ const admin_token = '4241b401-7236-493e-92b5-b72696b9d853';
config_api.initialize(); config_api.initialize();
db_api.initialize(db, users_db); db_api.initialize(db, users_db);
auth_api.initialize(db_api); auth_api.initialize(db_api);
downloader_api.initialize(db_api);
subscriptions_api.initialize(db_api, downloader_api);
categories_api.initialize(db_api);
// Set some defaults // Set some defaults
db.defaults( db.defaults(
@ -359,34 +355,6 @@ async function downloadReleaseFiles(tag) {
}); });
} }
// helper function to download file using fetch
async function fetchFile(url, path, file_label) {
var len = null;
const res = await fetch(url);
len = parseInt(res.headers.get("Content-Length"), 10);
var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
complete: '=',
incomplete: ' ',
width: 20,
total: len
});
const fileStream = fs.createWriteStream(path);
await new Promise((resolve, reject) => {
res.body.pipe(fileStream);
res.body.on("error", (err) => {
reject(err);
});
res.body.on('data', function (chunk) {
bar.tick(chunk.length);
});
fileStream.on("finish", function() {
resolve();
});
});
}
async function downloadReleaseZip(tag) { async function downloadReleaseZip(tag) {
return new Promise(async resolve => { return new Promise(async resolve => {
// get name of zip file, which depends on the version // get name of zip file, which depends on the version
@ -397,7 +365,7 @@ async function downloadReleaseZip(tag) {
let output_path = path.join(__dirname, `youtubedl-material-release-${tag}.zip`); let output_path = path.join(__dirname, `youtubedl-material-release-${tag}.zip`);
// download zip from release // download zip from release
await fetchFile(latest_zip_link, output_path, 'update ' + tag); await utils.fetchFile(latest_zip_link, output_path, 'update ' + tag);
resolve(true); resolve(true);
}); });
@ -568,8 +536,6 @@ async function loadConfig() {
watchSubscriptionsInterval(); watchSubscriptionsInterval();
} }
db_api.importUnregisteredFiles();
// start the server here // start the server here
startServer(); startServer();
@ -710,156 +676,8 @@ async function getUrlInfos(url) {
async function startYoutubeDL() { async function startYoutubeDL() {
// auto update youtube-dl // auto update youtube-dl
await autoUpdateYoutubeDL(); const update_available = await youtubedl_api.checkForYoutubeDLUpdate();
} if (update_available) await youtubedl_api.updateYoutubeDL(update_available);
// auto updates the underlying youtube-dl binary, not YoutubeDL-Material
async function autoUpdateYoutubeDL() {
const download_sources = {
'youtube-dl': {
'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
'func': downloadLatestYoutubeDLBinary
},
'youtube-dlc': {
'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
'func': downloadLatestYoutubeDLCBinary
},
'yt-dlp': {
'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
'func': downloadLatestYoutubeDLPBinary
}
}
return new Promise(async resolve => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
const tags_url = download_sources[default_downloader]['tags_url'];
// get current version
let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
if (!current_app_details_exists) {
logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader});
}
let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
let current_version = current_app_details['version'];
let current_downloader = current_app_details['downloader'];
let stored_binary_path = current_app_details['path'];
if (!stored_binary_path || typeof stored_binary_path !== 'string') {
// logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
const guessed_base_path = 'node_modules/youtube-dl/bin/';
const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
if (fs.existsSync(guessed_file_path)) {
stored_binary_path = guessed_file_path;
// logger.info('INFO: Guess successful! Update process continuing...')
} else {
logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
resolve(false);
return;
}
}
// got version, now let's check the latest version from the youtube-dl API
fetch(tags_url, {method: 'Get'})
.then(async res => res.json())
.then(async (json) => {
// check if the versions are different
if (!json || !json[0]) {
logger.error(`Failed to check ${default_downloader} version for an update.`)
resolve(false);
return false;
}
const latest_update_version = json[0]['name'];
if (current_version !== latest_update_version || default_downloader !== current_downloader) {
// versions different or different downloader is being used, download new update
logger.info(`Found new update for ${default_downloader}. Updating binary...`);
try {
await checkExistsWithTimeout(stored_binary_path, 10000);
} catch(e) {
logger.error(`Failed to update ${default_downloader} - ${e}`);
}
await download_sources[default_downloader]['func'](latest_update_version);
resolve(true);
} else {
resolve(false);
}
})
.catch(err => {
logger.error(`Failed to check ${default_downloader} version for an update.`)
logger.error(err);
resolve(false);
return false;
});
});
}
async function downloadLatestYoutubeDLBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dl');
}
async function downloadLatestYoutubeDLCBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dlc');
}
async function downloadLatestYoutubeDLPBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
updateDetailsJSON(new_version, 'yt-dlp');
}
function updateDetailsJSON(new_version, downloader) {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (new_version) details_json['version'] = new_version;
details_json['downloader'] = downloader;
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
}
async function checkExistsWithTimeout(filePath, timeout) {
return new Promise(function (resolve, reject) {
var timer = setTimeout(function () {
if (watcher) watcher.close();
reject(new Error('File did not exists and was not created during the timeout.'));
}, timeout);
fs.access(filePath, fs.constants.R_OK, function (err) {
if (!err) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
var dir = path.dirname(filePath);
var basename = path.basename(filePath);
var watcher = fs.watch(dir, function (eventType, filename) {
if (eventType === 'rename' && filename === basename) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
});
} }
app.use(function(req, res, next) { app.use(function(req, res, next) {
@ -1878,6 +1696,104 @@ app.post('/api/cancelDownload', optionalJwt, async (req, res) => {
res.send({success: success}); res.send({success: success});
}); });
// tasks
app.post('/api/getTasks', optionalJwt, async (req, res) => {
const tasks = await db_api.getRecords('tasks');
for (let task of tasks) {
if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task['key']]['job'].nextInvocation().getTime();
}
res.send({tasks: tasks});
});
app.post('/api/resetTasks', optionalJwt, async (req, res) => {
const tasks_keys = Object.keys(tasks_api.TASKS);
for (let i = 0; i < tasks_keys.length; i++) {
const task_key = tasks_keys[i];
tasks_api.TASKS[task_key]['job'] = null;
}
await db_api.removeAllRecords('tasks');
await tasks_api.setupTasks();
res.send({success: true});
});
app.post('/api/getTask', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const task = await db_api.getRecord('tasks', {key: task_key});
if (task['schedule']) task['next_invocation'] = tasks_api.TASKS[task_key]['job'].nextInvocation().getTime();
res.send({task: task});
});
app.post('/api/runTask', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const task = await db_api.getRecord('tasks', {key: task_key});
let success = true;
if (task['running'] || task['confirming']) success = false;
else await tasks_api.executeRun(task_key);
res.send({success: success});
});
app.post('/api/confirmTask', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const task = await db_api.getRecord('tasks', {key: task_key});
let success = true;
if (task['running'] || task['confirming'] || !task['data']) success = false;
else await tasks_api.executeConfirm(task_key);
res.send({success: success});
});
app.post('/api/updateTaskSchedule', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const new_schedule = req.body.new_schedule;
await tasks_api.updateTaskSchedule(task_key, new_schedule);
res.send({success: true});
});
app.post('/api/updateTaskData', optionalJwt, async (req, res) => {
const task_key = req.body.task_key;
const new_data = req.body.new_data;
const success = await db_api.updateRecord('tasks', {key: task_key}, {data: new_data});
res.send({success: success});
});
app.post('/api/getDBBackups', optionalJwt, async (req, res) => {
const backup_dir = path.join('appdata', 'db_backup');
const db_backups = [];
const candidate_backups = await utils.recFindByExt(backup_dir, 'bak', null, [], false);
for (let i = 0; i < candidate_backups.length; i++) {
const candidate_backup = candidate_backups[i];
// must have specific format
if (candidate_backup.split('.').length - 1 !== 4) continue;
const candidate_backup_path = candidate_backup;
const stats = fs.statSync(candidate_backup_path);
db_backups.push({ name: path.basename(candidate_backup), timestamp: parseInt(candidate_backup.split('.')[2]), size: stats.size, source: candidate_backup.includes('local') ? 'local' : 'remote' });
}
db_backups.sort((a,b) => b.timestamp - a.timestamp);
res.send({db_backups: db_backups});
});
app.post('/api/restoreDBBackup', optionalJwt, async (req, res) => {
const file_name = req.body.file_name;
const success = await db_api.restoreDB(file_name);
res.send({success: success});
});
// logs management // logs management
app.post('/api/logs', optionalJwt, async function(req, res) { app.post('/api/logs', optionalJwt, async function(req, res) {

@ -1,6 +1,7 @@
const config_api = require('../config'); const config_api = require('../config');
const consts = require('../consts'); const consts = require('../consts');
const logger = require('../logger'); const logger = require('../logger');
const db_api = require('../db');
const jwt = require('jsonwebtoken'); const jwt = require('jsonwebtoken');
const { uuid } = require('uuidv4'); const { uuid } = require('uuidv4');
@ -12,15 +13,12 @@ var JwtStrategy = require('passport-jwt').Strategy,
ExtractJwt = require('passport-jwt').ExtractJwt; ExtractJwt = require('passport-jwt').ExtractJwt;
// other required vars // other required vars
let db_api = null;
let SERVER_SECRET = null; let SERVER_SECRET = null;
let JWT_EXPIRATION = null; let JWT_EXPIRATION = null;
let opts = null; let opts = null;
let saltRounds = null; let saltRounds = null;
exports.initialize = function(db_api) { exports.initialize = function() {
setDB(db_api);
/************************* /*************************
* Authentication module * Authentication module
************************/ ************************/
@ -51,10 +49,6 @@ exports.initialize = function(db_api) {
})); }));
} }
function setDB(input_db_api) {
db_api = input_db_api;
}
exports.passport = require('passport'); exports.passport = require('passport');
exports.passport.serializeUser(function(user, done) { exports.passport.serializeUser(function(user, done) {

@ -1,14 +1,6 @@
const utils = require('./utils'); const utils = require('./utils');
const logger = require('./logger'); const logger = require('./logger');
const db_api = require('./db');
var db_api = null;
function setDB(input_db_api) { db_api = input_db_api }
function initialize(input_db_api) {
setDB(input_db_api);
}
/* /*
Categories: Categories:
@ -137,7 +129,6 @@ function applyCategoryRules(file_json, rules, category_name) {
// } // }
module.exports = { module.exports = {
initialize: initialize,
categorize: categorize, categorize: categorize,
getCategories: getCategories, getCategories: getCategories,
getCategoriesAsPlaylists: getCategoriesAsPlaylists getCategoriesAsPlaylists: getCategoriesAsPlaylists

@ -54,6 +54,10 @@ const tables = {
name: 'download_queue', name: 'download_queue',
primary_key: 'uid' primary_key: 'uid'
}, },
tasks: {
name: 'tasks',
primary_key: 'key'
},
test: { test: {
name: 'test' name: 'test'
} }
@ -300,6 +304,7 @@ exports.getFileDirectoriesAndDBs = async () => {
} }
exports.importUnregisteredFiles = async () => { exports.importUnregisteredFiles = async () => {
const imported_files = [];
const dirs_to_check = await exports.getFileDirectoriesAndDBs(); const dirs_to_check = await exports.getFileDirectoriesAndDBs();
// run through check list and check each file to see if it's missing from the db // run through check list and check each file to see if it's missing from the db
@ -316,12 +321,17 @@ exports.importUnregisteredFiles = async () => {
const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path))); const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path)));
if (!file_is_registered) { if (!file_is_registered) {
// add additional info // add additional info
await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); const file_obj = await exports.registerFileDB(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null);
logger.verbose(`Added discovered file to the database: ${file.id}`); if (file_obj) {
imported_files.push(file_obj['uid']);
logger.verbose(`Added discovered file to the database: ${file.id}`);
} else {
logger.error(`Failed to import ${file['path']} automatically.`);
}
} }
} }
} }
return imported_files;
} }
exports.addMetadataPropertyToDB = async (property_key) => { exports.addMetadataPropertyToDB = async (property_key) => {
@ -744,6 +754,66 @@ exports.removeRecord = async (table, filter_obj) => {
return !!(output['result']['ok']); return !!(output['result']['ok']);
} }
// exports.removeRecordsByUIDBulk = async (table, uids) => {
// // local db override
// if (using_local_db) {
// applyFilterLocalDB(local_db.get(table), filter_obj, 'remove').write();
// return true;
// }
// const table_collection = database.collection(table);
// let bulk = table_collection.initializeOrderedBulkOp(); // Initialize the Ordered Batch
// const item_ids_to_remove =
// for (let i = 0; i < item_ids_to_update.length; i++) {
// const item_id_to_update = item_ids_to_update[i];
// bulk.find({[key_label]: item_id_to_update }).updateOne({
// "$set": update_obj[item_id_to_update]
// });
// }
// const output = await bulk.execute();
// return !!(output['result']['ok']);
// }
exports.findDuplicatesByKey = async (table, key) => {
let duplicates = [];
if (using_local_db) {
// this can probably be optimized
const all_records = await exports.getRecords(table);
const existing_records = {};
for (let i = 0; i < all_records.length; i++) {
const record = all_records[i];
const value = record[key];
if (existing_records[value]) {
duplicates.push(record);
}
existing_records[value] = true;
}
return duplicates;
}
const duplicated_values = await database.collection(table).aggregate([
{"$group" : { "_id": `$${key}`, "count": { "$sum": 1 } } },
{"$match": {"_id" :{ "$ne" : null } , "count" : {"$gt": 1} } },
{"$project": {[key] : "$_id", "_id" : 0} }
]).toArray();
for (let i = 0; i < duplicated_values.length; i++) {
const duplicated_value = duplicated_values[i];
const duplicated_records = await exports.getRecords(table, duplicated_value, false);
if (duplicated_records.length > 1) {
duplicates = duplicates.concat(duplicated_records.slice(1, duplicated_records.length));
}
}
return duplicates;
}
exports.removeAllRecords = async (table = null, filter_obj = null) => { exports.removeAllRecords = async (table = null, filter_obj = null) => {
// local db override // local db override
const tables_to_remove = table ? [table] : tables_list; const tables_to_remove = table ? [table] : tables_list;
@ -917,6 +987,52 @@ const createDownloadsRecords = (downloads) => {
return new_downloads; return new_downloads;
} }
exports.backupDB = async () => {
const backup_dir = path.join('appdata', 'db_backup');
fs.ensureDirSync(backup_dir);
const backup_file_name = `${using_local_db ? 'local' : 'remote'}_db.json.${Date.now()/1000}.bak`;
const path_to_backups = path.join(backup_dir, backup_file_name);
logger.verbose(`Backing up ${using_local_db ? 'local' : 'remote'} DB to ${path_to_backups}`);
const table_to_records = {};
for (let i = 0; i < tables_list.length; i++) {
const table = tables_list[i];
table_to_records[table] = await exports.getRecords(table);
}
fs.writeJsonSync(path_to_backups, table_to_records);
return backup_file_name;
}
exports.restoreDB = async (file_name) => {
const path_to_backup = path.join('appdata', 'db_backup', file_name);
logger.debug('Reading database backup file.');
const table_to_records = fs.readJSONSync(path_to_backup);
if (!table_to_records) {
logger.error(`Failed to restore DB! Backup file '${path_to_backup}' could not be read.`);
return false;
}
logger.debug('Clearing database.');
await exports.removeAllRecords();
logger.debug('Database cleared! Beginning restore.');
let success = true;
for (let i = 0; i < tables_list.length; i++) {
const table = tables_list[i];
if (!table_to_records[table] || table_to_records[table].length === 0) continue;
success &= await exports.bulkInsertRecordsIntoTable(table, table_to_records[table]);
}
logger.debug('Restore finished!');
return success;
}
exports.transferDB = async (local_to_remote) => { exports.transferDB = async (local_to_remote) => {
const table_to_records = {}; const table_to_records = {};
for (let i = 0; i < tables_list.length; i++) { for (let i = 0; i < tables_list.length; i++) {
@ -926,9 +1042,8 @@ exports.transferDB = async (local_to_remote) => {
using_local_db = !local_to_remote; using_local_db = !local_to_remote;
if (local_to_remote) { if (local_to_remote) {
// backup local DB logger.debug('Backup up DB...');
logger.debug('Backup up Local DB...'); await exports.backupDB();
await fs.copyFile('appdata/local_db.json', `appdata/local_db.json.${Date.now()/1000}.bak`);
const db_connected = await exports.connectToDB(5, true); const db_connected = await exports.connectToDB(5, true);
if (!db_connected) { if (!db_connected) {
logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.'); logger.error('Failed to transfer database - could not connect to MongoDB. Verify that your connection URL is valid.');

@ -14,26 +14,19 @@ const twitch_api = require('./twitch');
const { create } = require('xmlbuilder2'); const { create } = require('xmlbuilder2');
const categories_api = require('./categories'); const categories_api = require('./categories');
const utils = require('./utils'); const utils = require('./utils');
const db_api = require('./db');
let db_api = null;
const mutex = new Mutex(); const mutex = new Mutex();
let should_check_downloads = true; let should_check_downloads = true;
const archivePath = path.join(__dirname, 'appdata', 'archives'); const archivePath = path.join(__dirname, 'appdata', 'archives');
function setDB(input_db_api) { db_api = input_db_api } if (db_api.database_initialized) {
setupDownloads();
exports.initialize = (input_db_api) => { } else {
setDB(input_db_api); db_api.database_initialized_bs.subscribe(init => {
categories_api.initialize(db_api); if (init) setupDownloads();
if (db_api.database_initialized) { });
setupDownloads();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) setupDownloads();
});
}
} }
exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => { exports.createDownload = async (url, type, options, user_uid = null, sub_id = null, sub_name = null) => {

@ -599,6 +599,15 @@
} }
} }
}, },
"call-bind": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz",
"integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==",
"requires": {
"function-bind": "^1.1.1",
"get-intrinsic": "^1.0.2"
}
},
"camelcase": { "camelcase": {
"version": "5.3.1", "version": "5.3.1",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz",
@ -909,6 +918,15 @@
"readable-stream": "^3.4.0" "readable-stream": "^3.4.0"
} }
}, },
"cron-parser": {
"version": "3.5.0",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-3.5.0.tgz",
"integrity": "sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==",
"requires": {
"is-nan": "^1.3.2",
"luxon": "^1.26.0"
}
},
"cross-spawn": { "cross-spawn": {
"version": "7.0.1", "version": "7.0.1",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.1.tgz",
@ -968,6 +986,15 @@
"resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz",
"integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==" "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ=="
}, },
"define-properties": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz",
"integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==",
"requires": {
"has-property-descriptors": "^1.0.0",
"object-keys": "^1.1.1"
}
},
"delayed-stream": { "delayed-stream": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@ -1358,11 +1385,26 @@
"rimraf": "2" "rimraf": "2"
} }
}, },
"function-bind": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz",
"integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A=="
},
"get-caller-file": { "get-caller-file": {
"version": "2.0.5", "version": "2.0.5",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
}, },
"get-intrinsic": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz",
"integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==",
"requires": {
"function-bind": "^1.1.1",
"has": "^1.0.3",
"has-symbols": "^1.0.1"
}
},
"get-stream": { "get-stream": {
"version": "5.1.0", "version": "5.1.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
@ -1460,11 +1502,32 @@
"har-schema": "^2.0.0" "har-schema": "^2.0.0"
} }
}, },
"has": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz",
"integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==",
"requires": {
"function-bind": "^1.1.1"
}
},
"has-flag": { "has-flag": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
}, },
"has-property-descriptors": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz",
"integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==",
"requires": {
"get-intrinsic": "^1.1.1"
}
},
"has-symbols": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
"integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A=="
},
"has-yarn": { "has-yarn": {
"version": "2.1.0", "version": "2.1.0",
"resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz",
@ -1635,6 +1698,15 @@
"is-path-inside": "^3.0.1" "is-path-inside": "^3.0.1"
} }
}, },
"is-nan": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz",
"integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==",
"requires": {
"call-bind": "^1.0.0",
"define-properties": "^1.1.3"
}
},
"is-npm": { "is-npm": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz",
@ -2034,6 +2106,11 @@
} }
} }
}, },
"long-timeout": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/long-timeout/-/long-timeout-0.1.1.tgz",
"integrity": "sha1-lyHXiLR+C8taJMLivuGg2lXatRQ="
},
"lowdb": { "lowdb": {
"version": "1.0.0", "version": "1.0.0",
"resolved": "https://registry.npmjs.org/lowdb/-/lowdb-1.0.0.tgz", "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-1.0.0.tgz",
@ -2051,6 +2128,11 @@
"resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz",
"integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==" "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA=="
}, },
"luxon": {
"version": "1.28.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz",
"integrity": "sha512-TfTiyvZhwBYM/7QdAVDh+7dBTBA29v4ik0Ce9zda3Mnf8on1S5KJI8P2jKFZ8+5C0jhmr0KwJEO/Wdpm0VeWJQ=="
},
"make-dir": { "make-dir": {
"version": "3.1.0", "version": "3.1.0",
"resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz",
@ -2407,6 +2489,16 @@
"iconv-lite": "^0.4.15" "iconv-lite": "^0.4.15"
} }
}, },
"node-schedule": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/node-schedule/-/node-schedule-2.1.0.tgz",
"integrity": "sha512-nl4JTiZ7ZQDc97MmpTq9BQjYhq7gOtoh7SiPH069gBFBj0PzD8HI7zyFs6rzqL8Y5tTiEEYLxgtbx034YPrbyQ==",
"requires": {
"cron-parser": "^3.5.0",
"long-timeout": "0.1.1",
"sorted-array-functions": "^1.3.0"
}
},
"nodemon": { "nodemon": {
"version": "2.0.7", "version": "2.0.7",
"resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.7.tgz",
@ -2475,6 +2567,11 @@
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
}, },
"object-keys": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz",
"integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="
},
"on-finished": { "on-finished": {
"version": "2.3.0", "version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@ -3053,6 +3150,11 @@
"is-arrayish": "^0.3.1" "is-arrayish": "^0.3.1"
} }
}, },
"sorted-array-functions": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/sorted-array-functions/-/sorted-array-functions-1.3.0.tgz",
"integrity": "sha512-2sqgzeFlid6N4Z2fUQ1cvFmTOLRi/sEDzSQ0OKYchqgoPmQBVyM3959qYx3fpS6Esef80KjmpgPeEr028dP3OA=="
},
"sparse-bitfield": { "sparse-bitfield": {
"version": "3.0.3", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz",

@ -52,6 +52,7 @@
"multer": "^1.4.2", "multer": "^1.4.2",
"node-fetch": "^2.6.7", "node-fetch": "^2.6.7",
"node-id3": "^0.1.14", "node-id3": "^0.1.14",
"node-schedule": "^2.1.0",
"nodemon": "^2.0.7", "nodemon": "^2.0.7",
"passport": "^0.4.1", "passport": "^0.4.1",
"passport-http": "^0.3.0", "passport-http": "^0.3.0",

@ -8,15 +8,8 @@ const logger = require('./logger');
const debugMode = process.env.YTDL_MODE === 'debug'; const debugMode = process.env.YTDL_MODE === 'debug';
let db_api = null; const db_api = require('./db');
let downloader_api = null; const downloader_api = require('./downloader');
function setDB(input_db_api) { db_api = input_db_api }
function initialize(input_db_api, input_downloader_api) {
setDB(input_db_api);
downloader_api = input_downloader_api;
}
async function subscribe(sub, user_uid = null) { async function subscribe(sub, user_uid = null) {
const result_obj = { const result_obj = {
@ -542,7 +535,6 @@ module.exports = {
unsubscribe : unsubscribe, unsubscribe : unsubscribe,
deleteSubscriptionFile : deleteSubscriptionFile, deleteSubscriptionFile : deleteSubscriptionFile,
getVideosForSub : getVideosForSub, getVideosForSub : getVideosForSub,
initialize : initialize,
updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple, updateSubscriptionPropertyMultiple : updateSubscriptionPropertyMultiple,
generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload generateOptionsForSubscriptionDownload: generateOptionsForSubscriptionDownload
} }

@ -0,0 +1,195 @@
const db_api = require('./db');
const youtubedl_api = require('./youtube-dl');
const fs = require('fs-extra');
const logger = require('./logger');
const scheduler = require('node-schedule');
const TASKS = {
backup_local_db: {
run: db_api.backupDB,
title: 'Backup DB',
job: null
},
missing_files_check: {
run: checkForMissingFiles,
confirm: deleteMissingFiles,
title: 'Missing files check',
job: null
},
missing_db_records: {
run: db_api.importUnregisteredFiles,
title: 'Import missing DB records',
job: null
},
duplicate_files_check: {
run: checkForDuplicateFiles,
confirm: removeDuplicates,
title: 'Find duplicate files in DB',
job: null
},
youtubedl_update_check: {
run: youtubedl_api.checkForYoutubeDLUpdate,
confirm: youtubedl_api.updateYoutubeDL,
title: 'Update youtube-dl',
job: null
}
}
function scheduleJob(task_key, schedule) {
// schedule has to be converted from our format to one node-schedule can consume
let converted_schedule = null;
if (schedule['type'] === 'timestamp') {
converted_schedule = new Date(schedule['data']['timestamp']);
} else if (schedule['type'] === 'recurring') {
const dayOfWeek = schedule['data']['dayOfWeek'] ? schedule['data']['dayOfWeek'] : null;
const hour = schedule['data']['hour'] ? schedule['data']['hour'] : null;
const minute = schedule['data']['minute'] ? schedule['data']['minute'] : null;
converted_schedule = new scheduler.RecurrenceRule(null, null, null, dayOfWeek, hour, minute);
} else {
logger.error(`Failed to schedule job '${task_key}' as the type '${schedule['type']}' is invalid.`)
return null;
}
return scheduler.scheduleJob(converted_schedule, async () => {
const task_state = await db_api.getRecord('tasks', {key: task_key});
if (task_state['running'] || task_state['confirming']) {
logger.verbose(`Skipping running task ${task_state['key']} as it is already in progress.`);
return;
}
// remove schedule if it's a one-time task
if (task_state['schedule']['type'] !== 'recurring') await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
// we're just "running" the task, any confirmation should be user-initiated
exports.executeRun(task_key);
});
}
if (db_api.database_initialized) {
exports.setupTasks();
} else {
db_api.database_initialized_bs.subscribe(init => {
if (init) exports.setupTasks();
});
}
exports.setupTasks = async () => {
const tasks_keys = Object.keys(TASKS);
for (let i = 0; i < tasks_keys.length; i++) {
const task_key = tasks_keys[i];
const task_in_db = await db_api.getRecord('tasks', {key: task_key});
if (!task_in_db) {
// insert task metadata into table if missing
await db_api.insertRecordIntoTable('tasks', {
key: task_key,
title: TASKS[task_key]['title'],
last_ran: null,
last_confirmed: null,
running: false,
confirming: false,
data: null,
error: null,
schedule: null,
options: {}
});
} else {
// reset task if necessary
await db_api.updateRecord('tasks', {key: task_key}, {running: false, confirming: false});
// schedule task and save job
if (task_in_db['schedule']) {
// prevent timestamp schedules from being set to the past
if (task_in_db['schedule']['type'] === 'timestamp' && task_in_db['schedule']['data']['timestamp'] < Date.now()) {
await db_api.updateRecord('tasks', {key: task_key}, {schedule: null});
continue;
}
TASKS[task_key]['job'] = scheduleJob(task_key, task_in_db['schedule']);
}
}
}
}
exports.executeTask = async (task_key) => {
if (!TASKS[task_key]) {
logger.error(`Task ${task_key} does not exist!`);
return;
}
logger.verbose(`Executing task ${task_key}`);
await exports.executeRun(task_key);
if (!TASKS[task_key]['confirm']) return;
await exports.executeConfirm(task_key);
logger.verbose(`Finished executing ${task_key}`);
}
exports.executeRun = async (task_key) => {
logger.verbose(`Running task ${task_key}`);
// don't set running to true when backup up DB as it will be stick "running" if restored
if (task_key !== 'backup_local_db') await db_api.updateRecord('tasks', {key: task_key}, {running: true});
const data = await TASKS[task_key].run();
await db_api.updateRecord('tasks', {key: task_key}, {data: TASKS[task_key]['confirm'] ? data : null, last_ran: Date.now()/1000, running: false});
logger.verbose(`Finished running task ${task_key}`);
}
exports.executeConfirm = async (task_key) => {
logger.verbose(`Confirming task ${task_key}`);
if (!TASKS[task_key]['confirm']) {
return null;
}
await db_api.updateRecord('tasks', {key: task_key}, {confirming: true});
const task_obj = await db_api.getRecord('tasks', {key: task_key});
const data = task_obj['data'];
await TASKS[task_key].confirm(data);
await db_api.updateRecord('tasks', {key: task_key}, {confirming: false, last_confirmed: Date.now()/1000, data: null});
logger.verbose(`Finished confirming task ${task_key}`);
}
exports.updateTaskSchedule = async (task_key, schedule) => {
logger.verbose(`Updating schedule for task ${task_key}`);
await db_api.updateRecord('tasks', {key: task_key}, {schedule: schedule});
if (TASKS[task_key]['job']) {
TASKS[task_key]['job'].cancel();
}
if (schedule) {
TASKS[task_key]['job'] = scheduleJob(task_key, schedule);
}
}
// missing files check
async function checkForMissingFiles() {
const missing_files = [];
const all_files = await db_api.getRecords('files');
for (let i = 0; i < all_files.length; i++) {
const file_to_check = all_files[i];
const file_exists = fs.existsSync(file_to_check['path']);
if (!file_exists) missing_files.push(file_to_check['uid']);
}
return {uids: missing_files};
}
async function deleteMissingFiles(data) {
const uids = data['uids'];
for (let i = 0; i < uids.length; i++) {
const uid = uids[i];
await db_api.removeRecord('files', {uid: uid});
}
}
// duplicate files check
async function checkForDuplicateFiles() {
const duplicate_files = await db_api.findDuplicatesByKey('files', 'path');
const duplicate_uids = duplicate_files.map(duplicate_file => duplicate_file['uid']);
if (duplicate_uids && duplicate_uids.length > 0) {
return {uids: duplicate_uids};
}
return {uids: []};
}
async function removeDuplicates(data) {
for (let i = 0; i < data['uids'].length; i++) {
await db_api.removeRecord('files', {uid: data['uids'][i]});
}
}
exports.TASKS = TASKS;

@ -70,6 +70,17 @@ describe('Database', async function() {
const success = await db_api.getRecord('test', {test: 'test'}); const success = await db_api.getRecord('test', {test: 'test'});
assert(success); assert(success);
}); });
it('Restore db', async function() {
const db_stats = await db_api.getDBStats();
const file_name = await db_api.backupDB();
await db_api.restoreDB(file_name);
const new_db_stats = await db_api.getDBStats();
assert(JSON.stringify(db_stats), JSON.stringify(new_db_stats));
});
}); });
describe('Export', function() { describe('Export', function() {
@ -83,12 +94,37 @@ describe('Database', async function() {
await db_api.removeAllRecords('test'); await db_api.removeAllRecords('test');
}); });
it('Add and read record', async function() { it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined}); await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null}); const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test'); assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'}); await db_api.removeRecord('test', {test_add: 'test'});
}); });
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
});
it('Update record', async function() { it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'}); await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true}); await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
@ -122,6 +158,7 @@ describe('Database', async function() {
}); });
it('Bulk add', async function() { it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000 const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = []; const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
@ -291,7 +328,6 @@ describe('Multi User', async function() {
describe('Downloader', function() { describe('Downloader', function() {
const downloader_api = require('../downloader'); const downloader_api = require('../downloader');
downloader_api.initialize(db_api);
const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'; const url = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c'; const sub_id = 'dc834388-3454-41bf-a618-e11cb8c7de1c';
const options = { const options = {
@ -348,5 +384,98 @@ describe('Downloader', function() {
const sample_json = fs.readJSONSync('./test/sample.info.json'); const sample_json = fs.readJSONSync('./test/sample.info.json');
downloader_api.generateNFOFile(sample_json, nfo_file_path); downloader_api.generateNFOFile(sample_json, nfo_file_path);
assert(fs.existsSync(nfo_file_path), true); assert(fs.existsSync(nfo_file_path), true);
fs.unlinkSync(nfo_file_path);
}); });
}); });
describe('Tasks', function() {
const tasks_api = require('../tasks');
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('tasks');
const dummy_task = {
run: async () => { await utils.wait(500); return true; },
confirm: async () => { await utils.wait(500); return true; },
title: 'Dummy task',
job: null
};
tasks_api.TASKS['dummy_task'] = dummy_task;
await tasks_api.initialize();
});
it('Backup db', async function() {
const backups_original = await utils.recFindByExt('appdata', 'bak');
const original_length = backups_original.length;
await tasks_api.executeTask('backup_local_db');
const backups_new = await utils.recFindByExt('appdata', 'bak');
const new_length = backups_new.length;
assert(original_length, new_length-1);
});
it('Check for missing files', async function() {
await db_api.removeAllRecords('files', {uid: 'test'});
const test_missing_file = {uid: 'test', path: 'test/missing_file.mp4'};
await db_api.insertRecordIntoTable('files', test_missing_file);
await tasks_api.executeTask('missing_files_check');
const task_obj = await db_api.getRecord('tasks', {key: 'missing_files_check'});
assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
});
it('Check for duplicate files', async function() {
this.timeout(300000);
await db_api.removeAllRecords('files', {uid: 'test1'});
await db_api.removeAllRecords('files', {uid: 'test2'});
const test_duplicate_file1 = {uid: 'test1', path: 'test/missing_file.mp4'};
const test_duplicate_file2 = {uid: 'test2', path: 'test/missing_file.mp4'};
const test_duplicate_file3 = {uid: 'test3', path: 'test/missing_file.mp4'};
await db_api.insertRecordIntoTable('files', test_duplicate_file1);
await db_api.insertRecordIntoTable('files', test_duplicate_file2);
await db_api.insertRecordIntoTable('files', test_duplicate_file3);
await tasks_api.executeTask('duplicate_files_check');
const task_obj = await db_api.getRecord('tasks', {key: 'duplicate_files_check'});
const duplicated_record_count = await db_api.getRecords('files', {path: 'test/missing_file.mp4'}, true);
assert(task_obj['data'] && task_obj['data']['uids'] && task_obj['data']['uids'].length >= 1, true);
assert(duplicated_record_count == 1, true);
});
it('Import unregistered files', async function() {
this.timeout(300000);
// pre-test cleanup
await db_api.removeAllRecords('files', {title: 'Sample File'});
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
// copies in files
fs.copyFileSync('test/sample.info.json', 'video/sample.info.json');
fs.copyFileSync('test/sample.mp4', 'video/sample.mp4');
await tasks_api.executeTask('missing_db_records');
const imported_file = await db_api.getRecord('files', {title: 'Sample File'});
assert(!!imported_file, true);
// post-test cleanup
if (fs.existsSync('video/sample.info.json')) fs.unlinkSync('video/sample.info.json');
if (fs.existsSync('video/sample.mp4')) fs.unlinkSync('video/sample.mp4');
});
it('Schedule and cancel task', async function() {
const today_4_hours = new Date();
today_4_hours.setHours(today_4_hours.getHours() + 4);
await tasks_api.updateTaskSchedule('dummy_task', today_4_hours);
assert(!!tasks_api.TASKS['dummy_task']['job'], true);
await tasks_api.updateTaskSchedule('dummy_task', null);
assert(!!tasks_api.TASKS['dummy_task']['job'], false);
});
it('Schedule and run task', async function() {
this.timeout(5000);
const today_1_second = new Date();
today_1_second.setSeconds(today_1_second.getSeconds() + 1);
await tasks_api.updateTaskSchedule('dummy_task', today_1_second);
assert(!!tasks_api.TASKS['dummy_task']['job'], true);
await utils.wait(2000);
const dummy_task_obj = await db_api.getRecord('tasks', {key: 'dummy_task'});
assert(dummy_task_obj['data'], true);
});
});

@ -1,10 +1,13 @@
const fs = require('fs-extra') const fs = require('fs-extra');
const path = require('path') const path = require('path');
const ffmpeg = require('fluent-ffmpeg'); const ffmpeg = require('fluent-ffmpeg');
const archiver = require('archiver');
const fetch = require('node-fetch');
const ProgressBar = require('progress');
const config_api = require('./config'); const config_api = require('./config');
const logger = require('./logger'); const logger = require('./logger');
const CONSTS = require('./consts') const CONSTS = require('./consts');
const archiver = require('archiver');
const is_windows = process.platform === 'win32'; const is_windows = process.platform === 'win32';
@ -266,7 +269,7 @@ function getCurrentDownloader() {
return details_json['downloader']; return details_json['downloader'];
} }
async function recFindByExt(base,ext,files,result) async function recFindByExt(base, ext, files, result, recursive = true)
{ {
files = files || (await fs.readdir(base)) files = files || (await fs.readdir(base))
result = result || [] result = result || []
@ -275,6 +278,7 @@ async function recFindByExt(base,ext,files,result)
var newbase = path.join(base,file) var newbase = path.join(base,file)
if ( (await fs.stat(newbase)).isDirectory() ) if ( (await fs.stat(newbase)).isDirectory() )
{ {
if (!recursive) continue;
result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result) result = await recFindByExt(newbase,ext,await fs.readdir(newbase),result)
} }
else else
@ -355,6 +359,62 @@ async function cropFile(file_path, start, end, ext) {
}); });
} }
async function checkExistsWithTimeout(filePath, timeout) {
return new Promise(function (resolve, reject) {
var timer = setTimeout(function () {
if (watcher) watcher.close();
reject(new Error('File did not exists and was not created during the timeout.'));
}, timeout);
fs.access(filePath, fs.constants.R_OK, function (err) {
if (!err) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
var dir = path.dirname(filePath);
var basename = path.basename(filePath);
var watcher = fs.watch(dir, function (eventType, filename) {
if (eventType === 'rename' && filename === basename) {
clearTimeout(timer);
if (watcher) watcher.close();
resolve();
}
});
});
}
// helper function to download file using fetch
async function fetchFile(url, path, file_label) {
var len = null;
const res = await fetch(url);
len = parseInt(res.headers.get("Content-Length"), 10);
var bar = new ProgressBar(` Downloading ${file_label} [:bar] :percent :etas`, {
complete: '=',
incomplete: ' ',
width: 20,
total: len
});
const fileStream = fs.createWriteStream(path);
await new Promise((resolve, reject) => {
res.body.pipe(fileStream);
res.body.on("error", (err) => {
reject(err);
});
res.body.on('data', function (chunk) {
bar.tick(chunk.length);
});
fileStream.on("finish", function() {
resolve();
});
});
}
// objects // objects
function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) { function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date, description, view_count, height, abr) {
@ -396,5 +456,7 @@ module.exports = {
cropFile: cropFile, cropFile: cropFile,
createEdgeNGrams: createEdgeNGrams, createEdgeNGrams: createEdgeNGrams,
wait: wait, wait: wait,
checkExistsWithTimeout: checkExistsWithTimeout,
fetchFile: fetchFile,
File: File File: File
} }

@ -0,0 +1,127 @@
const fs = require('fs-extra');
const fetch = require('node-fetch');
const logger = require('./logger');
const utils = require('./utils');
const CONSTS = require('./consts');
const config_api = require('./config.js');
const is_windows = process.platform === 'win32';
const download_sources = {
'youtube-dl': {
'tags_url': 'https://api.github.com/repos/ytdl-org/youtube-dl/tags',
'func': downloadLatestYoutubeDLBinary
},
'youtube-dlc': {
'tags_url': 'https://api.github.com/repos/blackjack4494/yt-dlc/tags',
'func': downloadLatestYoutubeDLCBinary
},
'yt-dlp': {
'tags_url': 'https://api.github.com/repos/yt-dlp/yt-dlp/tags',
'func': downloadLatestYoutubeDLPBinary
}
}
exports.checkForYoutubeDLUpdate = async () => {
return new Promise(async resolve => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
const tags_url = download_sources[default_downloader]['tags_url'];
// get current version
let current_app_details_exists = fs.existsSync(CONSTS.DETAILS_BIN_PATH);
if (!current_app_details_exists) {
logger.warn(`Failed to get youtube-dl binary details at location '${CONSTS.DETAILS_BIN_PATH}'. Generating file...`);
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, {"version":"2020.00.00", "downloader": default_downloader});
}
let current_app_details = JSON.parse(fs.readFileSync(CONSTS.DETAILS_BIN_PATH));
let current_version = current_app_details['version'];
let current_downloader = current_app_details['downloader'];
let stored_binary_path = current_app_details['path'];
if (!stored_binary_path || typeof stored_binary_path !== 'string') {
// logger.info(`INFO: Failed to get youtube-dl binary path at location: ${CONSTS.DETAILS_BIN_PATH}, attempting to guess actual path...`);
const guessed_base_path = 'node_modules/youtube-dl/bin/';
const guessed_file_path = guessed_base_path + 'youtube-dl' + (is_windows ? '.exe' : '');
if (fs.existsSync(guessed_file_path)) {
stored_binary_path = guessed_file_path;
// logger.info('INFO: Guess successful! Update process continuing...')
} else {
logger.error(`Guess '${guessed_file_path}' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install.`);
resolve(null);
return;
}
}
// got version, now let's check the latest version from the youtube-dl API
fetch(tags_url, {method: 'Get'})
.then(async res => res.json())
.then(async (json) => {
// check if the versions are different
if (!json || !json[0]) {
logger.error(`Failed to check ${default_downloader} version for an update.`)
resolve(null);
return;
}
const latest_update_version = json[0]['name'];
if (current_version !== latest_update_version || default_downloader !== current_downloader) {
// versions different or different downloader is being used, download new update
resolve(latest_update_version);
} else {
resolve(null);
}
return;
})
.catch(err => {
logger.error(`Failed to check ${default_downloader} version for an update.`)
logger.error(err);
resolve(null);
return;
});
});
}
exports.updateYoutubeDL = async (latest_update_version) => {
const default_downloader = config_api.getConfigItem('ytdl_default_downloader');
await download_sources[default_downloader]['func'](latest_update_version);
}
async function downloadLatestYoutubeDLBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/ytdl-org/youtube-dl/releases/latest/download/youtube-dl${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dl ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dl');
}
async function downloadLatestYoutubeDLCBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/blackjack4494/yt-dlc/releases/latest/download/youtube-dlc${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `youtube-dlc ${new_version}`);
updateDetailsJSON(new_version, 'youtube-dlc');
}
async function downloadLatestYoutubeDLPBinary(new_version) {
const file_ext = is_windows ? '.exe' : '';
const download_url = `https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp${file_ext}`;
const output_path = `node_modules/youtube-dl/bin/youtube-dl${file_ext}`;
await utils.fetchFile(download_url, output_path, `yt-dlp ${new_version}`);
updateDetailsJSON(new_version, 'yt-dlp');
}
function updateDetailsJSON(new_version, downloader) {
const details_json = fs.readJSONSync(CONSTS.DETAILS_BIN_PATH);
if (new_version) details_json['version'] = new_version;
details_json['downloader'] = downloader;
fs.writeJSONSync(CONSTS.DETAILS_BIN_PATH, details_json);
}

@ -21,6 +21,7 @@ export type { CreatePlaylistRequest } from './models/CreatePlaylistRequest';
export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse'; export type { CreatePlaylistResponse } from './models/CreatePlaylistResponse';
export type { CropFileSettings } from './models/CropFileSettings'; export type { CropFileSettings } from './models/CropFileSettings';
export type { DatabaseFile } from './models/DatabaseFile'; export type { DatabaseFile } from './models/DatabaseFile';
export { DBBackup } from './models/DBBackup';
export type { DBInfoResponse } from './models/DBInfoResponse'; export type { DBInfoResponse } from './models/DBInfoResponse';
export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest'; export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest';
export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request'; export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request';
@ -44,6 +45,8 @@ export type { GetAllDownloadsRequest } from './models/GetAllDownloadsRequest';
export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse'; export type { GetAllDownloadsResponse } from './models/GetAllDownloadsResponse';
export type { GetAllFilesResponse } from './models/GetAllFilesResponse'; export type { GetAllFilesResponse } from './models/GetAllFilesResponse';
export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse'; export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse';
export type { GetAllTasksResponse } from './models/GetAllTasksResponse';
export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse';
export type { GetDownloadRequest } from './models/GetDownloadRequest'; export type { GetDownloadRequest } from './models/GetDownloadRequest';
export type { GetDownloadResponse } from './models/GetDownloadResponse'; export type { GetDownloadResponse } from './models/GetDownloadResponse';
export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest'; export type { GetFileFormatsRequest } from './models/GetFileFormatsRequest';
@ -63,6 +66,8 @@ export type { GetPlaylistsResponse } from './models/GetPlaylistsResponse';
export type { GetRolesResponse } from './models/GetRolesResponse'; export type { GetRolesResponse } from './models/GetRolesResponse';
export type { GetSubscriptionRequest } from './models/GetSubscriptionRequest'; export type { GetSubscriptionRequest } from './models/GetSubscriptionRequest';
export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse'; export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse';
export type { GetTaskRequest } from './models/GetTaskRequest';
export type { GetTaskResponse } from './models/GetTaskResponse';
export type { GetUsersResponse } from './models/GetUsersResponse'; export type { GetUsersResponse } from './models/GetUsersResponse';
export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest'; export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest';
export type { inline_response_200_15 } from './models/inline_response_200_15'; export type { inline_response_200_15 } from './models/inline_response_200_15';
@ -71,6 +76,8 @@ export type { LoginResponse } from './models/LoginResponse';
export type { Playlist } from './models/Playlist'; export type { Playlist } from './models/Playlist';
export type { RegisterRequest } from './models/RegisterRequest'; export type { RegisterRequest } from './models/RegisterRequest';
export type { RegisterResponse } from './models/RegisterResponse'; export type { RegisterResponse } from './models/RegisterResponse';
export type { RestoreDBBackupRequest } from './models/RestoreDBBackupRequest';
export { Schedule } from './models/Schedule';
export type { SetConfigRequest } from './models/SetConfigRequest'; export type { SetConfigRequest } from './models/SetConfigRequest';
export type { SharingToggle } from './models/SharingToggle'; export type { SharingToggle } from './models/SharingToggle';
export type { SubscribeRequest } from './models/SubscribeRequest'; export type { SubscribeRequest } from './models/SubscribeRequest';
@ -79,6 +86,7 @@ export type { Subscription } from './models/Subscription';
export type { SubscriptionRequestData } from './models/SubscriptionRequestData'; export type { SubscriptionRequestData } from './models/SubscriptionRequestData';
export type { SuccessObject } from './models/SuccessObject'; export type { SuccessObject } from './models/SuccessObject';
export type { TableInfo } from './models/TableInfo'; export type { TableInfo } from './models/TableInfo';
export type { Task } from './models/Task';
export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest'; export type { TestConnectionStringRequest } from './models/TestConnectionStringRequest';
export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse'; export type { TestConnectionStringResponse } from './models/TestConnectionStringResponse';
export type { TransferDBRequest } from './models/TransferDBRequest'; export type { TransferDBRequest } from './models/TransferDBRequest';
@ -93,6 +101,8 @@ export type { UpdateConcurrentStreamResponse } from './models/UpdateConcurrentSt
export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest'; export type { UpdatePlaylistRequest } from './models/UpdatePlaylistRequest';
export type { UpdaterStatus } from './models/UpdaterStatus'; export type { UpdaterStatus } from './models/UpdaterStatus';
export type { UpdateServerRequest } from './models/UpdateServerRequest'; export type { UpdateServerRequest } from './models/UpdateServerRequest';
export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest';
export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest';
export type { UpdateUserRequest } from './models/UpdateUserRequest'; export type { UpdateUserRequest } from './models/UpdateUserRequest';
export type { User } from './models/User'; export type { User } from './models/User';
export { UserPermission } from './models/UserPermission'; export { UserPermission } from './models/UserPermission';

@ -0,0 +1,21 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface DBBackup {
name: string;
timestamp: number;
size: number;
source: DBBackup.source;
}
export namespace DBBackup {
export enum source {
LOCAL = 'local',
REMOTE = 'remote',
}
}

@ -2,6 +2,7 @@
/* tslint:disable */ /* tslint:disable */
/* eslint-disable */ /* eslint-disable */
import { FileType } from './FileType';
export interface DownloadFileRequest { export interface DownloadFileRequest {
uid?: string; uid?: string;
@ -9,5 +10,5 @@ export interface DownloadFileRequest {
sub_id?: string; sub_id?: string;
playlist_id?: string; playlist_id?: string;
url?: string; url?: string;
type?: string; type?: FileType;
} }

@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Task } from './Task';
export interface GetAllTasksResponse {
tasks?: Array<Task>;
}

@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { DBBackup } from './DBBackup';
export interface GetDBBackupsResponse {
tasks?: Array<DBBackup>;
}

@ -15,8 +15,5 @@ export interface GetFullTwitchChatRequest {
* User UID * User UID
*/ */
uuid?: string; uuid?: string;
/**
* Subscription
*/
sub?: Subscription; sub?: Subscription;
} }

@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface GetTaskRequest {
task_key: string;
}

@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Task } from './Task';
export interface GetTaskResponse {
task?: Task;
}

@ -0,0 +1,8 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface RestoreDBBackupRequest {
file_name: string;
}

@ -0,0 +1,24 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface Schedule {
type: Schedule.type;
data: {
dayOfWeek?: Array<number>,
hour?: number,
minute?: number,
timestamp?: number,
};
}
export namespace Schedule {
export enum type {
TIMESTAMP = 'timestamp',
RECURRING = 'recurring',
}
}

@ -0,0 +1,15 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface Task {
key: string;
last_ran: number;
last_confirmed: number;
running: boolean;
confirming: boolean;
data: any;
error: string;
schedule: any;
}

@ -0,0 +1,9 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export interface UpdateTaskDataRequest {
task_key: string;
new_data: any;
}

@ -0,0 +1,10 @@
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
import { Schedule } from './Schedule';
export interface UpdateTaskScheduleRequest {
task_key: string;
new_schedule: Schedule;
}

@ -8,6 +8,7 @@ import { PostsService } from './posts.services';
import { LoginComponent } from './components/login/login.component'; import { LoginComponent } from './components/login/login.component';
import { DownloadsComponent } from './components/downloads/downloads.component'; import { DownloadsComponent } from './components/downloads/downloads.component';
import { SettingsComponent } from './settings/settings.component'; import { SettingsComponent } from './settings/settings.component';
import { TasksComponent } from './components/tasks/tasks.component';
const routes: Routes = [ const routes: Routes = [
{ path: 'home', component: MainComponent, canActivate: [PostsService] }, { path: 'home', component: MainComponent, canActivate: [PostsService] },
@ -17,6 +18,7 @@ const routes: Routes = [
{ path: 'settings', component: SettingsComponent, canActivate: [PostsService] }, { path: 'settings', component: SettingsComponent, canActivate: [PostsService] },
{ path: 'login', component: LoginComponent }, { path: 'login', component: LoginComponent },
{ path: 'downloads', component: DownloadsComponent, canActivate: [PostsService] }, { path: 'downloads', component: DownloadsComponent, canActivate: [PostsService] },
{ path: 'tasks', component: TasksComponent, canActivate: [PostsService] },
{ path: '', redirectTo: '/home', pathMatch: 'full' } { path: '', redirectTo: '/home', pathMatch: 'full' }
]; ];

@ -44,6 +44,7 @@
<a *ngIf="postsService.config && postsService.config.Advanced.multi_user_mode && !postsService.isLoggedIn" mat-list-item (click)="sidenav.close()" routerLink='/login'><ng-container i18n="Navigation menu Login Page title">Login</ng-container></a> <a *ngIf="postsService.config && postsService.config.Advanced.multi_user_mode && !postsService.isLoggedIn" mat-list-item (click)="sidenav.close()" routerLink='/login'><ng-container i18n="Navigation menu Login Page title">Login</ng-container></a>
<a *ngIf="postsService.config && allowSubscriptions && postsService.hasPermission('subscriptions')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/subscriptions'><ng-container i18n="Navigation menu Subscriptions Page title">Subscriptions</ng-container></a> <a *ngIf="postsService.config && allowSubscriptions && postsService.hasPermission('subscriptions')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/subscriptions'><ng-container i18n="Navigation menu Subscriptions Page title">Subscriptions</ng-container></a>
<a *ngIf="postsService.config && enableDownloadsManager && postsService.hasPermission('downloads_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/downloads'><ng-container i18n="Navigation menu Downloads Page title">Downloads</ng-container></a> <a *ngIf="postsService.config && enableDownloadsManager && postsService.hasPermission('downloads_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/downloads'><ng-container i18n="Navigation menu Downloads Page title">Downloads</ng-container></a>
<a *ngIf="postsService.config && enableDownloadsManager && postsService.hasPermission('downloads_manager')" mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/tasks'><ng-container i18n="Navigation menu Tasks Page title">Tasks</ng-container></a>
<ng-container *ngIf="postsService.config && postsService.hasPermission('settings')"> <ng-container *ngIf="postsService.config && postsService.hasPermission('settings')">
<mat-divider></mat-divider> <mat-divider></mat-divider>
<a mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/settings'><ng-container i18n="Settings menu label">Settings</ng-container></a> <a mat-list-item (click)="postsService.sidepanel_mode === 'over' ? sidenav.close() : null" routerLink='/settings'><ng-container i18n="Settings menu label">Settings</ng-container></a>

@ -28,6 +28,7 @@ import { MatTabsModule } from '@angular/material/tabs';
import { MatPaginatorModule } from '@angular/material/paginator'; import { MatPaginatorModule } from '@angular/material/paginator';
import { MatSortModule } from '@angular/material/sort'; import { MatSortModule } from '@angular/material/sort';
import { MatTableModule } from '@angular/material/table'; import { MatTableModule } from '@angular/material/table';
import { MatDatepickerModule } from '@angular/material/datepicker';
import { DragDropModule } from '@angular/cdk/drag-drop'; import { DragDropModule } from '@angular/cdk/drag-drop';
import { ClipboardModule } from '@angular/cdk/clipboard'; import { ClipboardModule } from '@angular/cdk/clipboard';
import { TextFieldModule } from '@angular/cdk/text-field'; import { TextFieldModule } from '@angular/cdk/text-field';
@ -87,6 +88,9 @@ import { LinkifyPipe, SeeMoreComponent } from './components/see-more/see-more.co
import { H401Interceptor } from './http.interceptor'; import { H401Interceptor } from './http.interceptor';
import { ConcurrentStreamComponent } from './components/concurrent-stream/concurrent-stream.component'; import { ConcurrentStreamComponent } from './components/concurrent-stream/concurrent-stream.component';
import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-button.component'; import { SkipAdButtonComponent } from './components/skip-ad-button/skip-ad-button.component';
import { TasksComponent } from './components/tasks/tasks.component';
import { UpdateTaskScheduleDialogComponent } from './dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
import { RestoreDbDialogComponent } from './dialogs/restore-db-dialog/restore-db-dialog.component';
registerLocaleData(es, 'es'); registerLocaleData(es, 'es');
@ -135,7 +139,10 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
TwitchChatComponent, TwitchChatComponent,
SeeMoreComponent, SeeMoreComponent,
ConcurrentStreamComponent, ConcurrentStreamComponent,
SkipAdButtonComponent SkipAdButtonComponent,
TasksComponent,
UpdateTaskScheduleDialogComponent,
RestoreDbDialogComponent
], ],
imports: [ imports: [
CommonModule, CommonModule,
@ -171,6 +178,7 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible
MatPaginatorModule, MatPaginatorModule,
MatSortModule, MatSortModule,
MatTableModule, MatTableModule,
MatDatepickerModule,
MatChipsModule, MatChipsModule,
DragDropModule, DragDropModule,
ClipboardModule, ClipboardModule,

@ -0,0 +1,95 @@
<div [hidden]="!(tasks && tasks.length > 0)">
<div style="overflow: hidden;" [ngClass]="'mat-elevation-z8'">
<mat-table style="overflow: hidden" matSort [dataSource]="dataSource">
<!-- Title Column -->
<ng-container matColumnDef="title">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Title">Title</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<span class="one-line" [matTooltip]="element.title ? element.title : null">
{{element.title}}
</span>
</mat-cell>
</ng-container>
<!-- Last Ran Column -->
<ng-container matColumnDef="last_ran">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Last ran">Last ran</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<ng-container *ngIf="element.last_ran">{{element.last_ran*1000 | date: 'short'}}</ng-container>
<ng-container i18n="N/A" *ngIf="!element.last_ran">N/A</ng-container>
</mat-cell>
</ng-container>
<!-- Last Confirmed Column -->
<ng-container matColumnDef="last_confirmed">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Last confirmed">Last confirmed</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<ng-container *ngIf="element.last_confirmed">{{element.last_confirmed*1000 | date: 'short'}}</ng-container>
<ng-container i18n="N/A" *ngIf="!element.last_confirmed">N/A</ng-container>
</mat-cell>
</ng-container>
<!-- Status Column -->
<ng-container matColumnDef="status">
<mat-header-cell *matHeaderCellDef mat-sort-header> <ng-container i18n="Status">Status</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<span *ngIf="element.running || element.confirming"><mat-spinner matTooltip="Busy" i18n-matTooltip="Busy" [diameter]="25"></mat-spinner></span>
<span *ngIf="!(element.running || element.confirming) && element.schedule" style="display: flex">
<ng-container i18n="Scheduled">Scheduled for</ng-container>&nbsp;
{{element.next_invocation | date: 'short'}}<mat-icon style="font-size: 16px; display: inline-flex; align-items: center; padding-left: 5px;" *ngIf="element.schedule.type === 'recurring'">repeat</mat-icon>
</span>
<span *ngIf="!(element.running || element.confirming) && !element.schedule">
<ng-container i18n="Not scheduled">Not scheduled</ng-container>
</span>
</mat-cell>
</ng-container>
<!-- Actions Column -->
<ng-container matColumnDef="actions">
<mat-header-cell *matHeaderCellDef> <ng-container i18n="Actions">Actions</ng-container> </mat-header-cell>
<mat-cell *matCellDef="let element">
<div class="container">
<div class="row justify-content-center">
<div *ngIf="element.data?.uids?.length > 0 || (!element.data?.uids && element.data)" class="col-12 mt-2" style="display: flex; justify-content: center;">
<ng-container>
<button (click)="confirmTask(element.key)" [disabled]="element.running || element.confirming" mat-stroked-button>
<ng-container *ngIf="element.key == 'missing_files_check'">
<ng-container i18n="Clear missing files from DB">Clear missing files from DB:</ng-container>{{element.data.uids.length}}
</ng-container>
<ng-container *ngIf="element.key == 'duplicate_files_check'">
<ng-container i18n="Clear duplicate files from DB">Clear duplicate files from DB:</ng-container>&nbsp;{{element.data.uids.length}}
</ng-container>
<ng-container *ngIf="element.key == 'youtubedl_update_check'">
<ng-container i18n="Update binary to">Update binary to:</ng-container>&nbsp;{{element.data}}
</ng-container>
</button>
</ng-container>
</div>
<div class="col-3">
<button (click)="runTask(element.key)" [disabled]="element.running || element.confirming" mat-icon-button matTooltip="Run" i18n-matTooltip="Run"><mat-icon>play_arrow</mat-icon></button>
</div>
<div class="col-3">
<button (click)="scheduleTask(element)" mat-icon-button matTooltip="Schedule" i18n-matTooltip="Schedule"><mat-icon>schedule</mat-icon></button>
</div>
</div>
</div>
</mat-cell>
</ng-container>
<mat-header-row *matHeaderRowDef="displayedColumns"></mat-header-row>
<mat-row *matRowDef="let row; columns: displayedColumns;"></mat-row>
</mat-table>
<mat-paginator [pageSizeOptions]="[5, 10, 20]"
showFirstLastButtons
aria-label="Select page of tasks">
</mat-paginator>
</div>
<button style="margin-top: 10px; margin-left: 5px;" mat-stroked-button (click)="openRestoreDBBackupDialog()" i18n="Restore DB from backup button">Restore DB from backup</button>
<button style="margin-top: 10px; margin-left: 5px;" mat-stroked-button (click)="resetTasks()" color="warn" i18n="Reset tasks button">Reset tasks</button>
</div>
<div *ngIf="(!tasks || tasks.length === 0) && tasks_retrieved">
<h4 style="text-align: center; margin-top: 10px;" i18n="No tasks label">No tasks available!</h4>
</div>

@ -0,0 +1,32 @@
mat-header-cell, mat-cell {
justify-content: center;
}
.one-line {
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.icon-button-spinner {
position: absolute;
top: 7px;
left: 6px;
}
.downloads-action-button-div {
margin-top: 10px;
margin-left: 5px;
}
.rounded-top {
border-radius: 16px 16px 0px 0px !important;
}
.rounded-bottom {
border-radius: 0px 0px 16px 16px !important;
}
.rounded {
border-radius: 16px 16px 16px 16px !important;
}

@ -0,0 +1,25 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { TasksComponent } from './tasks.component';
describe('TasksComponent', () => {
let component: TasksComponent;
let fixture: ComponentFixture<TasksComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ TasksComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(TasksComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

@ -0,0 +1,164 @@
import { Component, OnInit, ViewChild } from '@angular/core';
import { MatDialog } from '@angular/material/dialog';
import { MatPaginator } from '@angular/material/paginator';
import { MatSort } from '@angular/material/sort';
import { MatTableDataSource } from '@angular/material/table';
import { ConfirmDialogComponent } from 'app/dialogs/confirm-dialog/confirm-dialog.component';
import { RestoreDbDialogComponent } from 'app/dialogs/restore-db-dialog/restore-db-dialog.component';
import { UpdateTaskScheduleDialogComponent } from 'app/dialogs/update-task-schedule-dialog/update-task-schedule-dialog.component';
import { PostsService } from 'app/posts.services';
@Component({
selector: 'app-tasks',
templateUrl: './tasks.component.html',
styleUrls: ['./tasks.component.scss']
})
export class TasksComponent implements OnInit {
interval_id = null;
tasks_check_interval = 1500;
tasks = null;
tasks_retrieved = false;
displayedColumns: string[] = ['title', 'last_ran', 'last_confirmed', 'status', 'actions'];
dataSource = null;
db_backups = [];
@ViewChild(MatPaginator) paginator: MatPaginator;
@ViewChild(MatSort) sort: MatSort;
constructor(private postsService: PostsService, private dialog: MatDialog) { }
ngOnInit(): void {
if (this.postsService.initialized) {
this.getTasksRecurring();
} else {
this.postsService.service_initialized.subscribe(init => {
if (init) {
this.getTasksRecurring();
}
});
}
}
ngOnDestroy(): void {
if (this.interval_id) { clearInterval(this.interval_id) }
}
getTasksRecurring(): void {
this.getTasks();
this.interval_id = setInterval(() => {
this.getTasks();
}, this.tasks_check_interval);
}
getTasks(): void {
this.postsService.getTasks().subscribe(res => {
if (this.tasks) {
if (JSON.stringify(this.tasks) === JSON.stringify(res['tasks'])) return;
for (const task of res['tasks']) {
const task_index = this.tasks.map(t => t.key).indexOf(task['key']);
this.tasks[task_index] = task;
}
this.dataSource = new MatTableDataSource<Task>(this.tasks);
} else {
this.tasks = res['tasks'];
this.dataSource = new MatTableDataSource<Task>(this.tasks);
this.dataSource.paginator = this.paginator;
this.dataSource.sort = this.sort;
}
});
}
runTask(task_key: string): void {
this.postsService.runTask(task_key).subscribe(res => {
this.getTasks();
this.getDBBackups();
if (res['success']) this.postsService.openSnackBar($localize`Successfully ran task!`);
else this.postsService.openSnackBar($localize`Failed to run task!`);
}, err => {
this.postsService.openSnackBar($localize`Failed to run task!`);
console.error(err);
});
}
confirmTask(task_key: string): void {
this.postsService.confirmTask(task_key).subscribe(res => {
this.getTasks();
if (res['success']) this.postsService.openSnackBar($localize`Successfully confirmed task!`);
else this.postsService.openSnackBar($localize`Failed to confirm task!`);
}, err => {
this.postsService.openSnackBar($localize`Failed to confirm task!`);
console.error(err);
});
}
scheduleTask(task: any): void {
// open dialog
const dialogRef = this.dialog.open(UpdateTaskScheduleDialogComponent, {
data: {
task: task
}
});
dialogRef.afterClosed().subscribe(schedule => {
if (schedule || schedule === null) {
this.postsService.updateTaskSchedule(task['key'], schedule).subscribe(res => {
this.getTasks();
console.log(res);
});
}
});
}
getDBBackups(): void {
this.postsService.getDBBackups().subscribe(res => {
this.db_backups = res['db_backups'];
});
}
openRestoreDBBackupDialog(): void {
this.dialog.open(RestoreDbDialogComponent, {
data: {
db_backups: this.db_backups
},
width: '80vw'
})
}
resetTasks(): void {
const dialogRef = this.dialog.open(ConfirmDialogComponent, {
data: {
dialogTitle: $localize`Reset tasks`,
dialogText: $localize`Would you like to reset your tasks? All your schedules will be removed as well.`,
submitText: $localize`Reset`,
warnSubmitColor: true
}
});
dialogRef.afterClosed().subscribe(confirmed => {
if (confirmed) {
this.postsService.resetTasks().subscribe(res => {
if (res['success']) {
this.postsService.openSnackBar($localize`Tasks successfully reset!`);
} else {
this.postsService.openSnackBar($localize`Failed to reset tasks!`);
}
}, err => {
this.postsService.openSnackBar($localize`Failed to reset tasks!`);
console.error(err);
});
}
});
}
}
export interface Task {
key: string;
title: string;
last_ran: number;
last_confirmed: number;
running: boolean;
confirming: boolean;
data: unknown;
}

@ -0,0 +1,29 @@
<h4 mat-dialog-title><ng-container i18n="Restore DB from backup">Restore DB from backup</ng-container></h4>
<mat-dialog-content>
<mat-selection-list [multiple]="false" [(ngModel)]="selected_backup">
<mat-list-option *ngFor="let db_backup of db_backups" [value]="db_backup.name" [matTooltip]="db_backup.name">
<div class="container-fluid">
<div class="row">
<div class="col-4">
{{db_backup.timestamp*1000 | date: 'short'}}
</div>
<div class="col-4">
{{(db_backup.size/1000).toFixed(2)}} kB
</div>
<div class="col-4" style="text-transform: capitalize;">
{{db_backup.source}}
</div>
</div>
</div>
</mat-list-option>
</mat-selection-list>
</mat-dialog-content>
<mat-dialog-actions>
<button mat-button mat-dialog-close><ng-container i18n="Restore DB cancel button">Cancel</ng-container></button>
<button mat-button [disabled]="restoring" (click)="restoreClicked()" [disabled]="!selected_backup || selected_backup.length !== 1"><ng-container i18n="Restore button">Restore</ng-container></button>
<div class="mat-spinner" *ngIf="restoring">
<mat-spinner [diameter]="25"></mat-spinner>
</div>
</mat-dialog-actions>

@ -0,0 +1,25 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { RestoreDbDialogComponent } from './restore-db-dialog.component';
describe('RestoreDbDialogComponent', () => {
let component: RestoreDbDialogComponent;
let fixture: ComponentFixture<RestoreDbDialogComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ RestoreDbDialogComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(RestoreDbDialogComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

@ -0,0 +1,51 @@
import { Component, Inject, OnInit } from '@angular/core';
import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
import { PostsService } from 'app/posts.services';
@Component({
selector: 'app-restore-db-dialog',
templateUrl: './restore-db-dialog.component.html',
styleUrls: ['./restore-db-dialog.component.scss']
})
export class RestoreDbDialogComponent implements OnInit {
db_backups = [];
selected_backup = null;
restoring = false;
constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef<RestoreDbDialogComponent>, private postsService: PostsService) {
if (this.data?.db_backups) {
this.db_backups = this.data.db_backups;
}
this.getDBBackups();
}
ngOnInit(): void {
}
getDBBackups(): void {
this.postsService.getDBBackups().subscribe(res => {
this.db_backups = res['db_backups'];
});
}
restoreClicked(): void {
this.restoring = true;
if (this.selected_backup.length !== 1) return;
this.postsService.restoreDBBackup(this.selected_backup[0]).subscribe(res => {
this.restoring = false;
if (res['success']) {
this.postsService.openSnackBar('Database successfully restored!');
this.dialogRef.close();
} else {
this.postsService.openSnackBar('Failed to restore database! See logs for more info.');
}
}, err => {
this.restoring = false;
this.postsService.openSnackBar('Failed to restore database! See browser console for more info.');
console.error(err);
});
}
}

@ -0,0 +1,53 @@
<h4 mat-dialog-title><ng-container i18n="Update task schedule">Update task schedule</ng-container></h4>
<mat-dialog-content>
<div class="container-fluid">
<div class="row">
<div class="col-12 mt-3">
<mat-checkbox [(ngModel)]="enabled"><ng-container i18n="Enabled">Enabled</ng-container></mat-checkbox>
</div>
<div class="col-12 mt-2">
<mat-checkbox [(ngModel)]="recurring" [disabled]="!enabled"><ng-container i18n="Recurring">Recurring</ng-container></mat-checkbox>
</div>
<div class="col-12 mt-2" *ngIf="recurring">
<mat-form-field>
<mat-select placeholder="Interval" [(ngModel)]="interval" [disabled]="!enabled">
<mat-option value="weekly">Weekly</mat-option>
<mat-option value="daily">Daily</mat-option>
</mat-select>
</mat-form-field>
</div>
<div *ngIf="!recurring" class="col-12 mt-2">
<mat-form-field>
<mat-label>Choose a date</mat-label>
<input [(ngModel)]="date" [min]="today" matInput [matDatepicker]="picker" [disabled]="!enabled">
<mat-datepicker-toggle matSuffix [for]="picker"></mat-datepicker-toggle>
<mat-datepicker #picker></mat-datepicker>
</mat-form-field>
</div>
<div *ngIf="recurring && interval === 'weekly'" class="col-12 mt-2">
<mat-button-toggle-group [(ngModel)]="days_of_week" [multiple]="true" [disabled]="!enabled" aria-label="Week day">
<!-- TODO: support translation -->
<mat-button-toggle [value]="0">M</mat-button-toggle>
<mat-button-toggle [value]="1">T</mat-button-toggle>
<mat-button-toggle [value]="2">W</mat-button-toggle>
<mat-button-toggle [value]="3">T</mat-button-toggle>
<mat-button-toggle [value]="4">F</mat-button-toggle>
<mat-button-toggle [value]="5">S</mat-button-toggle>
<mat-button-toggle [value]="6">S</mat-button-toggle>
</mat-button-toggle-group>
</div>
<div class="col-12 mt-2">
<mat-form-field>
<mat-label>Time</mat-label>
<input type="time" matInput [(ngModel)]="time" [disabled]="!enabled">
</mat-form-field>
</div>
</div>
</div>
</mat-dialog-content>
<mat-dialog-actions>
<button mat-button mat-dialog-close><ng-container i18n="Update task schedule cancel button">Cancel</ng-container></button>
<button mat-button (click)="updateTaskSchedule()"><ng-container i18n="Update button">Update</ng-container></button>
</mat-dialog-actions>

@ -0,0 +1,25 @@
import { ComponentFixture, TestBed } from '@angular/core/testing';
import { UpdateTaskScheduleDialogComponent } from './update-task-schedule-dialog.component';
describe('UpdateTaskScheduleDialogComponent', () => {
let component: UpdateTaskScheduleDialogComponent;
let fixture: ComponentFixture<UpdateTaskScheduleDialogComponent>;
beforeEach(async () => {
await TestBed.configureTestingModule({
declarations: [ UpdateTaskScheduleDialogComponent ]
})
.compileComponents();
});
beforeEach(() => {
fixture = TestBed.createComponent(UpdateTaskScheduleDialogComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
});

@ -0,0 +1,83 @@
import { Component, Inject, OnInit } from '@angular/core';
import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
import { Schedule } from 'api-types';
import { PostsService } from 'app/posts.services';
@Component({
selector: 'app-update-task-schedule-dialog',
templateUrl: './update-task-schedule-dialog.component.html',
styleUrls: ['./update-task-schedule-dialog.component.scss']
})
export class UpdateTaskScheduleDialogComponent implements OnInit {
enabled = true;
recurring = false;
days_of_week = [];
interval = 'daily';
time = null;
date = null;
today = new Date();
constructor(@Inject(MAT_DIALOG_DATA) public data: any, private dialogRef: MatDialogRef<UpdateTaskScheduleDialogComponent>, private postsService: PostsService) {
this.processTask(this.data.task);
this.postsService.getTask(this.data.task.key).subscribe(res => {
this.processTask(res['task']);
});
}
ngOnInit(): void {
}
processTask(task) {
if (!task['schedule']) {
this.enabled = false;
return;
}
const schedule: Schedule = task['schedule'];
this.recurring = schedule['type'] === Schedule.type.RECURRING;
if (this.recurring) {
this.time = `${schedule['data']['hour']}:${schedule['data']['minute']}`;
if (schedule['data']['dayOfWeek']) {
this.days_of_week = schedule['data']['dayOfWeek'];
this.interval = 'weekly';
} else {
this.interval = 'daily';
}
} else {
const schedule_date = new Date(schedule['data']['timestamp']);
this.time = `${schedule_date.getHours()}:${schedule_date.getMinutes()}`
this.date = schedule_date;
}
}
updateTaskSchedule(): void {
if (!this.enabled) {
this.dialogRef.close(null);
return;
}
if (!this.time) {
// needs time!
}
const hours = parseInt(this.time.split(':')[0]);
const minutes = parseInt(this.time.split(':')[1]);
const schedule: Schedule = {type: this.recurring ? Schedule.type.RECURRING : Schedule.type.TIMESTAMP, data: null};
if (this.recurring) {
schedule['data'] = {hour: hours, minute: minutes};
if (this.interval === 'weekly') {
schedule['data']['dayOfWeek'] = this.days_of_week;
}
} else {
this.date.setHours(hours, minutes);
console.log(this.date);
schedule['data'] = {timestamp: this.date.getTime()};
}
this.dialogRef.close(schedule);
}
}

@ -33,8 +33,7 @@
<button *ngIf="(!postsService.isLoggedIn || postsService.permissions.includes('sharing')) && !auto" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button> <button *ngIf="(!postsService.isLoggedIn || postsService.permissions.includes('sharing')) && !auto" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button>
</ng-container> </ng-container>
<ng-container *ngIf="db_file"> <ng-container *ngIf="db_file">
<button (click)="downloadVideo()" [disabled]="downloading" mat-icon-button><mat-icon>cloud_download</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button> <button (click)="downloadFile()" [disabled]="downloading" mat-icon-button><mat-icon>cloud_download</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button>
<button (click)="downloadFile()" [disabled]="downloading" mat-icon-button><mat-icon>save</mat-icon><mat-spinner *ngIf="downloading" class="spinner" [diameter]="35"></mat-spinner></button>
<button *ngIf="type !== 'subscription' && (!postsService.isLoggedIn || postsService.permissions.includes('sharing'))" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button> <button *ngIf="type !== 'subscription' && (!postsService.isLoggedIn || postsService.permissions.includes('sharing'))" (click)="openShareDialog()" mat-icon-button><mat-icon>share</mat-icon></button>
<button (click)="openFileInfoDialog()" *ngIf="db_file" mat-icon-button><mat-icon>info</mat-icon></button> <button (click)="openFileInfoDialog()" *ngIf="db_file" mat-icon-button><mat-icon>info</mat-icon></button>
</ng-container> </ng-container>

@ -9,7 +9,6 @@ import { ShareMediaDialogComponent } from '../dialogs/share-media-dialog/share-m
import { FileType } from '../../api-types'; import { FileType } from '../../api-types';
import { TwitchChatComponent } from 'app/components/twitch-chat/twitch-chat.component'; import { TwitchChatComponent } from 'app/components/twitch-chat/twitch-chat.component';
import { VideoInfoDialogComponent } from 'app/dialogs/video-info-dialog/video-info-dialog.component'; import { VideoInfoDialogComponent } from 'app/dialogs/video-info-dialog/video-info-dialog.component';
import { HttpClient, HttpParams } from '@angular/common/http';
export interface IMedia { export interface IMedia {
title: string; title: string;
@ -110,7 +109,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
} }
constructor(public postsService: PostsService, private route: ActivatedRoute, private dialog: MatDialog, private router: Router, constructor(public postsService: PostsService, private route: ActivatedRoute, private dialog: MatDialog, private router: Router,
public snackBar: MatSnackBar, private cdr: ChangeDetectorRef, private http: HttpClient) { public snackBar: MatSnackBar, private cdr: ChangeDetectorRef) {
} }
processConfig() { processConfig() {
@ -315,10 +314,8 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
downloadFile() { downloadFile() {
const filename = this.playlist[0].title; const filename = this.playlist[0].title;
const ext = (this.playlist[0].type === 'audio/mp3') ? '.mp3' : '.mp4'; const ext = (this.playlist[0].type === 'audio/mp3') ? '.mp3' : '.mp4';
const type = this.playlist[0].type;
const url = this.playlist[0].url;
this.downloading = true; this.downloading = true;
this.postsService.downloadFileFromServer(this.uid, this.uuid, this.sub_id, url, type).subscribe(res => { this.postsService.downloadFileFromServer(this.uid, this.uuid).subscribe(res => {
this.downloading = false; this.downloading = false;
const blob: Blob = res; const blob: Blob = res;
saveAs(blob, filename + ext); saveAs(blob, filename + ext);
@ -328,24 +325,6 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy {
}); });
} }
downloadVideo() {
const filename = this.currentItem.label;
const ext = (this.currentItem.type === 'audio/mp3') ? '.mp3' : '.mp4';
// const type = this.currentItem.type;
const url = this.currentItem.src;
this.downloading = true;
this.http.get(url, {
responseType: 'blob'
}).subscribe(res => {
const blob: Blob = res;
this.downloading = false;
saveAs(blob, filename + ext);
}, err => {
console.log(err);
this.downloading = false;
})
}
playlistPostCreationHandler(playlistID) { playlistPostCreationHandler(playlistID) {
// changes the route without moving from the current view or // changes the route without moving from the current view or
// triggering a navigation event // triggering a navigation event

@ -90,6 +90,12 @@ import {
DBInfoResponse, DBInfoResponse,
GetFileFormatsRequest, GetFileFormatsRequest,
GetFileFormatsResponse, GetFileFormatsResponse,
GetTaskRequest,
GetTaskResponse,
UpdateTaskScheduleRequest,
UpdateTaskDataRequest,
RestoreDBBackupRequest,
Schedule,
} from '../api-types'; } from '../api-types';
import { isoLangs } from './settings/locales_list'; import { isoLangs } from './settings/locales_list';
import { Title } from '@angular/platform-browser'; import { Title } from '@angular/platform-browser';
@ -351,15 +357,12 @@ export class PostsService implements CanActivate {
return this.http.post<GetAllFilesResponse>(this.path + 'getAllFiles', {sort: sort, range: range, text_search: text_search, file_type_filter: file_type_filter}, this.httpOptions); return this.http.post<GetAllFilesResponse>(this.path + 'getAllFiles', {sort: sort, range: range, text_search: text_search, file_type_filter: file_type_filter}, this.httpOptions);
} }
downloadFileFromServer(uid: string, uuid: string = null, sub_id: string = null, url: string = null, type: string = null) { downloadFileFromServer(uid: string, uuid: string = null) {
const body: DownloadFileRequest = { const body: DownloadFileRequest = {
uid: uid, uid: uid,
uuid: uuid, uuid: uuid
sub_id: sub_id,
url: url,
type: type
}; };
return this.http.post(this.path + 'downloadFile', body, {responseType: 'blob', params: this.httpOptions.params}); return this.http.post(this.path + 'downloadFileFromServer', body, {responseType: 'blob', params: this.httpOptions.params});
} }
getFullTwitchChat(id, type, uuid = null, sub = null) { getFullTwitchChat(id, type, uuid = null, sub = null) {
@ -544,38 +547,85 @@ export class PostsService implements CanActivate {
return this.http.post<GetDownloadResponse>(this.path + 'download', body, this.httpOptions); return this.http.post<GetDownloadResponse>(this.path + 'download', body, this.httpOptions);
} }
pauseDownload(download_uid) { pauseDownload(download_uid: string) {
return this.http.post<SuccessObject>(this.path + 'pauseDownload', {download_uid: download_uid}, this.httpOptions); const body: GetDownloadRequest = {download_uid: download_uid};
return this.http.post<SuccessObject>(this.path + 'pauseDownload', body, this.httpOptions);
} }
pauseAllDownloads() { pauseAllDownloads() {
return this.http.post<SuccessObject>(this.path + 'pauseAllDownloads', {}, this.httpOptions); return this.http.post<SuccessObject>(this.path + 'pauseAllDownloads', {}, this.httpOptions);
} }
resumeDownload(download_uid) { resumeDownload(download_uid: string) {
return this.http.post<SuccessObject>(this.path + 'resumeDownload', {download_uid: download_uid}, this.httpOptions); const body: GetDownloadRequest = {download_uid: download_uid};
return this.http.post<SuccessObject>(this.path + 'resumeDownload', body, this.httpOptions);
} }
resumeAllDownloads() { resumeAllDownloads() {
return this.http.post<SuccessObject>(this.path + 'resumeAllDownloads', {}, this.httpOptions); return this.http.post<SuccessObject>(this.path + 'resumeAllDownloads', {}, this.httpOptions);
} }
restartDownload(download_uid) { restartDownload(download_uid: string) {
return this.http.post<SuccessObject>(this.path + 'restartDownload', {download_uid: download_uid}, this.httpOptions); const body: GetDownloadRequest = {download_uid: download_uid};
return this.http.post<SuccessObject>(this.path + 'restartDownload', body, this.httpOptions);
} }
cancelDownload(download_uid) { cancelDownload(download_uid: string) {
return this.http.post<SuccessObject>(this.path + 'cancelDownload', {download_uid: download_uid}, this.httpOptions); const body: GetDownloadRequest = {download_uid: download_uid};
return this.http.post<SuccessObject>(this.path + 'cancelDownload', body, this.httpOptions);
} }
clearDownload(download_uid) { clearDownload(download_uid: string) {
return this.http.post<SuccessObject>(this.path + 'clearDownload', {download_uid: download_uid}, this.httpOptions); const body: GetDownloadRequest = {download_uid: download_uid};
return this.http.post<SuccessObject>(this.path + 'clearDownload', body, this.httpOptions);
} }
clearFinishedDownloads() { clearFinishedDownloads() {
return this.http.post<SuccessObject>(this.path + 'clearFinishedDownloads', {}, this.httpOptions); return this.http.post<SuccessObject>(this.path + 'clearFinishedDownloads', {}, this.httpOptions);
} }
getTasks() {
return this.http.post<SuccessObject>(this.path + 'getTasks', {}, this.httpOptions);
}
resetTasks() {
return this.http.post<SuccessObject>(this.path + 'resetTasks', {}, this.httpOptions);
}
getTask(task_key: string) {
const body: GetTaskRequest = {task_key: task_key};
return this.http.post<GetTaskResponse>(this.path + 'getTask', body, this.httpOptions);
}
runTask(task_key: string) {
const body: GetTaskRequest = {task_key: task_key};
return this.http.post<SuccessObject>(this.path + 'runTask', body, this.httpOptions);
}
confirmTask(task_key: string) {
const body: GetTaskRequest = {task_key: task_key};
return this.http.post<SuccessObject>(this.path + 'confirmTask', body, this.httpOptions);
}
updateTaskSchedule(task_key: string, schedule: Schedule) {
const body: UpdateTaskScheduleRequest = {task_key: task_key, new_schedule: schedule};
return this.http.post<SuccessObject>(this.path + 'updateTaskSchedule', body, this.httpOptions);
}
updateTaskData(task_key: string, data: any) {
const body: UpdateTaskDataRequest = {task_key: task_key, new_data: data};
return this.http.post<SuccessObject>(this.path + 'updateTaskData', body, this.httpOptions);
}
getDBBackups() {
return this.http.post<SuccessObject>(this.path + 'getDBBackups', {}, this.httpOptions);
}
restoreDBBackup(file_name: string) {
const body: RestoreDBBackupRequest = {file_name: file_name};
return this.http.post<SuccessObject>(this.path + 'restoreDBBackup', body, this.httpOptions);
}
getVersionInfo() { getVersionInfo() {
return this.http.get<VersionInfoResponse>(this.path + 'versionInfo', this.httpOptions); return this.http.get<VersionInfoResponse>(this.path + 'versionInfo', this.httpOptions);
} }

Loading…
Cancel
Save