aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cleaned up app.js backend code

Browse Source
download-manager
Isaac Abadi 4 years ago
parent 0360469c5a
commit 8cc653787f
3 changed files with 38 additions and 624 deletions
Show Stats Download Patch File Download Diff File

656
backend/app.js
Unescape Escape View File

@ -5,7 +5,6 @@ const auth_api = require('./authentication/auth');
const winston = require('winston');
const path = require('path');
const compression = require('compression');
const glob = require("glob")
const multer = require('multer');
const express = require("express");
const bodyParser = require("body-parser");
@ -13,13 +12,10 @@ const archiver = require('archiver');
const unzipper = require('unzipper');
const db_api = require('./db');
const utils = require('./utils')
const mergeFiles = require('merge-files');
const low = require('lowdb')
const ProgressBar = require('progress');
const NodeID3 = require('node-id3')
const fetch = require('node-fetch');
const URL = require('url').URL;
const url_api = require('url');
const CONSTS = require('./consts')
const read_last_lines = require('read-last-lines');
const ps = require('ps-node');
@ -42,7 +38,6 @@ var app = express();
// database setup
const FileSync = require('lowdb/adapters/FileSync');
const config = require('./config.js');
const adapter = new FileSync('./appdata/db.json');
const db = low(adapter)
@ -105,17 +100,16 @@ users_db.defaults(
).write();
// config values
var backendPort = null;
var audioFolderPath = null;
var videoFolderPath = null;
var useDefaultDownloadingAgent = null;
var customDownloadingAgent = null;
var allowSubscriptions = null;
var archivePath = path.join(__dirname, 'appdata', 'archives');
let url = null;
let backendPort = null;
let useDefaultDownloadingAgent = null;
let customDownloadingAgent = null;
let allowSubscriptions = null;
let archivePath = path.join(__dirname, 'appdata', 'archives');
// other needed values
var url_domain = null;
var updaterStatus = null;
let url_domain = null;
let updaterStatus = null;
const concurrentStreams = {};
@ -161,8 +155,6 @@ if (writeConfigMode) {
loadConfig();
}
var downloads = [];
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
@ -172,16 +164,6 @@ app.use(auth_api.passport.initialize());
// actual functions
async function checkMigrations() {
// 3.5->3.6 migration
const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value();
if (!files_to_db_migration_complete) {
logger.info('Beginning migration: 3.5->3.6+')
const success = await runFilesToDBMigration()
if (success) { logger.info('3.5->3.6+ migration complete!'); }
else { logger.error('Migration failed: 3.5->3.6+'); }
}
// 4.1->4.2 migration
const simplified_db_migration_complete = db.get('simplified_db_migration_complete').value();
@ -212,38 +194,6 @@ async function checkMigrations() {
return true;
}
async function runFilesToDBMigration() {
try {
let mp3s = await getMp3s();
let mp4s = await getMp4s();
for (let i = 0; i < mp3s.length; i++) {
let file_obj = mp3s[i];
const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
}
}
for (let i = 0; i < mp4s.length; i++) {
let file_obj = mp4s[i];
const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
db_api.registerFileDB(file_obj.id + '.mp4', 'video');
}
}
// sets migration to complete
db.set('files_to_db_migration_complete', true).write();
return true;
} catch(err) {
logger.error(err);
return false;
}
}
async function simplifyDBFileStructure() {
// back up db files
const old_db_file = fs.readJSONSync('./appdata/db.json');
@ -517,9 +467,10 @@ async function getLatestVersion() {
async function killAllDownloads() {
const lookupAsync = promisify(ps.lookup);
let resultList = null;
try {
await lookupAsync({
resultList = await lookupAsync({
command: 'youtube-dl'
});
} catch (err) {
@ -609,9 +560,6 @@ async function loadConfig() {
await db_api.importUnregisteredFiles();
// load in previous downloads
downloads = await db_api.getRecords('downloads');
// start the server here
startServer();
@ -621,9 +569,6 @@ async function loadConfig() {
function loadConfigValues() {
url = !debugMode ? config_api.getConfigItem('ytdl_url') : 'http://localhost:4200';
backendPort = config_api.getConfigItem('ytdl_port');
audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path');
videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path');
downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode');
useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent');
customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent');
allowSubscriptions = config_api.getConfigItem('ytdl_allow_subscriptions');
@ -726,420 +671,17 @@ function generateEnvVarConfigItem(key) {
return {key: key, value: process['env'][key]};
}
/**
* @param {'audio' | 'video'} type
* @param {string[]} fileNames
*/
async function getAudioOrVideoInfos(type, fileNames) {
let result = await Promise.all(fileNames.map(async fileName => {
let fileLocation = videoFolderPath+fileName;
if (type === 'audio') {
fileLocation += '.mp3.info.json';
} else if (type === 'video') {
fileLocation += '.info.json';
}
if (await fs.pathExists(fileLocation)) {
let data = await fs.readFile(fileLocation);
try {
return JSON.parse(data);
} catch (e) {
let suffix;
if (type === 'audio') {
suffix += '.mp3';
} else if (type === 'video') {
suffix += '.mp4';
}
logger.error(`Could not find info for file ${fileName}${suffix}`);
}
}
return null;
}));
return result.filter(data => data != null);
}
// downloads
async function downloadFileByURL_exec(url, type, options, sessionID = null) {
return new Promise(async resolve => {
var date = Date.now();
// audio / video specific vars
var is_audio = type === 'audio';
var ext = is_audio ? '.mp3' : '.mp4';
var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
let category = null;
// prepend with user if needed
if (options.user) {
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_path = path.join(usersFileFolder, options.user, type);
fs.ensureDirSync(user_path);
fileFolderPath = user_path + path.sep;
options.customFileFolderPath = fileFolderPath;
}
options.downloading_method = 'exec';
let downloadConfig = await generateArgs(url, type, options);
// adds download to download helper
const download_uid = uuid();
const session = sessionID ? sessionID : 'undeclared';
let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session);
if (!session_downloads) {
session_downloads = {session_id: session};
downloads.push(session_downloads);
}
session_downloads[download_uid] = {
uid: download_uid,
ui_uid: options.ui_uid,
downloading: true,
complete: false,
url: url,
type: type,
percent_complete: 0,
is_playlist: url.includes('playlist'),
timestamp_start: Date.now(),
filesize: null
};
const download = session_downloads[download_uid];
updateDownloads();
let download_checker = null;
// get video info prior to download
let info = await getVideoInfoByURL(url, downloadConfig, download);
if (!info && url.includes('youtu')) {
resolve(false);
return;
} else if (info) {
// check if it fits into a category. If so, then get info again using new downloadConfig
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
downloadConfig = await generateArgs(url, type, options);
info = await getVideoInfoByURL(url, downloadConfig, download);
}
// store info in download for future use
if (Array.isArray(info)) {
download['fileNames'] = [];
for (let info_obj of info) download['fileNames'].push(info_obj['_filename']);
} else {
download['_filename'] = info['_filename'];
}
download['filesize'] = utils.getExpectedFileSize(info);
download_checker = setInterval(() => checkDownloadPercent(download), 1000);
}
// download file
youtubedl.exec(url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored)
download['downloading'] = false;
download['timestamp_end'] = Date.now();
var file_objs = [];
let new_date = Date.now();
let difference = (new_date - date)/1000;
logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
if (err) {
logger.error(err.stderr);
download['error'] = err.stderr;
updateDownloads();
resolve(false);
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
download['error'] = 'No output. Check if video already exists in your archive.';
logger.warn(`No output received for video download, check if it exists in your archive.`)
updateDownloads();
resolve(false);
return;
}
var file_names = [];
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
// get filepath with no extension
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
var full_file_path = filepath_no_extension + ext;
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, options.user);
}
// renames file if necessary due to bug
if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) {
try {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) {
}
}
if (type === 'audio') {
let tags = {
title: output_json['title'],
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
}
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
}
if (options.cropFileSettings) {
await utils.cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
}
// registers file in DB
const file_obj = await db_api.registerFileDB2(full_file_path, type, options.user, category, null, options.cropFileSettings);
// TODO: remove the following line
if (file_name) file_names.push(file_name);
file_objs.push(file_obj);
}
let is_playlist = file_names.length > 1;
if (options.merged_string !== null && options.merged_string !== undefined) {
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
let diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
download['complete'] = true;
download['fileNames'] = is_playlist ? file_names : [full_file_path]
updateDownloads();
let container = null;
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, options.user);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
logger.error('Downloaded file failed to result in metadata object.');
}
resolve({
file_uids: file_objs.map(file_obj => file_obj.uid),
container: container
});
}
});
});
}
async function generateArgs(url, type, options) {
var videopath = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
var globalArgs = config_api.getConfigItem('ytdl_custom_args');
let useCookies = config_api.getConfigItem('ytdl_use_cookies');
var is_audio = type === 'audio';
var fileFolderPath = is_audio ? audioFolderPath : videoFolderPath;
if (options.customFileFolderPath) fileFolderPath = options.customFileFolderPath;
var customArgs = options.customArgs;
var customOutput = options.customOutput;
var customQualityConfiguration = options.customQualityConfiguration;
// video-specific args
var selectedHeight = options.selectedHeight;
// audio-specific args
var maxBitrate = options.maxBitrate;
var youtubeUsername = options.youtubeUsername;
var youtubePassword = options.youtubePassword;
let downloadConfig = null;
// TODO: fix
let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
const is_youtube = url.includes('youtu');
if (!is_audio && !is_youtube) {
// tiktok videos fail when using the default format
qualityPath = null;
} else if (!is_audio && !is_youtube && (url.includes('reddit') || url.includes('pornhub'))) {
qualityPath = ['-f', 'bestvideo+bestaudio']
}
if (customArgs) {
downloadConfig = customArgs.split(',,');
} else {
if (customQualityConfiguration) {
qualityPath = ['-f', customQualityConfiguration];
} else if (selectedHeight && selectedHeight !== '' && !is_audio) {
qualityPath = ['-f', `'(mp4)[height=${selectedHeight}'`];
} else if (is_audio) {
qualityPath = ['--audio-quality', maxBitrate ? maxBitrate : '0']
}
if (customOutput) {
customOutput = options.noRelativePath ? customOutput : path.join(fileFolderPath, customOutput);
downloadConfig = ['-o', `${customOutput}.%(ext)s`, '--write-info-json', '--print-json'];
} else {
downloadConfig = ['-o', path.join(fileFolderPath, videopath + (is_audio ? '.%(ext)s' : '.mp4')), '--write-info-json', '--print-json'];
}
if (qualityPath && options.downloading_method === 'exec') downloadConfig.push(...qualityPath);
if (is_audio && !options.skip_audio_args) {
downloadConfig.push('-x');
downloadConfig.push('--audio-format', 'mp3');
}
if (youtubeUsername && youtubePassword) {
downloadConfig.push('--username', youtubeUsername, '--password', youtubePassword);
}
if (useCookies) {
if (await fs.pathExists(path.join(__dirname, 'appdata', 'cookies.txt'))) {
downloadConfig.push('--cookies', path.join('appdata', 'cookies.txt'));
} else {
logger.warn('Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.');
}
}
if (!useDefaultDownloadingAgent && customDownloadingAgent) {
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
}
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = options.user ? path.join(fileFolderPath, 'archives') : archivePath;
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
// create archive file if it doesn't exist
if (!(await fs.pathExists(archive_path))) {
await fs.close(await fs.open(archive_path, 'w'));
}
let blacklist_path = options.user ? path.join(fileFolderPath, 'archives', `blacklist_${type}.txt`) : path.join(archivePath, `blacklist_${type}.txt`);
// create blacklist file if it doesn't exist
if (!(await fs.pathExists(blacklist_path))) {
await fs.close(await fs.open(blacklist_path, 'w'));
}
let merged_path = path.join(fileFolderPath, `merged_${type}.txt`);
await fs.ensureFile(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
let status = await mergeFiles(inputPathList, merged_path);
options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail');
}
if (globalArgs && globalArgs !== '') {
// adds global args
if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
// if global args has an output, replce the original output with that of global args
const original_output_index = downloadConfig.indexOf('-o');
downloadConfig.splice(original_output_index, 2);
}
downloadConfig = downloadConfig.concat(globalArgs.split(',,'));
}
const rate_limit = config_api.getConfigItem('ytdl_download_rate_limit');
if (rate_limit && downloadConfig.indexOf('-r') === -1 && downloadConfig.indexOf('--limit-rate') === -1) {
downloadConfig.push('-r', rate_limit);
}
const default_downloader = utils.getCurrentDownloader() || config_api.getConfigItem('ytdl_default_downloader');
if (default_downloader === 'yt-dlp') {
downloadConfig.push('--no-clean-infojson');
}
}
// filter out incompatible args
downloadConfig = filterArgs(downloadConfig, is_audio);
logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
return downloadConfig;
}
async function getVideoInfoByURL(url, args = [], download = null) {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
new_args.splice(archiveArgIndex, 2);
}
// actually get info
youtubedl.getInfo(url, new_args, (err, output) => {
if (output) {
resolve(output);
} else {
logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`);
if (err.stderr) {
logger.error(`${err.stderr}`)
}
if (download) {
download['error'] = `Failed pre-check for video info: ${err}`;
updateDownloads();
}
resolve(null);
}
});
});
}
function filterArgs(args, isAudio) {
const video_only_args = ['--add-metadata', '--embed-subs', '--xattrs'];
const audio_only_args = ['-x', '--extract-audio', '--embed-thumbnail'];
const args_to_remove = isAudio ? video_only_args : audio_only_args;
return args.filter(x => !args_to_remove.includes(x));
}
// currently only works for single urls
async function getUrlInfos(urls) {
async function getUrlInfos(url) {
let startDate = Date.now();
let result = [];
return new Promise(resolve => {
youtubedl.exec(urls.join(' '), ['--dump-json'], {maxBuffer: Infinity}, (err, output) => {
youtubedl.exec(url, ['--dump-json'], {maxBuffer: Infinity}, (err, output) => {
let new_date = Date.now();
let difference = (new_date - startDate)/1000;
logger.debug(`URL info retrieval delay: ${difference} seconds.`);
if (err) {
logger.error(`Error during parsing: ${err}`);
logger.error(`Error during retrieving formats for ${url}: ${err}`);
resolve(null);
}
let try_putput = null;
@ -1147,54 +689,13 @@ async function getUrlInfos(urls) {
try_putput = JSON.parse(output);
result = try_putput;
} catch(e) {
// probably multiple urls
logger.error('failed to parse for urls starting with ' + urls[0]);
// logger.info(output);
logger.error(`Failed to retrieve available formats for url: ${url}`);
}
resolve(result);
});
});
}
// download management functions
async function updateDownloads() {
await db_api.removeAllRecords('downloads');
if (downloads.length !== 0) await db_api.insertRecordsIntoTable('downloads', downloads);
}
function checkDownloadPercent(download) {
/*
This is more of an art than a science, we're just selecting files that start with the file name,
thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
be divided by the "total expected bytes."
*/
const file_id = download['file_id'];
// assume it's a playlist for logic reasons
const fileNames = Array.isArray(download['fileNames']) ? download['fileNames']
: [path.format(path.parse(utils.removeFileExtension(download['_filename'])))];
const resulting_file_size = download['filesize'];
if (!resulting_file_size) return;
let sum_size = 0;
glob(`{${fileNames.join(',')}, }*`, (err, files) => {
files.forEach(file => {
try {
const file_stats = fs.statSync(file);
if (file_stats && file_stats.size) {
sum_size += file_stats.size;
}
} catch (e) {
}
});
download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2);
});
}
// youtube-dl functions
async function startYoutubeDL() {
@ -1538,11 +1039,10 @@ app.get('/api/getMp4s', optionalJwt, async function(req, res) {
});
app.post('/api/getFile', optionalJwt, async function (req, res) {
var uid = req.body.uid;
var type = req.body.type;
var uuid = req.body.uuid;
const uid = req.body.uid;
const uuid = req.body.uuid;
var file = await db_api.getRecord('files', {uid: uid});
let file = await db_api.getRecord('files', {uid: uid});
if (uuid && !file['sharingEnabled']) file = null;
@ -1672,8 +1172,7 @@ app.post('/api/enableSharing', optionalJwt, async (req, res) => {
} else if (is_playlist) {
await db_api.updateRecord(`playlists`, {id: uid}, {sharingEnabled: true});
} else if (false) {
// TODO: Implement. Main blocker right now is subscription videos are not stored in the DB, they are searched for every
// time they are requested from the subscription directory.
// TODO: Implement.
} else {
// error
success = false;
@ -1693,16 +1192,8 @@ app.post('/api/disableSharing', optionalJwt, async function(req, res) {
var type = req.body.type;
var uid = req.body.uid;
var is_playlist = req.body.is_playlist;
let success = null;
// multi-user mode
if (req.isAuthenticated()) {
// if multi user mode, use this method instead
success = auth_api.changeSharingMode(req.user.uid, uid, is_playlist, false);
res.send({success: success});
return;
}
// single-user mode
try {
success = true;
if (!is_playlist && type !== 'subscription') {
@ -1904,53 +1395,11 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
if (subscription.name && !subscription.streamingOnly) {
var parsed_files = await db_api.getRecords('files', {sub_id: subscription.id}); // subscription.videos;
subscription['videos'] = parsed_files;
if (!parsed_files) {
parsed_files = [];
let base_path = null;
if (user_uid)
base_path = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions');
else
base_path = config_api.getConfigItem('ytdl_subscriptions_base_path');
let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
let files;
try {
files = await utils.recFindByExt(appended_base_path, 'mp4');
} catch(e) {
files = null;
logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path);
res.sendStatus(500);
return;
}
for (let i = 0; i < files.length; i++) {
let file = files[i];
var file_path = file.substring(appended_base_path.length, file.length);
var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = utils.getJSONMp4(id, appended_base_path);
if (!jsonobj) continue;
var title = jsonobj.title;
var thumbnail = jsonobj.thumbnail;
var duration = jsonobj.duration;
var url = jsonobj.webpage_url;
var uploader = jsonobj.uploader;
var upload_date = jsonobj.upload_date;
upload_date = `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}`;
var size = stats.size;
var isaudio = false;
var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date, jsonobj.description, jsonobj.view_count, jsonobj.height, jsonobj.abr);
parsed_files.push(file_obj);
}
} else {
// loop through files for extra processing
for (let i = 0; i < parsed_files.length; i++) {
const file = parsed_files[i];
// check if chat exists for twitch videos
if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
}
// loop through files for extra processing
for (let i = 0; i < parsed_files.length; i++) {
const file = parsed_files[i];
// check if chat exists for twitch videos
if (file && file['url'].includes('twitch.tv')) file['chat_exists'] = fs.existsSync(file['path'].substring(0, file['path'].length - 4) + '.twitch_chat.json');
}
res.send({
@ -2153,7 +1602,7 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
// delete generated zip file
fs.unlinkSync(file_path_to_download);
} catch(e) {
logger.error("Failed to remove file", file);
logger.error(`Failed to remove file after sending to client: ${file_path_to_download}`);
}
}
});
@ -2230,10 +1679,8 @@ app.get('/api/stream', optionalJwt, async (req, res) => {
const type = req.query.type;
const uuid = req.query.uuid ? req.query.uuid : (req.user ? req.user.uid : null);
const sub_id = req.query.sub_id;
const ext = type === 'audio' ? '.mp3' : '.mp4';
const mimetype = type === 'audio' ? 'audio/mp3' : 'video/mp4';
var head;
let optionalParams = url_api.parse(req.url,true).query;
let uid = decodeURIComponent(req.query.uid);
let file_path = null;
@ -2293,6 +1740,7 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => {
// Downloads management
app.get('/api/downloads', async (req, res) => {
const downloads = await db_api.getRecords('download_queue');
res.send({downloads: downloads});
});
@ -2308,33 +1756,8 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => {
}
});
app.post('/api/clearDownloads', async (req, res) => {
let success = false;
var delete_all = req.body.delete_all;
if (!req.body.session_id) req.body.session_id = 'undeclared';
var session_id = req.body.session_id;
var download_id = req.body.download_id;
if (delete_all) {
// delete all downloads
downloads = [];
success = true;
} else if (download_id) {
// delete just 1 download
const session_downloads = downloads.find(session => session['session_id'] === session_id);
if (session_downloads && session_downloads[download_id]) {
delete session_downloads[download_id];
success = true;
} else if (!session_downloads) {
logger.error(`Session ${session_id} has no downloads.`)
} else if (!session_downloads[download_id]) {
logger.error(`Download '${download_id}' for session '${session_id}' could not be found`);
}
} else if (session_id) {
// delete a session's downloads
downloads = downloads.filter(session => session['session_id'] !== session_id);
}
updateDownloads();
res.send({success: success, downloads: downloads});
app.post('/api/clearFinishedDownloads', async (req, res) => {
});
// logs management
@ -2342,7 +1765,7 @@ app.get('/api/thumbnail/:path', optionalJwt, async (req, res) => {
app.post('/api/logs', async function(req, res) {
let logs = null;
let lines = req.body.lines;
logs_path = path.join('appdata', 'logs', 'combined.log')
const logs_path = path.join('appdata', 'logs', 'combined.log')
if (await fs.pathExists(logs_path)) {
if (lines) logs = await read_last_lines.read(logs_path, lines);
else logs = await fs.readFile(logs_path, 'utf8');
@ -2357,8 +1780,8 @@ app.post('/api/logs', async function(req, res) {
});
app.post('/api/clearAllLogs', async function(req, res) {
logs_path = path.join('appdata', 'logs', 'combined.log');
logs_err_path = path.join('appdata', 'logs', 'error.log');
const logs_path = path.join('appdata', 'logs', 'combined.log');
const logs_err_path = path.join('appdata', 'logs', 'error.log');
let success = false;
try {
await Promise.all([
@ -2375,18 +1798,9 @@ app.post('/api/clearAllLogs', async function(req, res) {
});
});
app.post('/api/getVideoInfos', async (req, res) => {
let fileNames = req.body.fileNames;
let urlMode = !!req.body.urlMode;
let type = req.body.type;
let result = null;
if (!urlMode) {
if (type === 'audio' || type === 'video') {
result = await getAudioOrVideoInfos(type, fileNames);
}
} else {
result = await getUrlInfos(fileNames);
}
app.post('/api/getFileFormats', async (req, res) => {
let url = req.body.url;
let result = await getUrlInfos(url);
res.send({
result: result,
success: !!result
@ -2461,7 +1875,7 @@ app.post('/api/deleteUser', optionalJwt, async (req, res) => {
} else {
logger.error(`Could not find user with uid ${uid}`);
}
res.send({success: true});
res.send({success: success});
} catch (err) {
logger.error(err);
res.send({success: false});

2
src/app/main/main.component.ts
Unescape Escape View File

@ -623,7 +623,7 @@ export class MainComponent implements OnInit {
}
if (!(this.cachedAvailableFormats[url] && this.cachedAvailableFormats[url]['formats'])) {
this.cachedAvailableFormats[url]['formats_loading'] = true;
this.postsService.getFileInfo([url], 'irrelevant', true).subscribe(res => {
this.postsService.getFileFormats([url]).subscribe(res => {
this.cachedAvailableFormats[url]['formats_loading'] = false;
const infos = res['result'];
if (!infos || !infos.formats) {

4
src/app/posts.services.ts
Unescape Escape View File

@ -295,8 +295,8 @@ export class PostsService implements CanActivate {
return this.http.post(this.path + 'downloadArchive', {sub: sub}, {responseType: 'blob', params: this.httpOptions.params});
}
getFileInfo(fileNames, type, urlMode) {
return this.http.post(this.path + 'getVideoInfos', {fileNames: fileNames, type: type, urlMode: urlMode}, this.httpOptions);
getFileFormats(url) {
return this.http.post(this.path + 'getFileFormats', {url: url}, this.httpOptions);
}
getLogs(lines = 50) {

Write Preview
Loading…
Cancel
Save