Began scaffolding work for download manager

download-manager
Isaac Abadi 4 years ago
parent b1385f451b
commit 01b6e22f83

@ -0,0 +1,284 @@
async function collectInfo(download_uid) {
const download = db_api.getRecord('download_queue', {uid: download_uid});
const url = download['url'];
const type = download['type'];
const options = download['options'];
const args = download['args'];
// get video info prior to download
const info = await getVideoInfoByURL(url, args);
if (!info) {
// info failed, record error and pause download
}
// check if it fits into a category. If so, then get info again using new args
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
args = await generateArgs(url, type, options);
info = await getVideoInfoByURL(url, args);
// must update args
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args});
}
await db_api.updateRecord('download_queue', {uid: download_uid}, {remote_metadata: info});
}
async function downloadFileByURL_exec(url, type, options) {
const download = db_api.getRecord('download_queue', {uid: download_uid});
const url = download['url'];
const type = download['type'];
const options = download['options'];
const args = download['args'];
const category = download['category'];
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
if (options.user) {
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_path = path.join(usersFileFolder, options.user, type);
fs.ensureDirSync(user_path);
fileFolderPath = user_path + path.sep;
multiUserMode = {
user: options.user,
file_path: fileFolderPath
}
options.customFileFolderPath = fileFolderPath;
}
}
async function downloadFileByURL_exec_old(url, type, options, sessionID = null) {
return new Promise(async resolve => {
var date = Date.now();
// audio / video specific vars
var is_audio = type === 'audio';
var ext = is_audio ? '.mp3' : '.mp4';
var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
let category = null;
// prepend with user if needed
let multiUserMode = null;
if (options.user) {
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const user_path = path.join(usersFileFolder, options.user, type);
fs.ensureDirSync(user_path);
fileFolderPath = user_path + path.sep;
options.customFileFolderPath = fileFolderPath;
}
options.downloading_method = 'exec';
let downloadConfig = await generateArgs(url, type, options);
// adds download to download helper
const download_uid = uuid();
const session = sessionID ? sessionID : 'undeclared';
let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session);
if (!session_downloads) {
session_downloads = {session_id: session};
downloads.push(session_downloads);
}
session_downloads[download_uid] = {
uid: download_uid,
ui_uid: options.ui_uid,
downloading: true,
complete: false,
url: url,
type: type,
percent_complete: 0,
is_playlist: url.includes('playlist'),
timestamp_start: Date.now(),
filesize: null
};
const download = session_downloads[download_uid];
updateDownloads();
let download_checker = null;
// get video info prior to download
let info = await getVideoInfoByURL(url, downloadConfig, download);
if (!info && url.includes('youtu')) {
resolve(false);
return;
} else if (info) {
// check if it fits into a category. If so, then get info again using new downloadConfig
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if (category && category['custom_output']) {
options.customOutput = category['custom_output'];
options.noRelativePath = true;
downloadConfig = await generateArgs(url, type, options);
info = await getVideoInfoByURL(url, downloadConfig, download);
}
// store info in download for future use
if (Array.isArray(info)) {
download['fileNames'] = [];
for (let info_obj of info) download['fileNames'].push(info_obj['_filename']);
} else {
download['_filename'] = info['_filename'];
}
download['filesize'] = utils.getExpectedFileSize(info);
download_checker = setInterval(() => checkDownloadPercent(download), 1000);
}
// download file
youtubedl.exec(url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored)
download['downloading'] = false;
download['timestamp_end'] = Date.now();
var file_objs = [];
let new_date = Date.now();
let difference = (new_date - date)/1000;
logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
if (err) {
logger.error(err.stderr);
download['error'] = err.stderr;
updateDownloads();
resolve(false);
return;
} else if (output) {
if (output.length === 0 || output[0].length === 0) {
download['error'] = 'No output. Check if video already exists in your archive.';
logger.warn(`No output received for video download, check if it exists in your archive.`)
updateDownloads();
resolve(false);
return;
}
var file_names = [];
for (let i = 0; i < output.length; i++) {
let output_json = null;
try {
output_json = JSON.parse(output[i]);
} catch(e) {
output_json = null;
}
if (!output_json) {
continue;
}
// get filepath with no extension
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
var full_file_path = filepath_no_extension + ext;
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
&& config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) {
let vodId = url.split('twitch.tv/videos/')[1];
vodId = vodId.split('?')[0];
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, options.user);
}
// renames file if necessary due to bug
if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) {
try {
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
} catch(e) {
}
}
if (type === 'audio') {
let tags = {
title: output_json['title'],
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
}
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
}
const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length);
const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null;
if (options.cropFileSettings) {
await cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
}
// registers file in DB
const file_obj = await db_api.registerFileDB2(full_file_path, type, options.user, category, null, options.cropFileSettings);
// TODO: remove the following line
if (file_name) file_names.push(file_name);
file_objs.push(file_obj);
}
let is_playlist = file_names.length > 1;
if (options.merged_string !== null && options.merged_string !== undefined) {
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
let diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
download['complete'] = true;
download['fileNames'] = is_playlist ? file_names : [full_file_path]
updateDownloads();
let container = null;
if (file_objs.length > 1) {
// create playlist
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, options.user);
} else if (file_objs.length === 1) {
container = file_objs[0];
} else {
logger.error('Downloaded file failed to result in metadata object.');
}
resolve({
file_uids: file_objs.map(file_obj => file_obj.uid),
container: container
});
}
});
});
}
// helper functions
async function getVideoInfoByURL(url, args = [], download = null) {
return new Promise(resolve => {
// remove bad args
const new_args = [...args];
const archiveArgIndex = new_args.indexOf('--download-archive');
if (archiveArgIndex !== -1) {
new_args.splice(archiveArgIndex, 2);
}
// actually get info
youtubedl.getInfo(url, new_args, (err, output) => {
if (output) {
resolve(output);
} else {
logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`);
if (err.stderr) {
logger.error(`${err.stderr}`)
}
if (download) {
download['error'] = `Failed pre-check for video info: ${err}`;
updateDownloads();
}
resolve(null);
}
});
});
}

@ -287,4 +287,39 @@ describe('Multi User', async function() {
// });
// });
describe('Downloader', function() {
const url = '';
const options = {
ui_uid: uuid(),
user: 'admin'
}
const download = {
url: url,
options: options,
type: 'video'
}
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('download_queue');
await db_api.insertRecordIntoTable('download_queue', download)
});
it('Get file info', async function() {
});
it('Download file', async function() {
});
it('Queue file', async function() {
});
it('Pause file', async function() {
});
});
});
Loading…
Cancel
Save