Began scaffolding work for download manager
Isaac Abadi
4 years ago
parent
b1385f451b
commit
01b6e22f83
@ -0,0 +1,284 @@
|
||||
|
||||
|
||||
async function collectInfo(download_uid) {
|
||||
const download = db_api.getRecord('download_queue', {uid: download_uid});
|
||||
|
||||
const url = download['url'];
|
||||
const type = download['type'];
|
||||
const options = download['options'];
|
||||
const args = download['args'];
|
||||
|
||||
// get video info prior to download
|
||||
const info = await getVideoInfoByURL(url, args);
|
||||
|
||||
if (!info) {
|
||||
// info failed, record error and pause download
|
||||
}
|
||||
|
||||
// check if it fits into a category. If so, then get info again using new args
|
||||
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
|
||||
|
||||
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
|
||||
if (category && category['custom_output']) {
|
||||
options.customOutput = category['custom_output'];
|
||||
options.noRelativePath = true;
|
||||
args = await generateArgs(url, type, options);
|
||||
info = await getVideoInfoByURL(url, args);
|
||||
|
||||
// must update args
|
||||
await db_api.updateRecord('download_queue', {uid: download_uid}, {args: args});
|
||||
}
|
||||
|
||||
await db_api.updateRecord('download_queue', {uid: download_uid}, {remote_metadata: info});
|
||||
}
|
||||
|
||||
async function downloadFileByURL_exec(url, type, options) {
|
||||
const download = db_api.getRecord('download_queue', {uid: download_uid});
|
||||
|
||||
const url = download['url'];
|
||||
const type = download['type'];
|
||||
const options = download['options'];
|
||||
const args = download['args'];
|
||||
const category = download['category'];
|
||||
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath; // TODO: fix
|
||||
if (options.user) {
|
||||
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
|
||||
const user_path = path.join(usersFileFolder, options.user, type);
|
||||
fs.ensureDirSync(user_path);
|
||||
fileFolderPath = user_path + path.sep;
|
||||
multiUserMode = {
|
||||
user: options.user,
|
||||
file_path: fileFolderPath
|
||||
}
|
||||
options.customFileFolderPath = fileFolderPath;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
async function downloadFileByURL_exec_old(url, type, options, sessionID = null) {
|
||||
return new Promise(async resolve => {
|
||||
var date = Date.now();
|
||||
|
||||
// audio / video specific vars
|
||||
var is_audio = type === 'audio';
|
||||
var ext = is_audio ? '.mp3' : '.mp4';
|
||||
var fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath;
|
||||
let category = null;
|
||||
|
||||
// prepend with user if needed
|
||||
let multiUserMode = null;
|
||||
if (options.user) {
|
||||
let usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
|
||||
const user_path = path.join(usersFileFolder, options.user, type);
|
||||
fs.ensureDirSync(user_path);
|
||||
fileFolderPath = user_path + path.sep;
|
||||
options.customFileFolderPath = fileFolderPath;
|
||||
}
|
||||
|
||||
options.downloading_method = 'exec';
|
||||
let downloadConfig = await generateArgs(url, type, options);
|
||||
|
||||
// adds download to download helper
|
||||
const download_uid = uuid();
|
||||
const session = sessionID ? sessionID : 'undeclared';
|
||||
let session_downloads = downloads.find(potential_session_downloads => potential_session_downloads['session_id'] === session);
|
||||
if (!session_downloads) {
|
||||
session_downloads = {session_id: session};
|
||||
downloads.push(session_downloads);
|
||||
}
|
||||
session_downloads[download_uid] = {
|
||||
uid: download_uid,
|
||||
ui_uid: options.ui_uid,
|
||||
downloading: true,
|
||||
complete: false,
|
||||
url: url,
|
||||
type: type,
|
||||
percent_complete: 0,
|
||||
is_playlist: url.includes('playlist'),
|
||||
timestamp_start: Date.now(),
|
||||
filesize: null
|
||||
};
|
||||
const download = session_downloads[download_uid];
|
||||
updateDownloads();
|
||||
|
||||
let download_checker = null;
|
||||
|
||||
// get video info prior to download
|
||||
let info = await getVideoInfoByURL(url, downloadConfig, download);
|
||||
if (!info && url.includes('youtu')) {
|
||||
resolve(false);
|
||||
return;
|
||||
} else if (info) {
|
||||
// check if it fits into a category. If so, then get info again using new downloadConfig
|
||||
if (!Array.isArray(info) || config_api.getConfigItem('ytdl_allow_playlist_categorization')) category = await categories_api.categorize(info);
|
||||
|
||||
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
|
||||
if (category && category['custom_output']) {
|
||||
options.customOutput = category['custom_output'];
|
||||
options.noRelativePath = true;
|
||||
downloadConfig = await generateArgs(url, type, options);
|
||||
info = await getVideoInfoByURL(url, downloadConfig, download);
|
||||
}
|
||||
|
||||
// store info in download for future use
|
||||
if (Array.isArray(info)) {
|
||||
download['fileNames'] = [];
|
||||
for (let info_obj of info) download['fileNames'].push(info_obj['_filename']);
|
||||
} else {
|
||||
download['_filename'] = info['_filename'];
|
||||
}
|
||||
download['filesize'] = utils.getExpectedFileSize(info);
|
||||
download_checker = setInterval(() => checkDownloadPercent(download), 1000);
|
||||
}
|
||||
|
||||
// download file
|
||||
youtubedl.exec(url, downloadConfig, {maxBuffer: Infinity}, async function(err, output) {
|
||||
if (download_checker) clearInterval(download_checker); // stops the download checker from running as the download finished (or errored)
|
||||
|
||||
download['downloading'] = false;
|
||||
download['timestamp_end'] = Date.now();
|
||||
var file_objs = [];
|
||||
let new_date = Date.now();
|
||||
let difference = (new_date - date)/1000;
|
||||
logger.debug(`${is_audio ? 'Audio' : 'Video'} download delay: ${difference} seconds.`);
|
||||
if (err) {
|
||||
logger.error(err.stderr);
|
||||
|
||||
download['error'] = err.stderr;
|
||||
updateDownloads();
|
||||
resolve(false);
|
||||
return;
|
||||
} else if (output) {
|
||||
if (output.length === 0 || output[0].length === 0) {
|
||||
download['error'] = 'No output. Check if video already exists in your archive.';
|
||||
logger.warn(`No output received for video download, check if it exists in your archive.`)
|
||||
updateDownloads();
|
||||
|
||||
resolve(false);
|
||||
return;
|
||||
}
|
||||
var file_names = [];
|
||||
for (let i = 0; i < output.length; i++) {
|
||||
let output_json = null;
|
||||
try {
|
||||
output_json = JSON.parse(output[i]);
|
||||
} catch(e) {
|
||||
output_json = null;
|
||||
}
|
||||
|
||||
if (!output_json) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// get filepath with no extension
|
||||
const filepath_no_extension = utils.removeFileExtension(output_json['_filename']);
|
||||
|
||||
var full_file_path = filepath_no_extension + ext;
|
||||
var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length);
|
||||
|
||||
if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1
|
||||
&& config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) {
|
||||
let vodId = url.split('twitch.tv/videos/')[1];
|
||||
vodId = vodId.split('?')[0];
|
||||
twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, options.user);
|
||||
}
|
||||
|
||||
// renames file if necessary due to bug
|
||||
if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) {
|
||||
try {
|
||||
fs.renameSync(output_json['_filename'] + '.webm', output_json['_filename']);
|
||||
logger.info('Renamed ' + file_name + '.webm to ' + file_name);
|
||||
} catch(e) {
|
||||
}
|
||||
}
|
||||
|
||||
if (type === 'audio') {
|
||||
let tags = {
|
||||
title: output_json['title'],
|
||||
artist: output_json['artist'] ? output_json['artist'] : output_json['uploader']
|
||||
}
|
||||
let success = NodeID3.write(tags, utils.removeFileExtension(output_json['_filename']) + '.mp3');
|
||||
if (!success) logger.error('Failed to apply ID3 tag to audio file ' + output_json['_filename']);
|
||||
}
|
||||
|
||||
const file_path = options.noRelativePath ? path.basename(full_file_path) : full_file_path.substring(fileFolderPath.length, full_file_path.length);
|
||||
const customPath = options.noRelativePath ? path.dirname(full_file_path).split(path.sep).pop() : null;
|
||||
|
||||
if (options.cropFileSettings) {
|
||||
await cropFile(full_file_path, options.cropFileSettings.cropFileStart, options.cropFileSettings.cropFileEnd, ext);
|
||||
}
|
||||
|
||||
// registers file in DB
|
||||
const file_obj = await db_api.registerFileDB2(full_file_path, type, options.user, category, null, options.cropFileSettings);
|
||||
|
||||
// TODO: remove the following line
|
||||
if (file_name) file_names.push(file_name);
|
||||
|
||||
file_objs.push(file_obj);
|
||||
}
|
||||
|
||||
let is_playlist = file_names.length > 1;
|
||||
|
||||
if (options.merged_string !== null && options.merged_string !== undefined) {
|
||||
let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8');
|
||||
let diff = current_merged_archive.replace(options.merged_string, '');
|
||||
const archive_path = options.user ? path.join(fileFolderPath, 'archives', `archive_${type}.txt`) : path.join(archivePath, `archive_${type}.txt`);
|
||||
fs.appendFileSync(archive_path, diff);
|
||||
}
|
||||
|
||||
download['complete'] = true;
|
||||
download['fileNames'] = is_playlist ? file_names : [full_file_path]
|
||||
updateDownloads();
|
||||
|
||||
let container = null;
|
||||
|
||||
if (file_objs.length > 1) {
|
||||
// create playlist
|
||||
const playlist_name = file_objs.map(file_obj => file_obj.title).join(', ');
|
||||
const duration = file_objs.reduce((a, b) => a + utils.durationStringToNumber(b.duration), 0);
|
||||
container = await db_api.createPlaylist(playlist_name, file_objs.map(file_obj => file_obj.uid), type, options.user);
|
||||
} else if (file_objs.length === 1) {
|
||||
container = file_objs[0];
|
||||
} else {
|
||||
logger.error('Downloaded file failed to result in metadata object.');
|
||||
}
|
||||
|
||||
resolve({
|
||||
file_uids: file_objs.map(file_obj => file_obj.uid),
|
||||
container: container
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// helper functions
|
||||
|
||||
async function getVideoInfoByURL(url, args = [], download = null) {
|
||||
return new Promise(resolve => {
|
||||
// remove bad args
|
||||
const new_args = [...args];
|
||||
|
||||
const archiveArgIndex = new_args.indexOf('--download-archive');
|
||||
if (archiveArgIndex !== -1) {
|
||||
new_args.splice(archiveArgIndex, 2);
|
||||
}
|
||||
|
||||
// actually get info
|
||||
youtubedl.getInfo(url, new_args, (err, output) => {
|
||||
if (output) {
|
||||
resolve(output);
|
||||
} else {
|
||||
logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`);
|
||||
if (err.stderr) {
|
||||
logger.error(`${err.stderr}`)
|
||||
}
|
||||
if (download) {
|
||||
download['error'] = `Failed pre-check for video info: ${err}`;
|
||||
updateDownloads();
|
||||
}
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
Loading…
Reference in New Issue