const fs = require ( 'fs-extra' ) ;
const { uuid } = require ( 'uuidv4' ) ;
const path = require ( 'path' ) ;
const NodeID3 = require ( 'node-id3' )
const Mutex = require ( 'async-mutex' ) . Mutex ;
const youtubedl = require ( 'youtube-dl' ) ;
const logger = require ( './logger' ) ;
const config _api = require ( './config' ) ;
const twitch _api = require ( './twitch' ) ;
const { create } = require ( 'xmlbuilder2' ) ;
const categories _api = require ( './categories' ) ;
const utils = require ( './utils' ) ;
const db _api = require ( './db' ) ;
const files _api = require ( './files' ) ;
const notifications _api = require ( './notifications' ) ;
const archive _api = require ( './archive' ) ;
const mutex = new Mutex ( ) ;
let should _check _downloads = true ;
if ( db _api . database _initialized ) {
setupDownloads ( ) ;
} else {
db _api . database _initialized _bs . subscribe ( init => {
if ( init ) setupDownloads ( ) ;
} ) ;
}
/ *
This file handles all the downloading functionality .
To download a file , we go through 4 steps . Here they are with their respective index & function :
0 : Create the download
- createDownload ( )
1 : Get info for the download ( we need this step for categories and archive functionality )
- collectInfo ( )
2 : Download the file
- downloadQueuedFile ( )
3 : Complete
- N / A
We use checkDownloads ( ) to move downloads through the steps and call their respective functions .
* /
exports . createDownload = async ( url , type , options , user _uid = null , sub _id = null , sub _name = null , prefetched _info = null ) => {
return await mutex . runExclusive ( async ( ) => {
const download = {
url : url ,
type : type ,
title : '' ,
user _uid : user _uid ,
sub _id : sub _id ,
sub _name : sub _name ,
prefetched _info : prefetched _info ,
options : options ,
uid : uuid ( ) ,
step _index : 0 ,
paused : false ,
running : false ,
finished _step : true ,
error : null ,
percent _complete : null ,
finished : false ,
timestamp _start : Date . now ( )
} ;
await db _api . insertRecordIntoTable ( 'download_queue' , download ) ;
should _check _downloads = true ;
return download ;
} ) ;
}
exports . pauseDownload = async ( download _uid ) => {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
if ( download [ 'paused' ] ) {
logger . warn ( ` Download ${ download _uid } is already paused! ` ) ;
return false ;
} else if ( download [ 'finished' ] ) {
logger . info ( ` Download ${ download _uid } could not be paused before completing. ` ) ;
return false ;
}
return await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { paused : true , running : false } ) ;
}
exports . resumeDownload = async ( download _uid ) => {
return await mutex . runExclusive ( async ( ) => {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
if ( ! download [ 'paused' ] ) {
logger . warn ( ` Download ${ download _uid } is not paused! ` ) ;
return false ;
}
const success = db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { paused : false } ) ;
should _check _downloads = true ;
return success ;
} )
}
exports . restartDownload = async ( download _uid ) => {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
await exports . clearDownload ( download _uid ) ;
const new _download = await exports . createDownload ( download [ 'url' ] , download [ 'type' ] , download [ 'options' ] , download [ 'user_uid' ] ) ;
should _check _downloads = true ;
return new _download ;
}
exports . cancelDownload = async ( download _uid ) => {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
if ( download [ 'cancelled' ] ) {
logger . warn ( ` Download ${ download _uid } is already cancelled! ` ) ;
return false ;
} else if ( download [ 'finished' ] ) {
logger . info ( ` Download ${ download _uid } could not be cancelled before completing. ` ) ;
return false ;
}
return await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { cancelled : true , running : false } ) ;
}
exports . clearDownload = async ( download _uid ) => {
return await db _api . removeRecord ( 'download_queue' , { uid : download _uid } ) ;
}
async function handleDownloadError ( download , error _message , error _type = null ) {
if ( ! download || ! download [ 'uid' ] ) return ;
notifications _api . sendDownloadErrorNotification ( download , download [ 'user_uid' ] , error _message , error _type ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download [ 'uid' ] } , { error : error _message , finished : true , running : false , error _type : error _type } ) ;
}
async function setupDownloads ( ) {
await fixDownloadState ( ) ;
setInterval ( checkDownloads , 1000 ) ;
}
async function fixDownloadState ( ) {
const downloads = await db _api . getRecords ( 'download_queue' ) ;
downloads . sort ( ( download1 , download2 ) => download1 . timestamp _start - download2 . timestamp _start ) ;
const running _downloads = downloads . filter ( download => ! download [ 'finished' ] && ! download [ 'error' ] ) ;
for ( let i = 0 ; i < running _downloads . length ; i ++ ) {
const running _download = running _downloads [ i ] ;
const update _obj = { finished _step : true , paused : true , running : false } ;
if ( running _download [ 'step_index' ] > 0 ) {
update _obj [ 'step_index' ] = running _download [ 'step_index' ] - 1 ;
}
await db _api . updateRecord ( 'download_queue' , { uid : running _download [ 'uid' ] } , update _obj ) ;
}
}
async function checkDownloads ( ) {
if ( ! should _check _downloads ) return ;
const downloads = await db _api . getRecords ( 'download_queue' ) ;
downloads . sort ( ( download1 , download2 ) => download1 . timestamp _start - download2 . timestamp _start ) ;
await mutex . runExclusive ( async ( ) => {
// avoid checking downloads unnecessarily, but double check that should_check_downloads is still true
const running _downloads = downloads . filter ( download => ! download [ 'paused' ] && ! download [ 'finished' ] ) ;
if ( running _downloads . length === 0 ) {
should _check _downloads = false ;
logger . verbose ( 'Disabling checking downloads as none are available.' ) ;
}
return ;
} ) ;
let running _downloads _count = downloads . filter ( download => download [ 'running' ] ) . length ;
const waiting _downloads = downloads . filter ( download => ! download [ 'paused' ] && download [ 'finished_step' ] && ! download [ 'finished' ] ) ;
for ( let i = 0 ; i < waiting _downloads . length ; i ++ ) {
const waiting _download = waiting _downloads [ i ] ;
const max _concurrent _downloads = config _api . getConfigItem ( 'ytdl_max_concurrent_downloads' ) ;
if ( max _concurrent _downloads < 0 || running _downloads _count >= max _concurrent _downloads ) break ;
if ( waiting _download [ 'finished_step' ] && ! waiting _download [ 'finished' ] ) {
if ( waiting _download [ 'sub_id' ] ) {
const sub _missing = ! ( await db _api . getRecord ( 'subscriptions' , { id : waiting _download [ 'sub_id' ] } ) ) ;
if ( sub _missing ) {
handleDownloadError ( waiting _download , ` Download failed as subscription with id ' ${ waiting _download [ 'sub_id' ] } ' is missing! ` , 'sub_id_missing' ) ;
continue ;
}
}
// move to next step
running _downloads _count ++ ;
if ( waiting _download [ 'step_index' ] === 0 ) {
collectInfo ( waiting _download [ 'uid' ] ) ;
} else if ( waiting _download [ 'step_index' ] === 1 ) {
downloadQueuedFile ( waiting _download [ 'uid' ] ) ;
}
}
}
}
async function collectInfo ( download _uid ) {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
if ( download [ 'paused' ] ) {
return ;
}
logger . verbose ( ` Collecting info for download ${ download _uid } ` ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { step _index : 1 , finished _step : false , running : true } ) ;
const url = download [ 'url' ] ;
const type = download [ 'type' ] ;
const options = download [ 'options' ] ;
if ( download [ 'user_uid' ] && ! options . customFileFolderPath ) {
let usersFileFolder = config _api . getConfigItem ( 'ytdl_users_base_path' ) ;
const user _path = path . join ( usersFileFolder , download [ 'user_uid' ] , type ) ;
options . customFileFolderPath = user _path + path . sep ;
}
let args = await exports . generateArgs ( url , type , options , download [ 'user_uid' ] ) ;
// get video info prior to download
let info = download [ 'prefetched_info' ] ? download [ 'prefetched_info' ] : await exports . getVideoInfoByURL ( url , args , download _uid ) ;
if ( ! info ) {
// info failed, error presumably already recorded
return ;
}
// in subscriptions we don't care if archive mode is enabled, but we already removed archived videos from subs by this point
const useYoutubeDLArchive = config _api . getConfigItem ( 'ytdl_use_youtubedl_archive' ) ;
if ( useYoutubeDLArchive && ! options . ignoreArchive ) {
const exists _in _archive = await archive _api . existsInArchive ( info [ 'extractor' ] , info [ 'id' ] , type , download [ 'user_uid' ] , download [ 'sub_id' ] ) ;
if ( exists _in _archive ) {
const error = ` File ' ${ info [ 'title' ] } ' already exists in archive! Disable the archive or override to continue downloading. ` ;
logger . warn ( error ) ;
if ( download _uid ) {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
await handleDownloadError ( download , error , 'exists_in_archive' ) ;
return ;
}
}
}
let category = null ;
// check if it fits into a category. If so, then get info again using new args
if ( ! Array . isArray ( info ) || config _api . getConfigItem ( 'ytdl_allow_playlist_categorization' ) ) category = await categories _api . categorize ( info ) ;
// set custom output if the category has one and re-retrieve info so the download manager has the right file name
if ( category && category [ 'custom_output' ] ) {
options . customOutput = category [ 'custom_output' ] ;
options . noRelativePath = true ;
args = await exports . generateArgs ( url , type , options , download [ 'user_uid' ] ) ;
info = await exports . getVideoInfoByURL ( url , args , download _uid ) ;
}
const stripped _category = category ? { name : category [ 'name' ] , uid : category [ 'uid' ] } : null ;
// setup info required to calculate download progress
const expected _file _size = utils . getExpectedFileSize ( info ) ;
const files _to _check _for _progress = [ ] ;
// store info in download for future use
if ( Array . isArray ( info ) ) {
for ( let info _obj of info ) files _to _check _for _progress . push ( utils . removeFileExtension ( info _obj [ '_filename' ] ) ) ;
} else {
files _to _check _for _progress . push ( utils . removeFileExtension ( info [ '_filename' ] ) ) ;
}
const playlist _title = Array . isArray ( info ) ? info [ 0 ] [ 'playlist_title' ] || info [ 0 ] [ 'playlist' ] : null ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { args : args ,
finished _step : true ,
running : false ,
options : options ,
files _to _check _for _progress : files _to _check _for _progress ,
expected _file _size : expected _file _size ,
title : playlist _title ? playlist _title : info [ 'title' ] ,
category : stripped _category ,
prefetched _info : null
} ) ;
}
async function downloadQueuedFile ( download _uid ) {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
if ( download [ 'paused' ] ) {
return ;
}
logger . verbose ( ` Downloading ${ download _uid } ` ) ;
return new Promise ( async resolve => {
const audioFolderPath = config _api . getConfigItem ( 'ytdl_audio_folder_path' ) ;
const videoFolderPath = config _api . getConfigItem ( 'ytdl_video_folder_path' ) ;
const usersFolderPath = config _api . getConfigItem ( 'ytdl_users_base_path' ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { step _index : 2 , finished _step : false , running : true } ) ;
const url = download [ 'url' ] ;
const type = download [ 'type' ] ;
const options = download [ 'options' ] ;
const args = download [ 'args' ] ;
const category = download [ 'category' ] ;
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath ;
if ( options . customFileFolderPath ) {
fileFolderPath = options . customFileFolderPath ;
} else if ( download [ 'user_uid' ] ) {
fileFolderPath = path . join ( usersFolderPath , download [ 'user_uid' ] , type ) ;
}
fs . ensureDirSync ( fileFolderPath ) ;
const start _time = Date . now ( ) ;
const download _checker = setInterval ( ( ) => checkDownloadPercent ( download [ 'uid' ] ) , 1000 ) ;
// download file
youtubedl . exec ( url , args , { maxBuffer : Infinity } , async function ( err , output ) {
const file _objs = [ ] ;
let end _time = Date . now ( ) ;
let difference = ( end _time - start _time ) / 1000 ;
logger . debug ( ` ${ type === 'audio' ? 'Audio' : 'Video' } download delay: ${ difference } seconds. ` ) ;
clearInterval ( download _checker ) ;
if ( err ) {
logger . error ( err . stderr ) ;
await handleDownloadError ( download , err . stderr , 'unknown_error' ) ;
resolve ( false ) ;
return ;
} else if ( output ) {
if ( output . length === 0 || output [ 0 ] . length === 0 ) {
// ERROR!
const error _message = ` No output received for video download, check if it exists in your archive. ` ;
await handleDownloadError ( download , error _message , 'no_output' ) ;
logger . warn ( error _message ) ;
resolve ( false ) ;
return ;
}
for ( let i = 0 ; i < output . length ; i ++ ) {
let output _json = null ;
try {
// we have to do this because sometimes there will be leading characters before the actual json
const start _idx = output [ i ] . indexOf ( '{"' ) ;
const clean _output = output [ i ] . slice ( start _idx , output [ i ] . length ) ;
output _json = JSON . parse ( clean _output ) ;
} catch ( e ) {
output _json = null ;
}
if ( ! output _json ) {
continue ;
}
// get filepath with no extension
const filepath _no _extension = utils . removeFileExtension ( output _json [ '_filename' ] ) ;
const ext = type === 'audio' ? '.mp3' : '.mp4' ;
var full _file _path = filepath _no _extension + ext ;
var file _name = filepath _no _extension . substring ( fileFolderPath . length , filepath _no _extension . length ) ;
if ( type === 'video' && url . includes ( 'twitch.tv/videos/' ) && url . split ( 'twitch.tv/videos/' ) . length > 1
&& config _api . getConfigItem ( 'ytdl_twitch_auto_download_chat' ) ) {
let vodId = url . split ( 'twitch.tv/videos/' ) [ 1 ] ;
vodId = vodId . split ( '?' ) [ 0 ] ;
twitch _api . downloadTwitchChatByVODID ( vodId , file _name , type , download [ 'user_uid' ] ) ;
}
// renames file if necessary due to bug
if ( ! fs . existsSync ( output _json [ '_filename' ] ) && fs . existsSync ( output _json [ '_filename' ] + '.webm' ) ) {
try {
fs . renameSync ( output _json [ '_filename' ] + '.webm' , output _json [ '_filename' ] ) ;
logger . info ( 'Renamed ' + file _name + '.webm to ' + file _name ) ;
} catch ( e ) {
logger . error ( ` Failed to rename file ${ output _json [ '_filename' ] } to its appropriate extension. ` ) ;
}
}
if ( type === 'audio' ) {
let tags = {
title : output _json [ 'title' ] ,
artist : output _json [ 'artist' ] ? output _json [ 'artist' ] : output _json [ 'uploader' ]
}
let success = NodeID3 . write ( tags , utils . removeFileExtension ( output _json [ '_filename' ] ) + '.mp3' ) ;
if ( ! success ) logger . error ( 'Failed to apply ID3 tag to audio file ' + output _json [ '_filename' ] ) ;
}
if ( config _api . getConfigItem ( 'ytdl_generate_nfo_files' ) ) {
exports . generateNFOFile ( output _json , ` ${ filepath _no _extension } .nfo ` ) ;
}
if ( options . cropFileSettings ) {
await utils . cropFile ( full _file _path , options . cropFileSettings . cropFileStart , options . cropFileSettings . cropFileEnd , ext ) ;
}
// registers file in DB
const file _obj = await files _api . registerFileDB ( full _file _path , type , download [ 'user_uid' ] , category , download [ 'sub_id' ] ? download [ 'sub_id' ] : null , options . cropFileSettings ) ;
await archive _api . addToArchive ( output _json [ 'extractor' ] , output _json [ 'id' ] , type , output _json [ 'title' ] , download [ 'user_uid' ] , download [ 'sub_id' ] ) ;
notifications _api . sendDownloadNotification ( file _obj , download [ 'user_uid' ] ) ;
file _objs . push ( file _obj ) ;
}
let container = null ;
if ( file _objs . length > 1 ) {
// create playlist
const playlist _name = file _objs . map ( file _obj => file _obj . title ) . join ( ', ' ) ;
container = await files _api . createPlaylist ( playlist _name , file _objs . map ( file _obj => file _obj . uid ) , download [ 'user_uid' ] ) ;
} else if ( file _objs . length === 1 ) {
container = file _objs [ 0 ] ;
} else {
const error _message = 'Downloaded file failed to result in metadata object.' ;
logger . error ( error _message ) ;
await handleDownloadError ( download , error _message , 'no_metadata' ) ;
}
const file _uids = file _objs . map ( file _obj => file _obj . uid ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { finished _step : true , finished : true , running : false , step _index : 3 , percent _complete : 100 , file _uids : file _uids , container : container } ) ;
resolve ( ) ;
}
} ) ;
} ) ;
}
// helper functions
exports . generateArgs = async ( url , type , options , user _uid = null , simulated = false ) => {
const default _downloader = utils . getCurrentDownloader ( ) || config _api . getConfigItem ( 'ytdl_default_downloader' ) ;
if ( ! simulated && ( default _downloader === 'youtube-dl' || default _downloader === 'youtube-dlc' ) ) {
logger . warn ( 'It is recommended you use yt-dlp! To prevent failed downloads, change the downloader in your settings menu to yt-dlp and restart your instance.' )
}
const audioFolderPath = config _api . getConfigItem ( 'ytdl_audio_folder_path' ) ;
const videoFolderPath = config _api . getConfigItem ( 'ytdl_video_folder_path' ) ;
const usersFolderPath = config _api . getConfigItem ( 'ytdl_users_base_path' ) ;
const videopath = config _api . getConfigItem ( 'ytdl_default_file_output' ) ? config _api . getConfigItem ( 'ytdl_default_file_output' ) : '%(title)s' ;
const globalArgs = config _api . getConfigItem ( 'ytdl_custom_args' ) ;
const useCookies = config _api . getConfigItem ( 'ytdl_use_cookies' ) ;
const is _audio = type === 'audio' ;
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath ; // TODO: fix
if ( options . customFileFolderPath ) {
fileFolderPath = options . customFileFolderPath ;
} else if ( user _uid ) {
fileFolderPath = path . join ( usersFolderPath , user _uid , fileFolderPath ) ;
}
if ( options . customFileFolderPath ) fileFolderPath = options . customFileFolderPath ;
const customArgs = options . customArgs ;
let customOutput = options . customOutput ;
const customQualityConfiguration = options . customQualityConfiguration ;
// video-specific args
const selectedHeight = options . selectedHeight ;
const maxHeight = options . maxHeight ;
const heightParam = selectedHeight || maxHeight ;
// audio-specific args
const maxBitrate = options . maxBitrate ;
const youtubeUsername = options . youtubeUsername ;
const youtubePassword = options . youtubePassword ;
let downloadConfig = null ;
let qualityPath = ( is _audio && ! options . skip _audio _args ) ? [ '-f' , 'bestaudio' ] : [ '-f' , 'bestvideo+bestaudio' , '--merge-output-format' , 'mp4' ] ;
const is _youtube = url . includes ( 'youtu' ) ;
if ( ! is _audio && ! is _youtube ) {
// tiktok videos fail when using the default format
qualityPath = null ;
}
if ( customArgs ) {
downloadConfig = customArgs . split ( ',,' ) ;
} else {
if ( customQualityConfiguration ) {
qualityPath = [ '-f' , customQualityConfiguration , '--merge-output-format' , 'mp4' ] ;
} else if ( heightParam && heightParam !== '' && ! is _audio ) {
const heightFilter = ( maxHeight && default _downloader === 'yt-dlp' ) ? [ '-S' , ` res: ${ heightParam } ` ] : [ '-f' , ` best[height ${ maxHeight ? '<' : '' } = ${ heightParam } ]+bestaudio ` ]
qualityPath = [ ... heightFilter , '--merge-output-format' , 'mp4' ] ;
} else if ( is _audio ) {
qualityPath = [ '--audio-quality' , maxBitrate ? maxBitrate : '0' ]
}
if ( customOutput ) {
customOutput = options . noRelativePath ? customOutput : path . join ( fileFolderPath , customOutput ) ;
downloadConfig = [ '-o' , ` ${ customOutput } .%(ext)s ` , '--write-info-json' , '--print-json' ] ;
} else {
downloadConfig = [ '-o' , path . join ( fileFolderPath , videopath + ( is _audio ? '.%(ext)s' : '.mp4' ) ) , '--write-info-json' , '--print-json' ] ;
}
if ( qualityPath ) downloadConfig . push ( ... qualityPath ) ;
if ( is _audio && ! options . skip _audio _args ) {
downloadConfig . push ( '-x' ) ;
downloadConfig . push ( '--audio-format' , 'mp3' ) ;
}
if ( youtubeUsername && youtubePassword ) {
downloadConfig . push ( '--username' , youtubeUsername , '--password' , youtubePassword ) ;
}
if ( useCookies ) {
if ( await fs . pathExists ( path . join ( _ _dirname , 'appdata' , 'cookies.txt' ) ) ) {
downloadConfig . push ( '--cookies' , path . join ( 'appdata' , 'cookies.txt' ) ) ;
} else {
logger . warn ( 'Cookies file could not be found. You can either upload one, or disable \'use cookies\' in the Advanced tab in the settings.' ) ;
}
}
const useDefaultDownloadingAgent = config _api . getConfigItem ( 'ytdl_use_default_downloading_agent' ) ;
const customDownloadingAgent = config _api . getConfigItem ( 'ytdl_custom_downloading_agent' ) ;
if ( ! useDefaultDownloadingAgent && customDownloadingAgent ) {
downloadConfig . splice ( 0 , 0 , '--external-downloader' , customDownloadingAgent ) ;
}
if ( config _api . getConfigItem ( 'ytdl_include_thumbnail' ) ) {
downloadConfig . push ( '--write-thumbnail' ) ;
}
if ( globalArgs && globalArgs !== '' ) {
// adds global args
if ( downloadConfig . indexOf ( '-o' ) !== - 1 && globalArgs . split ( ',,' ) . indexOf ( '-o' ) !== - 1 ) {
// if global args has an output, replce the original output with that of global args
const original _output _index = downloadConfig . indexOf ( '-o' ) ;
downloadConfig . splice ( original _output _index , 2 ) ;
}
downloadConfig = downloadConfig . concat ( globalArgs . split ( ',,' ) ) ;
}
if ( options . additionalArgs && options . additionalArgs !== '' ) {
downloadConfig = utils . injectArgs ( downloadConfig , options . additionalArgs . split ( ',,' ) ) ;
}
const rate _limit = config _api . getConfigItem ( 'ytdl_download_rate_limit' ) ;
if ( rate _limit && downloadConfig . indexOf ( '-r' ) === - 1 && downloadConfig . indexOf ( '--limit-rate' ) === - 1 ) {
downloadConfig . push ( '-r' , rate _limit ) ;
}
if ( default _downloader === 'yt-dlp' ) {
downloadConfig = utils . filterArgs ( downloadConfig , [ '--print-json' ] ) ;
// in yt-dlp -j --no-simulate is preferable
downloadConfig . push ( '--no-clean-info-json' , '-j' , '--no-simulate' ) ;
}
}
// filter out incompatible args
downloadConfig = filterArgs ( downloadConfig , is _audio ) ;
if ( ! simulated ) logger . verbose ( ` ${ default _downloader } args being used: ${ downloadConfig . join ( ',' ) } ` ) ;
return downloadConfig ;
}
exports . getVideoInfoByURL = async ( url , args = [ ] , download _uid = null ) => {
return new Promise ( resolve => {
// remove bad args
const temp _args = utils . filterArgs ( args , [ '--no-simulate' ] ) ;
const new _args = [ ... temp _args ] ;
const archiveArgIndex = new _args . indexOf ( '--download-archive' ) ;
if ( archiveArgIndex !== - 1 ) {
new _args . splice ( archiveArgIndex , 2 ) ;
}
new _args . push ( '--dump-json' ) ;
youtubedl . exec ( url , new _args , { maxBuffer : Infinity } , async ( err , output ) => {
if ( output ) {
let outputs = [ ] ;
try {
for ( let i = 0 ; i < output . length ; i ++ ) {
let output _json = null ;
try {
output _json = JSON . parse ( output [ i ] ) ;
} catch ( e ) {
output _json = null ;
}
if ( ! output _json ) {
continue ;
}
outputs . push ( output _json ) ;
}
resolve ( outputs . length === 1 ? outputs [ 0 ] : outputs ) ;
} catch ( e ) {
const error = ` Error while retrieving info on video with URL ${ url } with the following message: output JSON could not be parsed. Output JSON: ${ output } ` ;
logger . error ( error ) ;
if ( download _uid ) {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
await handleDownloadError ( download , error , 'parse_failed' ) ;
}
resolve ( null ) ;
}
} else {
let error _message = ` Error while retrieving info on video with URL ${ url } with the following message: ${ err } ` ;
if ( err . stderr ) error _message += ` \n \n ${ err . stderr } ` ;
logger . error ( error _message ) ;
if ( download _uid ) {
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
await handleDownloadError ( download , error _message , 'info_retrieve_failed' ) ;
}
resolve ( null ) ;
}
} ) ;
} ) ;
}
function filterArgs ( args , isAudio ) {
const video _only _args = [ '--add-metadata' , '--embed-subs' , '--xattrs' ] ;
const audio _only _args = [ '-x' , '--extract-audio' , '--embed-thumbnail' ] ;
return utils . filterArgs ( args , isAudio ? video _only _args : audio _only _args ) ;
}
async function checkDownloadPercent ( download _uid ) {
/ *
This is more of an art than a science , we ' re just selecting files that start with the file name ,
thus capturing the parts being downloaded in files named like so : '<video title>.<format>.<ext>.part' .
Any file that starts with < video title > will be counted as part of the "bytes downloaded" , which will
be divided by the "total expected bytes."
* /
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
const files _to _check _for _progress = download [ 'files_to_check_for_progress' ] ;
const resulting _file _size = download [ 'expected_file_size' ] ;
if ( ! resulting _file _size ) return ;
let sum _size = 0 ;
for ( let i = 0 ; i < files _to _check _for _progress . length ; i ++ ) {
const file _to _check _for _progress = files _to _check _for _progress [ i ] ;
const dir = path . dirname ( file _to _check _for _progress ) ;
if ( ! fs . existsSync ( dir ) ) continue ;
fs . readdir ( dir , async ( err , files ) => {
for ( let j = 0 ; j < files . length ; j ++ ) {
const file = files [ j ] ;
if ( ! file . includes ( path . basename ( file _to _check _for _progress ) ) ) continue ;
try {
const file _stats = fs . statSync ( path . join ( dir , file ) ) ;
if ( file _stats && file _stats . size ) {
sum _size += file _stats . size ;
}
} catch ( e ) { }
}
const percent _complete = ( sum _size / resulting _file _size * 100 ) . toFixed ( 2 ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { percent _complete : percent _complete } ) ;
} ) ;
}
}
exports . generateNFOFile = ( info , output _path ) => {
const nfo _obj = {
episodedetails : {
title : info [ 'fulltitle' ] ,
episode : info [ 'playlist_index' ] ? info [ 'playlist_index' ] : undefined ,
premiered : utils . formatDateString ( info [ 'upload_date' ] ) ,
plot : ` ${ info [ 'uploader_url' ] } \n ${ info [ 'description' ] } \n ${ info [ 'playlist_title' ] ? info [ 'playlist_title' ] : '' } ` ,
director : info [ 'artist' ] ? info [ 'artist' ] : info [ 'uploader' ]
}
} ;
const doc = create ( nfo _obj ) ;
const xml = doc . end ( { prettyPrint : true } ) ;
fs . writeFileSync ( output _path , xml ) ;
}