@ -1,58 +1,114 @@
const fs = require ( 'fs-extra' ) ;
const { uuid } = require ( 'uuidv4' ) ;
const path = require ( 'path' ) ;
const queue = require ( 'queue' ) ;
const mergeFiles = require ( 'merge-files' ) ;
const NodeID3 = require ( 'node-id3' )
const glob = require ( "glob" )
const youtubedl = require ( 'youtube-dl' ) ;
const logger = require ( './logger' ) ;
const config _api = require ( './config' ) ;
const twitch _api = require ( './twitch' ) ;
const categories _api = require ( './categories' ) ;
const utils = require ( './utils' ) ;
let db _api = null ;
let logger = null ;
const STEP _INDEX _TO _LABEL = {
0 : 'Creating download' ,
1 : 'Getting info' ,
2 : 'Downloading file'
}
const archivePath = path . join ( _ _dirname , 'appdata' , 'archives' ) ;
function setDB ( input _db _api ) { db _api = input _db _api }
function setLogger ( input _logger ) { logger = input _logger ; }
exports . initialize = ( input _db _api , input _logger ) => {
exports . initialize = ( input _db _api ) => {
setDB ( input _db _api ) ;
setLogger ( input _logger ) ;
setInterval ( checkDownloads , 10000 ) ;
categories _api . initialize ( db _api ) ;
// temporary
db _api . removeAllRecords ( 'download_queue' ) ;
}
exports . createDownload = async ( url , type , options ) => {
const download = {
url : url ,
type : type ,
options : options ,
uid : uuid ( ) ,
step _index : 0 ,
paused : false ,
finished _step : true ,
error : null ,
percent _complete : null ,
finished : false ,
timestamp _start : Date . now ( )
} ;
await db _api . insertRecordIntoTable ( 'download_queue' , download ) ;
return download ;
}
exports . pauseDownload = ( ) => {
}
// questions
// how do we want to manage queued downloads that errored in any step? do we set the index back and finished_step to true or let the manager do it?
async function checkDownloads ( ) {
logger . verbose ( 'Checking downloads' ) ;
const downloads = await db _api . getRecords ( 'download_queue' ) ;
downloads . sort ( ( download1 , download2 ) => download1 . timestamp _start - download2 . timestamp _start ) ;
downloads = downloads . filter ( download => ! download . paused ) ;
for ( let i = 0 ; i < downloads . length ; i ++ ) {
if ( i === config _api . getConfigItem ( 'ytdl_' ) )
const running _downloads = downloads . filter ( download => ! download . paused ) ;
for ( let i = 0 ; i < running _downloads . length ; i ++ ) {
const running _download = running _downloads [ i ] ;
if ( i === 5 /*config_api.getConfigItem('ytdl_max_concurrent_downloads')*/ ) break ;
if ( running _download [ 'finished_step' ] && ! running _download [ 'finished' ] ) {
// move to next step
if ( running _download [ 'step_index' ] === 0 ) {
collectInfo ( running _download [ 'uid' ] ) ;
} else if ( running _download [ 'step_index' ] === 1 ) {
downloadQueuedFile ( running _download [ 'uid' ] ) ;
}
}
}
}
async function createDownload ( url , type , options ) {
const download = { url : url , type : type , options : options , uid : uuid ( ) } ;
await db _api . insertRecord ( download ) ;
return download ;
}
async function collectInfo ( download _uid ) {
const download = db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
logger . verbose ( ` Collecting info for download ${ download _uid } ` ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { step _index : 1 , finished _step : false } ) ;
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
const url = download [ 'url' ] ;
const type = download [ 'type' ] ;
const options = download [ 'options' ] ;
const args = download [ 'args' ] ;
if ( options . user && ! options . customFileFolderPath ) {
let usersFileFolder = config _api . getConfigItem ( 'ytdl_users_base_path' ) ;
const user _path = path . join ( usersFileFolder , options . user , type ) ;
options . customFileFolderPath = user _path + path . sep ;
}
let args = await generateArgs ( url , type , options ) ;
// get video info prior to download
const info = await getVideoInfoByURL ( url , args ) ;
le t info = await getVideoInfoByURL ( url , args , download _uid ) ;
if ( ! info ) {
// info failed, record error and pause download
const error = 'Failed to get info, see server logs for specific error.' ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { error : error , paused : true } ) ;
return ;
}
let category = null ;
// check if it fits into a category. If so, then get info again using new args
if ( ! Array . isArray ( info ) || config _api . getConfigItem ( 'ytdl_allow_playlist_categorization' ) ) category = await categories _api . categorize ( info ) ;
@ -61,22 +117,37 @@ async function collectInfo(download_uid) {
options . customOutput = category [ 'custom_output' ] ;
options . noRelativePath = true ;
args = await generateArgs ( url , type , options ) ;
info = await getVideoInfoByURL ( url , args ) ;
// must update args
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { args : args } ) ;
info = await getVideoInfoByURL ( url , args , download _uid ) ;
}
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { remote _metadata : info } ) ;
}
// setup info required to calculate download progress
async function downloadQueuedFile ( url , type , options ) {
const expected _file _size = utils . getExpectedFileSize ( info ) ;
const files _to _check _for _progress = [ ] ;
// store info in download for future use
if ( Array . isArray ( info ) ) {
for ( let info _obj of info ) files _to _check _for _progress . push ( utils . removeFileExtension ( info _obj [ '_filename' ] ) ) ;
} else {
files _to _check _for _progress . push ( utils . removeFileExtension ( info [ '_filename' ] ) ) ;
}
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { args : args ,
finished _step : true ,
options : options ,
files _to _check _for _progress : files _to _check _for _progress ,
expected _file _size : expected _file _size
} ) ;
}
async function downloadFileByURL _exec ( url , type , options ) {
return new Promise ( resolve => {
const download = db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
async function downloadQueuedFile ( download _uid ) {
logger . verbose ( ` Downloading ${ download _uid } ` ) ;
return new Promise ( async resolve => {
const audioFolderPath = config _api . getConfigItem ( 'ytdl_audio_folder_path' ) ;
const videoFolderPath = config _api . getConfigItem ( 'ytdl_video_folder_path' ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { step _index : 2 , finished _step : false } ) ;
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
const url = download [ 'url' ] ;
const type = download [ 'type' ] ;
@ -84,20 +155,22 @@ async function downloadFileByURL_exec(url, type, options) {
const args = download [ 'args' ] ;
const category = download [ 'category' ] ;
let fileFolderPath = type === 'audio' ? audioFolderPath : videoFolderPath ; // TODO: fix
if ( options . user ) {
let usersFileFolder = config _api . getConfigItem ( 'ytdl_users_base_path' ) ;
const user _path = path . join ( usersFileFolder , options . user , type ) ;
fs . ensureDirSync ( user _path ) ;
fileFolderPath = user _path + path . sep ;
options . customFileFolderPath = fileFolderPath ;
if ( options . customFileFolderPath ) {
fileFolderPath = options . customFileFolderPath ;
}
fs . ensureDirSync ( fileFolderPath ) ;
const start _time = Date . now ( ) ;
const download _checker = setInterval ( ( ) => checkDownloadPercent ( download [ 'uid' ] ) , 1000 ) ;
// download file
youtubedl . exec ( url , args , { maxBuffer : Infinity } , async function ( err , output ) {
const file _objs = [ ] ;
let new _date = Date . now ( ) ;
let difference = ( new _date - date ) / 1000 ;
logger . debug ( ` ${ is _audio ? 'Audio' : 'Video' } download delay: ${ difference } seconds. ` ) ;
let end _time = Date . now ( ) ;
let difference = ( end _time - start _time ) / 1000 ;
logger . debug ( ` ${ type === 'audio' ? 'Audio' : 'Video' } download delay: ${ difference } seconds. ` ) ;
clearInterval ( download _checker ) ;
if ( err ) {
logger . error ( err . stderr ) ;
@ -107,7 +180,6 @@ async function downloadFileByURL_exec(url, type, options) {
if ( output . length === 0 || output [ 0 ] . length === 0 ) {
// ERROR!
logger . warn ( ` No output received for video download, check if it exists in your archive. ` )
resolve ( false ) ;
return ;
}
@ -127,11 +199,12 @@ async function downloadFileByURL_exec(url, type, options) {
// get filepath with no extension
const filepath _no _extension = utils . removeFileExtension ( output _json [ '_filename' ] ) ;
const ext = type === 'audio' ? '.mp3' : '.mp4' ;
var full _file _path = filepath _no _extension + ext ;
var file _name = filepath _no _extension . substring ( fileFolderPath . length , filepath _no _extension . length ) ;
if ( type === 'video' && url . includes ( 'twitch.tv/videos/' ) && url . split ( 'twitch.tv/videos/' ) . length > 1
&& config . getConfigItem ( 'ytdl_use_twitch_api' ) && config . getConfigItem ( 'ytdl_twitch_auto_download_chat' ) ) {
&& config _api . getConfigItem ( 'ytdl_use_twitch_api' ) && config _api . getConfigItem ( 'ytdl_twitch_auto_download_chat' ) ) {
let vodId = url . split ( 'twitch.tv/videos/' ) [ 1 ] ;
vodId = vodId . split ( '?' ) [ 0 ] ;
twitch _api . downloadTwitchChatByVODID ( vodId , file _name , type , options . user ) ;
@ -143,6 +216,7 @@ async function downloadFileByURL_exec(url, type, options) {
fs . renameSync ( output _json [ '_filename' ] + '.webm' , output _json [ '_filename' ] ) ;
logger . info ( 'Renamed ' + file _name + '.webm to ' + file _name ) ;
} catch ( e ) {
}
}
@ -156,7 +230,7 @@ async function downloadFileByURL_exec(url, type, options) {
}
if ( options . cropFileSettings ) {
await cropFile( full _file _path , options . cropFileSettings . cropFileStart , options . cropFileSettings . cropFileEnd , ext ) ;
await utils. cropFile( full _file _path , options . cropFileSettings . cropFileStart , options . cropFileSettings . cropFileEnd , ext ) ;
}
// registers file in DB
@ -184,10 +258,9 @@ async function downloadFileByURL_exec(url, type, options) {
logger . error ( 'Downloaded file failed to result in metadata object.' ) ;
}
resolve ( {
file _uids : file _objs . map ( file _obj => file _obj . uid ) ,
container : container
} ) ;
const file _uids = file _objs . map ( file _obj => file _obj . uid ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { finished _step : true , finished : true , percent _complete : 100 , file _uids : file _uids , container : container } ) ;
resolve ( ) ;
}
} ) ;
} ) ;
@ -196,17 +269,20 @@ async function downloadFileByURL_exec(url, type, options) {
// helper functions
async function generateArgs ( url , type , options ) {
const audioFolderPath = config _api . getConfigItem ( 'ytdl_audio_folder_path' ) ;
const videoFolderPath = config _api . getConfigItem ( 'ytdl_video_folder_path' ) ;
const videopath = config _api . getConfigItem ( 'ytdl_default_file_output' ) ? config _api . getConfigItem ( 'ytdl_default_file_output' ) : '%(title)s' ;
const globalArgs = config _api . getConfigItem ( 'ytdl_custom_args' ) ;
const useCookies = config _api . getConfigItem ( 'ytdl_use_cookies' ) ;
const is _audio = type === 'audio' ;
cons t fileFolderPath = is _audio ? audioFolderPath : videoFolderPath ;
le t fileFolderPath = is _audio ? audioFolderPath : videoFolderPath ;
if ( options . customFileFolderPath ) fileFolderPath = options . customFileFolderPath ;
const customArgs = options . customArgs ;
cons t customOutput = options . customOutput ;
le t customOutput = options . customOutput ;
const customQualityConfiguration = options . customQualityConfiguration ;
// video-specific args
@ -246,7 +322,7 @@ async function generateArgs(url, type, options) {
downloadConfig = [ '-o' , path . join ( fileFolderPath , videopath + ( is _audio ? '.%(ext)s' : '.mp4' ) ) , '--write-info-json' , '--print-json' ] ;
}
if ( qualityPath && options . downloading _method === 'exec' ) downloadConfig . push ( ... qualityPath ) ;
if ( qualityPath ) downloadConfig . push ( ... qualityPath ) ;
if ( is _audio && ! options . skip _audio _args ) {
downloadConfig . push ( '-x' ) ;
@ -265,6 +341,8 @@ async function generateArgs(url, type, options) {
}
}
const useDefaultDownloadingAgent = config _api . getConfigItem ( 'ytdl_use_default_downloading_agent' ) ;
const customDownloadingAgent = config _api . getConfigItem ( 'ytdl_custom_downloading_agent' ) ;
if ( ! useDefaultDownloadingAgent && customDownloadingAgent ) {
downloadConfig . splice ( 0 , 0 , '--external-downloader' , customDownloadingAgent ) ;
}
@ -284,7 +362,7 @@ async function generateArgs(url, type, options) {
await fs . ensureFile ( merged _path ) ;
// merges blacklist and regular archive
let inputPathList = [ archive _path , blacklist _path ] ;
let status = await mergeFiles ( inputPathList , merged _path ) ;
await mergeFiles ( inputPathList , merged _path ) ;
options . merged _string = await fs . readFile ( merged _path , "utf8" ) ;
@ -324,7 +402,7 @@ async function generateArgs(url, type, options) {
return downloadConfig ;
}
async function getVideoInfoByURL ( url , args = [ ] , download = null ) {
async function getVideoInfoByURL ( url , args = [ ] , download _uid = null ) {
return new Promise ( resolve => {
// remove bad args
const new _args = [ ... args ] ;
@ -334,21 +412,85 @@ async function getVideoInfoByURL(url, args = [], download = null) {
new _args . splice ( archiveArgIndex , 2 ) ;
}
// actually get info
youtubedl . getInfo ( url , new _args , ( err , output ) => {
new _args . push ( '--dump-json' ) ;
youtubedl . exec ( url , new _args , { maxBuffer : Infinity } , async ( err , output ) => {
if ( output ) {
resolve ( output ) ;
let outputs = [ ] ;
try {
for ( let i = 0 ; i < output . length ; i ++ ) {
let output _json = null ;
try {
output _json = JSON . parse ( output [ i ] ) ;
} catch ( e ) {
output _json = null ;
}
if ( ! output _json ) {
continue ;
}
outputs . push ( output _json ) ;
}
resolve ( outputs . length === 1 ? outputs [ 0 ] : outputs ) ;
} catch ( e ) {
logger . error ( ` Error while retrieving info on video with URL ${ url } with the following message: output JSON could not be parsed. Output JSON: ${ output } ` ) ;
if ( download _uid ) {
const error = 'Failed to get info, see server logs for specific error.' ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { error : error , paused : true } ) ;
}
resolve ( null ) ;
}
} else {
logger . error ( ` Error while retrieving info on video with URL ${ url } with the following message: ${ err } ` ) ;
if ( err . stderr ) {
logger . error ( ` ${ err . stderr } ` )
}
if ( download ) {
download [ 'error' ] = ` Failed pre-check for video info: ${ err } ` ;
updateDownloads ( ) ;
if ( download _uid ) {
const error = 'Failed to get info, see server logs for specific error.' ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { error : error , paused : true } ) ;
}
resolve ( null ) ;
}
} ) ;
} ) ;
}
function filterArgs ( args , isAudio ) {
const video _only _args = [ '--add-metadata' , '--embed-subs' , '--xattrs' ] ;
const audio _only _args = [ '-x' , '--extract-audio' , '--embed-thumbnail' ] ;
const args _to _remove = isAudio ? video _only _args : audio _only _args ;
return args . filter ( x => ! args _to _remove . includes ( x ) ) ;
}
async function checkDownloadPercent ( download _uid ) {
/ *
This is more of an art than a science , we ' re just selecting files that start with the file name ,
thus capturing the parts being downloaded in files named like so : '<video title>.<format>.<ext>.part' .
Any file that starts with < video title > will be counted as part of the "bytes downloaded" , which will
be divided by the "total expected bytes."
* /
const download = await db _api . getRecord ( 'download_queue' , { uid : download _uid } ) ;
const files _to _check _for _progress = download [ 'files_to_check_for_progress' ] ;
const resulting _file _size = download [ 'expected_file_size' ] ;
if ( ! resulting _file _size ) return ;
let sum _size = 0 ;
glob ( ` { ${ files _to _check _for _progress . join ( ',' ) } , }* ` , async ( err , files ) => {
files . forEach ( async file => {
try {
const file _stats = fs . statSync ( file ) ;
if ( file _stats && file _stats . size ) {
sum _size += file _stats . size ;
}
} catch ( e ) {
}
} ) ;
const percent _complete = ( sum _size / resulting _file _size * 100 ) . toFixed ( 2 ) ;
await db _api . updateRecord ( 'download_queue' , { uid : download _uid } , { percent _complete : percent _complete } ) ;
} ) ;
}