@ -77,6 +77,7 @@ db.defaults(
video : [ ]
} ,
configWriteFlag : false ,
downloads : { } ,
subscriptions : [ ] ,
pin _md5 : '' ,
files _to _db _migration _complete : false
@ -101,6 +102,10 @@ var archivePath = path.join(__dirname, 'appdata', 'archives');
var options = null ; // encryption options
var url _domain = null ;
var updaterStatus = null ;
var last _downloads _check = null ;
var downloads _check _interval = 1000 ;
var timestamp _server _start = Date . now ( ) ;
if ( debugMode ) logger . info ( 'YTDL-Material in debug mode!' ) ;
@ -234,7 +239,6 @@ async function startServer() {
logger . info ( ` YoutubeDL-Material ${ CONSTS [ 'CURRENT_VERSION' ] } started on PORT ${ backendPort } ` ) ;
} ) ;
}
}
async function restartServer ( ) {
@ -547,6 +551,9 @@ async function loadConfig() {
// check migrations
await checkMigrations ( ) ;
// load in previous downloads
downloads = db . get ( 'downloads' ) . value ( ) ;
// start the server here
startServer ( ) ;
@ -999,6 +1006,7 @@ function registerFileDB(full_file_path, type) {
}
file _object [ 'uid' ] = uuid ( ) ;
file _object [ 'registered' ] = Date . now ( ) ;
path _object = path . parse ( file _object [ 'path' ] ) ;
file _object [ 'path' ] = path . format ( path _object ) ;
db . get ( ` files. ${ type } ` )
@ -1081,6 +1089,304 @@ function getVideoInfos(fileNames) {
return result ;
}
// downloads
async function downloadFileByURL _exec ( url , type , options , sessionID = null ) {
return new Promise ( async resolve => {
var date = Date . now ( ) ;
const downloadConfig = await generateArgs ( url , type , options ) ;
// adds download to download helper
const download _uid = uuid ( ) ;
const session = sessionID ? sessionID : 'undeclared' ;
if ( ! downloads [ session ] ) downloads [ session ] = { } ;
downloads [ session ] [ download _uid ] = {
uid : download _uid ,
downloading : true ,
complete : false ,
url : url ,
type : type ,
percent _complete : 0 ,
is _playlist : url . includes ( 'playlist' ) ,
timestamp _start : Date . now ( )
} ;
const download = downloads [ session ] [ download _uid ] ;
updateDownloads ( ) ;
await new Promise ( resolve => {
youtubedl . exec ( url , [ ... downloadConfig , '--dump-json' ] , { } , function ( err , output ) {
if ( output ) {
let json = JSON . parse ( output [ 0 ] ) ;
const output _no _ext = removeFileExtension ( json [ '_filename' ] ) ;
download [ 'expected_path' ] = output _no _ext + '.mp4' ;
download [ 'expected_json_path' ] = output _no _ext + '.info.json' ;
resolve ( true ) ;
} else if ( err ) {
logger . error ( err . stderr ) ;
} else {
logger . error ( ` Video info retrieval failed. Download progress will be unavailable for URL ${ url } ` ) ;
}
} ) ;
} ) ;
youtubedl . exec ( url , downloadConfig , { } , function ( err , output ) {
download [ 'downloading' ] = false ;
download [ 'timestamp_end' ] = Date . now ( ) ;
var file _uid = null ;
let new _date = Date . now ( ) ;
let difference = ( new _date - date ) / 1000 ;
logger . debug ( ` Video download delay: ${ difference } seconds. ` ) ;
if ( err ) {
logger . error ( err . stderr ) ;
download [ 'error' ] = err . stderr ;
updateDownloads ( ) ;
resolve ( false ) ;
throw err ;
} else if ( output ) {
if ( output . length === 0 || output [ 0 ] . length === 0 ) {
download [ 'error' ] = 'No output. Check if video already exists in your archive.' ;
updateDownloads ( ) ;
resolve ( false ) ;
return ;
}
var file _names = [ ] ;
for ( let i = 0 ; i < output . length ; i ++ ) {
let output _json = null ;
try {
output _json = JSON . parse ( output [ i ] ) ;
} catch ( e ) {
output _json = null ;
}
var modified _file _name = output _json ? output _json [ 'title' ] : null ;
if ( ! output _json ) {
continue ;
}
// get filepath with no extension
const filepath _no _extension = removeFileExtension ( output _json [ '_filename' ] ) ;
var full _file _path = filepath _no _extension + '.mp4' ;
var file _name = filepath _no _extension . substring ( audioFolderPath . length , filepath _no _extension . length ) ;
// renames file if necessary due to bug
if ( ! fs . existsSync ( output _json [ '_filename' ] && fs . existsSync ( output _json [ '_filename' ] + '.webm' ) ) ) {
try {
fs . renameSync ( output _json [ '_filename' ] + '.webm' , output _json [ '_filename' ] ) ;
logger . info ( 'Renamed ' + file _name + '.webm to ' + file _name ) ;
} catch ( e ) {
}
}
// registers file in DB
file _uid = registerFileDB ( full _file _path . substring ( videoFolderPath . length , full _file _path . length ) , 'video' ) ;
if ( file _name ) file _names . push ( file _name ) ;
}
let is _playlist = file _names . length > 1 ;
if ( options . merged _string !== null ) {
let current _merged _archive = fs . readFileSync ( videoFolderPath + 'merged.txt' , 'utf8' ) ;
let diff = current _merged _archive . replace ( options . merged _string , '' ) ;
const archive _path = path . join ( archivePath , 'archive_video.txt' ) ;
fs . appendFileSync ( archive _path , diff ) ;
}
download [ 'complete' ] = true ;
updateDownloads ( ) ;
var videopathEncoded = encodeURIComponent ( file _names [ 0 ] ) ;
resolve ( {
videopathEncoded : videopathEncoded ,
file _names : is _playlist ? file _names : null ,
uid : file _uid
} ) ;
}
} ) ;
} ) ;
}
async function downloadFileByURL _normal ( url , type , options , sessionID = null ) {
return new Promise ( async resolve => {
var date = Date . now ( ) ;
var file _uid = null ;
const downloadConfig = await generateArgs ( url , type , options ) ;
// adds download to download helper
const download _uid = uuid ( ) ;
const session = sessionID ? sessionID : 'undeclared' ;
if ( ! downloads [ session ] ) downloads [ session ] = { } ;
downloads [ session ] [ download _uid ] = {
uid : download _uid ,
downloading : true ,
complete : false ,
url : url ,
type : type ,
percent _complete : 0 ,
is _playlist : url . includes ( 'playlist' ) ,
timestamp _start : Date . now ( )
} ;
const download = downloads [ session ] [ download _uid ] ;
updateDownloads ( ) ;
const video = youtubedl ( url ,
// Optional arguments passed to youtube-dl.
downloadConfig ,
// Additional options can be given for calling `child_process.execFile()`.
{ cwd : _ _dirname } ) ;
let video _info = null ;
let file _size = 0 ;
// Will be called when the download starts.
video . on ( 'info' , function ( info ) {
video _info = info ;
file _size = video _info . size ;
console . log ( 'Download started' )
fs . writeJSONSync ( removeFileExtension ( video _info . _filename ) + '.info.json' , video _info ) ;
video . pipe ( fs . createWriteStream ( video _info . _filename , { flags : 'w' } ) )
} ) ;
// Will be called if download was already completed and there is nothing more to download.
video . on ( 'complete' , function complete ( info ) {
'use strict'
console . log ( 'filename: ' + info . _filename + ' already downloaded.' )
} )
let download _pos = 0 ;
video . on ( 'data' , function data ( chunk ) {
download _pos += chunk . length
// `size` should not be 0 here.
if ( file _size ) {
let percent = ( download _pos / file _size * 100 ) . toFixed ( 2 )
download [ 'percent_complete' ] = percent ;
}
} ) ;
video . on ( 'end' , function ( ) {
console . log ( 'finished downloading!' )
let new _date = Date . now ( ) ;
let difference = ( new _date - date ) / 1000 ;
logger . debug ( ` Video download delay: ${ difference } seconds. ` ) ;
download [ 'complete' ] = true ;
updateDownloads ( ) ;
// registers file in DB
const base _file _name = video _info . _filename . substring ( videoFolderPath . length , video _info . _filename . length ) ;
file _uid = registerFileDB ( base _file _name , type ) ;
if ( options . merged _string ) {
let current _merged _archive = fs . readFileSync ( videoFolderPath + 'merged.txt' , 'utf8' ) ;
let diff = current _merged _archive . replace ( options . merged _string , '' ) ;
const archive _path = path . join ( archivePath , 'archive_video.txt' ) ;
fs . appendFileSync ( archive _path , diff ) ;
}
videopathEncoded = encodeURIComponent ( removeFileExtension ( base _file _name ) ) ;
resolve ( {
videopathEncoded : videopathEncoded ,
file _names : /*is_playlist ? file_names :*/ null , // playlist support is not ready
uid : file _uid
} ) ;
} ) ;
video . on ( 'error' , function error ( err ) {
logger . error ( err ) ;
download [ error ] = err ;
updateDownloads ( ) ;
resolve ( false ) ;
} ) ;
} ) ;
}
async function generateArgs ( url , type , options ) {
return new Promise ( async resolve => {
var videopath = '%(title)s' ;
var globalArgs = config _api . getConfigItem ( 'ytdl_custom_args' ) ;
var customArgs = options . customArgs ;
var customOutput = options . customOutput ;
var selectedHeight = options . selectedHeight ;
var customQualityConfiguration = options . customQualityConfiguration ;
var youtubeUsername = options . youtubeUsername ;
var youtubePassword = options . youtubePassword ;
let downloadConfig = null ;
let qualityPath = 'best[ext=mp4]' ;
if ( url . includes ( 'tiktok' ) || url . includes ( 'pscp.tv' ) ) {
// tiktok videos fail when using the default format
qualityPath = 'best' ;
}
if ( customArgs ) {
downloadConfig = customArgs . split ( ' ' ) ;
} else {
if ( customQualityConfiguration ) {
qualityPath = customQualityConfiguration ;
} else if ( selectedHeight && selectedHeight !== '' ) {
qualityPath = ` bestvideo[height= ${ selectedHeight } ]+bestaudio/best[height= ${ selectedHeight } ] ` ;
}
if ( customOutput ) {
downloadConfig = [ '-o' , videoFolderPath + customOutput + ".mp4" , '-f' , qualityPath , '--write-info-json' , '--print-json' ] ;
} else {
downloadConfig = [ '-o' , videoFolderPath + videopath + ".mp4" , '-f' , qualityPath , '--write-info-json' , '--print-json' ] ;
}
if ( youtubeUsername && youtubePassword ) {
downloadConfig . push ( '--username' , youtubeUsername , '--password' , youtubePassword ) ;
}
if ( ! useDefaultDownloadingAgent && customDownloadingAgent ) {
downloadConfig . splice ( 0 , 0 , '--external-downloader' , customDownloadingAgent ) ;
}
let useYoutubeDLArchive = config _api . getConfigItem ( 'ytdl_use_youtubedl_archive' ) ;
if ( useYoutubeDLArchive ) {
const archive _path = path . join ( archivePath , 'archive_video.txt' ) ;
// create archive file if it doesn't exist
if ( ! fs . existsSync ( archive _path ) ) {
fs . closeSync ( fs . openSync ( archive _path , 'w' ) ) ;
}
let blacklist _path = path . join ( archivePath , 'blacklist_video.txt' ) ;
// create blacklist file if it doesn't exist
if ( ! fs . existsSync ( blacklist _path ) ) {
fs . closeSync ( fs . openSync ( blacklist _path , 'w' ) ) ;
}
let merged _path = videoFolderPath + 'merged.txt' ;
// merges blacklist and regular archive
let inputPathList = [ archive _path , blacklist _path ] ;
let status = await mergeFiles ( inputPathList , merged _path ) ;
options . merged _string = fs . readFileSync ( merged _path , "utf8" ) ;
downloadConfig . push ( '--download-archive' , merged _path ) ;
}
if ( globalArgs && globalArgs !== '' ) {
// adds global args
downloadConfig = downloadConfig . concat ( globalArgs . split ( ' ' ) ) ;
}
}
resolve ( downloadConfig ) ;
} ) ;
}
// currently only works for single urls
async function getUrlInfos ( urls ) {
let startDate = Date . now ( ) ;
@ -1117,6 +1423,57 @@ function writeToBlacklist(type, line) {
fs . appendFileSync ( blacklistPath , line ) ;
}
// download management functions
function updateDownloads ( ) {
db . assign ( { downloads : downloads } ) . write ( ) ;
}
/ *
function checkDownloads ( ) {
for ( let [ session _id , session _downloads ] of Object . entries ( downloads ) ) {
for ( let [ download _uid , download _obj ] of Object . entries ( session _downloads ) ) {
if ( download _obj && ! download _obj [ 'complete' ] && ! download _obj [ 'error' ]
&& download _obj . timestamp _start > timestamp _server _start ) {
// download is still running (presumably)
download _obj . percent _complete = getDownloadPercent ( download _obj ) ;
}
}
}
}
* /
function getDownloadPercent ( download _obj ) {
if ( ! download _obj . final _size ) {
if ( fs . existsSync ( download _obj . expected _json _path ) ) {
const file _json = JSON . parse ( fs . readFileSync ( download _obj . expected _json _path , 'utf8' ) ) ;
let calculated _filesize = null ;
if ( file _json [ 'format_id' ] ) {
calculated _filesize = 0 ;
const formats _used = file _json [ 'format_id' ] . split ( '+' ) ;
for ( let i = 0 ; i < file _json [ 'formats' ] . length ; i ++ ) {
if ( formats _used . includes ( file _json [ 'formats' ] [ i ] [ 'format_id' ] ) ) {
calculated _filesize += file _json [ 'formats' ] [ i ] [ 'filesize' ] ;
}
}
}
download _obj . final _size = calculated _filesize ;
} else {
console . log ( 'could not find json file' ) ;
}
}
if ( fs . existsSync ( download _obj . expected _path ) ) {
const stats = fs . statSync ( download _obj . expected _path ) ;
const size = stats . size ;
return ( size / download _obj . final _size ) * 100 ;
} else {
console . log ( 'could not find file' ) ;
return 0 ;
}
}
// youtube-dl functions
async function startYoutubeDL ( ) {
// auto update youtube-dl
await autoUpdateYoutubeDL ( ) ;
@ -1366,28 +1723,39 @@ app.post('/api/tomp3', async function(req, res) {
// adds download to download helper
const download _uid = uuid ( ) ;
downloads [ download _uid ] = {
const session = req . query . sessionID ? req . query . sessionID : 'undeclared' ;
if ( ! downloads [ session ] ) downloads [ session ] = { } ;
downloads [ session ] [ download _uid ] = {
uid : download _uid ,
downloading : true ,
complete : false ,
url : url ,
type : 'audio'
type : 'audio' ,
percent _complete : 0 ,
is _playlist : url . includes ( 'playlist' ) ,
timestamp _start : Date . now ( )
} ;
updateDownloads ( ) ;
youtubedl . exec ( url , downloadConfig , { } , function ( err , output ) {
downloads [ download_uid ] [ 'downloading' ] = false ;
downloads [ session] [ download_uid ] [ 'downloading' ] = false ;
var uid = null ;
let new _date = Date . now ( ) ;
let difference = ( new _date - date ) / 1000 ;
logger . debug ( ` Audio download delay: ${ difference } seconds. ` ) ;
if ( err ) {
audiopath = "-1" ;
logger . error ( err . stderr ) ;
downloads [ session ] [ download _uid ] [ 'error' ] = err . stderr ;
updateDownloads ( ) ;
res . sendStatus ( 500 ) ;
throw err ;
} else if ( output ) {
var file _names = [ ] ;
if ( output . length === 0 || output [ 0 ] . length === 0 ) {
downloads [ session ] [ download _uid ] [ 'error' ] = 'No output. Check if video already exists in your archive.' ;
updateDownloads ( ) ;
res . sendStatus ( 500 ) ;
return ;
}
@ -1435,7 +1803,8 @@ app.post('/api/tomp3', async function(req, res) {
fs . unlinkSync ( merged _path )
}
downloads [ download _uid ] [ 'complete' ] = true ;
downloads [ session ] [ download _uid ] [ 'complete' ] = true ;
updateDownloads ( ) ;
var audiopathEncoded = encodeURIComponent ( file _names [ 0 ] ) ;
res . send ( {
@ -1449,164 +1818,23 @@ app.post('/api/tomp3', async function(req, res) {
app . post ( '/api/tomp4' , async function ( req , res ) {
var url = req . body . url ;
var date = Date . now ( ) ;
var videopath = '%(title)s' ;
var globalArgs = config _api . getConfigItem ( 'ytdl_custom_args' ) ;
var customArgs = req . body . customArgs ;
var customOutput = req . body . customOutput ;
var selectedHeight = req . body . selectedHeight ;
var customQualityConfiguration = req . body . customQualityConfiguration ;
var youtubeUsername = req . body . youtubeUsername ;
var youtubePassword = req . body . youtubePassword ;
let merged _string = null ;
let downloadConfig = null ;
let qualityPath = 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4' ;
if ( url . includes ( 'tiktok' ) || url . includes ( 'pscp.tv' ) ) {
// tiktok videos fail when using the default format
qualityPath = 'best' ;
var options = {
customArgs : req . body . customArgs ,
customOutput : req . body . customOutput ,
selectedHeight : req . body . selectedHeight ,
customQualityConfiguration : req . body . customQualityConfiguration ,
youtubeUsername : req . body . youtubeUsername ,
youtubePassword : req . body . youtubePassword
}
if ( customArgs ) {
downloadConfig = customArgs . split ( ' ' ) ;
const result _obj = await downloadFileByURL _normal ( url , 'video' , options , req . query . sessionID )
if ( result _obj ) {
res . send ( result _obj ) ;
} else {
if ( customQualityConfiguration ) {
qualityPath = customQualityConfiguration ;
} else if ( selectedHeight && selectedHeight !== '' ) {
qualityPath = ` bestvideo[height= ${ selectedHeight } ]+bestaudio/best[height= ${ selectedHeight } ] ` ;
}
if ( customOutput ) {
downloadConfig = [ '-o' , videoFolderPath + customOutput + ".mp4" , '-f' , qualityPath , '--write-info-json' , '--print-json' ] ;
} else {
downloadConfig = [ '-o' , videoFolderPath + videopath + ".mp4" , '-f' , qualityPath , '--write-info-json' , '--print-json' ] ;
}
if ( youtubeUsername && youtubePassword ) {
downloadConfig . push ( '--username' , youtubeUsername , '--password' , youtubePassword ) ;
}
if ( ! useDefaultDownloadingAgent && customDownloadingAgent ) {
downloadConfig . splice ( 0 , 0 , '--external-downloader' , customDownloadingAgent ) ;
}
let useYoutubeDLArchive = config _api . getConfigItem ( 'ytdl_use_youtubedl_archive' ) ;
if ( useYoutubeDLArchive ) {
const archive _path = path . join ( archivePath , 'archive_video.txt' ) ;
// create archive file if it doesn't exist
if ( ! fs . existsSync ( archive _path ) ) {
fs . closeSync ( fs . openSync ( archive _path , 'w' ) ) ;
}
let blacklist _path = path . join ( archivePath , 'blacklist_video.txt' ) ;
// create blacklist file if it doesn't exist
if ( ! fs . existsSync ( blacklist _path ) ) {
fs . closeSync ( fs . openSync ( blacklist _path , 'w' ) ) ;
}
let merged _path = videoFolderPath + 'merged.txt' ;
// merges blacklist and regular archive
let inputPathList = [ archive _path , blacklist _path ] ;
let status = await mergeFiles ( inputPathList , merged _path ) ;
merged _string = fs . readFileSync ( merged _path , "utf8" ) ;
downloadConfig . push ( '--download-archive' , merged _path ) ;
}
if ( globalArgs && globalArgs !== '' ) {
// adds global args
downloadConfig = downloadConfig . concat ( globalArgs . split ( ' ' ) ) ;
}
res . sendStatus ( 500 ) ;
}
// adds download to download helper
const download _uid = uuid ( ) ;
downloads [ download _uid ] = {
uid : download _uid ,
downloading : true ,
complete : false ,
url : url ,
type : 'video' ,
percent _complete : 0 ,
is _playlist : url . includes ( 'playlist' )
} ;
youtubedl . exec ( url , downloadConfig , { } , function ( err , output ) {
downloads [ download _uid ] [ 'downloading' ] = false ;
var uid = null ;
let new _date = Date . now ( ) ;
let difference = ( new _date - date ) / 1000 ;
logger . debug ( ` Video download delay: ${ difference } seconds. ` ) ;
if ( err ) {
videopath = "-1" ;
logger . error ( err . stderr ) ;
res . sendStatus ( 500 ) ;
throw err ;
} else if ( output ) {
if ( output . length === 0 || output [ 0 ] . length === 0 ) {
res . sendStatus ( 500 ) ;
return ;
}
var file _names = [ ] ;
for ( let i = 0 ; i < output . length ; i ++ ) {
let output _json = null ;
try {
output _json = JSON . parse ( output [ i ] ) ;
} catch ( e ) {
output _json = null ;
}
var modified _file _name = output _json ? output _json [ 'title' ] : null ;
if ( ! output _json ) {
continue ;
}
// get filepath with no extension
const filepath _no _extension = removeFileExtension ( output _json [ '_filename' ] ) ;
var full _file _path = filepath _no _extension + '.mp4' ;
var file _name = filepath _no _extension . substring ( audioFolderPath . length , filepath _no _extension . length ) ;
// renames file if necessary due to bug
if ( ! fs . existsSync ( output _json [ '_filename' ] && fs . existsSync ( output _json [ '_filename' ] + '.webm' ) ) ) {
try {
fs . renameSync ( output _json [ '_filename' ] + '.webm' , output _json [ '_filename' ] ) ;
logger . info ( 'Renamed ' + file _name + '.webm to ' + file _name ) ;
} catch ( e ) {
}
}
// registers file in DB
uid = registerFileDB ( full _file _path . substring ( videoFolderPath . length , full _file _path . length ) , 'video' ) ;
if ( file _name ) file _names . push ( file _name ) ;
}
let is _playlist = file _names . length > 1 ;
if ( ! is _playlist ) audiopath = file _names [ 0 ] ;
if ( merged _string !== null ) {
let current _merged _archive = fs . readFileSync ( videoFolderPath + 'merged.txt' , 'utf8' ) ;
let diff = current _merged _archive . replace ( merged _string , '' ) ;
const archive _path = path . join ( archivePath , 'archive_video.txt' ) ;
fs . appendFileSync ( archive _path , diff ) ;
}
downloads [ download _uid ] [ 'complete' ] = true ;
var videopathEncoded = encodeURIComponent ( file _names [ 0 ] ) ;
res . send ( {
videopathEncoded : videopathEncoded ,
file _names : is _playlist ? file _names : null ,
uid : uid
} ) ;
res . end ( "yes" ) ;
}
} ) ;
res . end ( "yes" ) ;
} ) ;
// gets the status of the mp3 file that's being downloaded
@ -2314,9 +2542,47 @@ app.get('/api/audio/:id', function(req , res){
// Downloads management
app . get ( '/api/downloads' , async ( req , res ) => {
/ *
if ( ! last _downloads _check || Date . now ( ) - last _downloads _check > downloads _check _interval ) {
last _downloads _check = Date . now ( ) ;
updateDownloads ( ) ;
}
* /
res . send ( { downloads : downloads } ) ;
} ) ;
app . post ( '/api/clearDownloads' , async ( req , res ) => {
let success = false ;
var delete _all = req . body . delete _all ;
if ( ! req . body . session _id ) req . body . session _id = 'undeclared' ;
var session _id = req . body . session _id ;
var download _id = req . body . download _id ;
if ( delete _all ) {
// delete all downloads
downloads = { } ;
success = true ;
} else if ( download _id ) {
// delete just 1 download
if ( downloads [ session _id ] [ download _id ] ) {
delete downloads [ session _id ] [ download _id ] ;
success = true ;
} else if ( ! downloads [ session _id ] ) {
logger . error ( ` Session ${ session _id } has no downloads. ` )
} else if ( ! downloads [ session _id ] [ download _id ] ) {
logger . error ( ` Download ' ${ download _id } ' for session ' ${ session _id } ' could not be found ` ) ;
}
} else if ( session _id ) {
// delete a session's downloads
if ( downloads [ session _id ] ) {
delete downloads [ session _id ] ;
success = true ;
} else {
logger . error ( ` Session ${ session _id } has no downloads. ` )
}
}
updateDownloads ( ) ;
res . send ( { success : success , downloads : downloads } ) ;
} ) ;
app . post ( '/api/getVideoInfos' , async ( req , res ) => {
let fileNames = req . body . fileNames ;