@ -1,6 +1,6 @@
var async = require ( 'async' ) ;
const { uuid } = require ( 'uuidv4' ) ;
var fs = require ( 'fs ') ;
var fs = require ( 'fs -extra ') ;
var path = require ( 'path' ) ;
var youtubedl = require ( 'youtube-dl' ) ;
var compression = require ( 'compression' ) ;
@ -8,8 +8,10 @@ var https = require('https');
var express = require ( "express" ) ;
var bodyParser = require ( "body-parser" ) ;
var archiver = require ( 'archiver' ) ;
var unzipper = require ( 'unzipper' ) ;
var mergeFiles = require ( 'merge-files' ) ;
const low = require ( 'lowdb' )
var ProgressBar = require ( 'progress' ) ;
var md5 = require ( 'md5' ) ;
const NodeID3 = require ( 'node-id3' )
const downloader = require ( 'youtube-dl/lib/downloader' )
@ -19,6 +21,8 @@ const shortid = require('shortid')
const url _api = require ( 'url' ) ;
var config _api = require ( './config.js' ) ;
var subscriptions _api = require ( './subscriptions' )
const CONSTS = require ( './consts' )
const { spawn } = require ( 'child_process' )
var app = express ( ) ;
@ -26,6 +30,8 @@ const FileSync = require('lowdb/adapters/FileSync')
const adapter = new FileSync ( './appdata/db.json' ) ;
const db = low ( adapter )
// var GithubContent = require('github-content');
// Set some defaults
db . defaults (
{
@ -56,12 +62,23 @@ var archivePath = path.join(__dirname, 'appdata', 'archives');
// other needed values
var options = null ; // encryption options
var url _domain = null ;
var updaterStatus = null ;
// check if debug mode
let debugMode = process . env . YTDL _MODE === 'debug' ;
if ( debugMode ) console . log ( 'YTDL-Material in debug mode!' ) ;
// check if just updated
const just _restarted = fs . existsSync ( 'restart.json' ) ;
if ( just _restarted ) {
updaterStatus = {
updating : false ,
details : 'Update complete! You are now on ' + CONSTS [ 'CURRENT_VERSION' ]
}
fs . unlinkSync ( 'restart.json' ) ;
}
// updates & starts youtubedl
startYoutubeDL ( ) ;
@ -122,15 +139,248 @@ async function startServer() {
if ( usingEncryption )
{
https . createServer ( options , app ) . listen ( backendPort , function ( ) {
console . log ( 'HTTPS: Started on PORT ' + backendPort ) ;
console . log ( ` YoutubeDL-Material ${ CONSTS [ 'CURRENT_VERSION' ] } started on port ${ backendPort } - using SSL ` ) ;
} ) ;
}
else
{
app . listen ( backendPort , function ( ) {
console . log ( "HTTP: Started on PORT " + backendPort ) ;
console . log ( ` YoutubeDL-Material ${ CONSTS [ 'CURRENT_VERSION' ] } started on PORT ${ backendPort } ` ) ;
} ) ;
}
}
async function restartServer ( ) {
const restartProcess = ( ) => {
spawn ( 'node' , [ 'app.js' ] , {
detached : true ,
stdio : 'inherit'
} ) . unref ( )
process . exit ( )
}
console . log ( 'Update complete! Restarting server...' ) ;
// the following line restarts the server through nodemon
fs . writeFileSync ( 'restart.json' , 'internal use only' ) ;
}
async function updateServer ( tag ) {
// no tag provided means update to the latest version
if ( ! tag ) {
const new _version _available = await isNewVersionAvailable ( ) ;
if ( ! new _version _available ) {
console . log ( 'ERROR: Failed to update - no update is available.' ) ;
return false ;
}
}
return new Promise ( async resolve => {
// backup current dir
updaterStatus = {
updating : true ,
'details' : 'Backing up key server files...'
}
let backup _succeeded = await backupServerLite ( ) ;
if ( ! backup _succeeded ) {
resolve ( false ) ;
return false ;
}
updaterStatus = {
updating : true ,
'details' : 'Downloading requested release...'
}
// grab new package.json and public folder
// await downloadReleaseFiles(tag);
updaterStatus = {
updating : true ,
'details' : 'Installing new dependencies...'
}
// run npm install
await installDependencies ( ) ;
updaterStatus = {
updating : true ,
'details' : 'Update complete! Restarting server...'
}
restartServer ( ) ;
} , err => {
updaterStatus = {
updating : false ,
error : true ,
'details' : 'Update failed. Check error logs for more info.'
}
} ) ;
}
async function downloadReleaseFiles ( tag ) {
tag = tag ? tag : await getLatestVersion ( ) ;
return new Promise ( async resolve => {
console . log ( 'Downloading new files...' )
// downloads the latest release zip file
await downloadReleaseZip ( tag ) ;
// deletes contents of public dir
fs . removeSync ( path . join ( _ _dirname , 'public' ) ) ;
fs . mkdirSync ( path . join ( _ _dirname , 'public' ) ) ;
let replace _ignore _list = [ 'youtubedl-material/appdata/default.json' ,
'youtubedl-material/appdata/db.json' ]
console . log ( ` Installing update ${ tag } ... ` )
// downloads new package.json and adds new public dir files from the downloaded zip
fs . createReadStream ( path . join ( _ _dirname , ` youtubedl-material-latest-release- ${ tag } .zip ` ) ) . pipe ( unzipper . Parse ( ) )
. on ( 'entry' , function ( entry ) {
var fileName = entry . path ;
var type = entry . type ; // 'Directory' or 'File'
var size = entry . size ;
var is _dir = fileName . substring ( fileName . length - 1 , fileName . length ) === '/'
if ( ! is _dir && fileName . includes ( 'youtubedl-material/public/' ) ) {
// get public folder files
var actualFileName = fileName . replace ( 'youtubedl-material/public/' , '' ) ;
if ( actualFileName . length !== 0 && actualFileName . substring ( actualFileName . length - 1 , actualFileName . length ) !== '/' ) {
fs . ensureDirSync ( path . join ( _ _dirname , 'public' , path . dirname ( actualFileName ) ) ) ;
entry . pipe ( fs . createWriteStream ( path . join ( _ _dirname , 'public' , actualFileName ) ) ) ;
} else {
entry . autodrain ( ) ;
}
} else if ( ! is _dir && ! replace _ignore _list . includes ( fileName ) ) {
// get package.json
var actualFileName = fileName . replace ( 'youtubedl-material/' , '' ) ;
if ( debugMode ) console . log ( 'Downloading file ' + actualFileName ) ;
entry . pipe ( fs . createWriteStream ( path . join ( _ _dirname , actualFileName ) ) ) ;
} else {
entry . autodrain ( ) ;
}
} )
. on ( 'close' , function ( ) {
resolve ( true ) ;
} ) ;
} ) ;
}
// helper function to download file using fetch
async function fetchFile ( url , path , file _label ) {
var len = null ;
const res = await fetch ( url ) ;
len = parseInt ( res . headers . get ( "Content-Length" ) , 10 ) ;
var bar = new ProgressBar ( ` Downloading ${ file _label } [:bar] :percent :etas ` , {
complete : '=' ,
incomplete : ' ' ,
width : 20 ,
total : len
} ) ;
const fileStream = fs . createWriteStream ( path ) ;
await new Promise ( ( resolve , reject ) => {
res . body . pipe ( fileStream ) ;
res . body . on ( "error" , ( err ) => {
reject ( err ) ;
} ) ;
res . body . on ( 'data' , function ( chunk ) {
bar . tick ( chunk . length ) ;
} ) ;
fileStream . on ( "finish" , function ( ) {
resolve ( ) ;
} ) ;
} ) ;
}
async function downloadReleaseZip ( tag ) {
console . log ( 'downloading' ) ;
return new Promise ( async resolve => {
// get name of zip file, which depends on the version
const latest _release _link = ` https://github.com/Tzahi12345/YoutubeDL-Material/releases/download/ ${ tag } / ` ;
const tag _without _v = tag . substring ( 1 , tag . length ) ;
const zip _file _name = ` youtubedl-material- ${ tag _without _v } .zip `
const latest _zip _link = latest _release _link + zip _file _name ;
let output _path = path . join ( _ _dirname , ` youtubedl-material-release- ${ tag } .zip ` ) ;
// download zip from release
await fetchFile ( latest _zip _link , output _path , 'update ' + tag ) ;
resolve ( true ) ;
} ) ;
}
async function installDependencies ( ) {
return new Promise ( resolve => {
var child _process = require ( 'child_process' ) ;
child _process . execSync ( 'npm install' , { stdio : [ 0 , 1 , 2 ] } ) ;
resolve ( true ) ;
} ) ;
}
async function backupServerLite ( ) {
return new Promise ( async resolve => {
let output _path = ` backup- ${ Date . now ( ) } .zip ` ;
console . log ( ` Backing up your non-video/audio files to ${ output _path } . This may take up to a few seconds/minutes. ` ) ;
let output = fs . createWriteStream ( path . join ( _ _dirname , output _path ) ) ;
var archive = archiver ( 'zip' , {
gzip : true ,
zlib : { level : 9 } // Sets the compression level.
} ) ;
archive . on ( 'error' , function ( err ) {
console . log ( err ) ;
resolve ( false ) ;
} ) ;
// pipe archive data to the output file
archive . pipe ( output ) ;
// ignore certain directories (ones with video or audio files)
const files _to _ignore = [ path . join ( config _api . getConfigItem ( 'ytdl_subscriptions_base_path' ) , '**' ) ,
path . join ( config _api . getConfigItem ( 'ytdl_audio_folder_path' ) , '**' ) ,
path . join ( config _api . getConfigItem ( 'ytdl_video_folder_path' ) , '**' ) ,
'backup-*.zip' ] ;
archive . glob ( '**/*' , {
ignore : files _to _ignore
} ) ;
await archive . finalize ( ) ;
// wait a tiny bit for the zip to reload in fs
setTimeout ( function ( ) {
resolve ( true ) ;
} , 100 ) ;
} ) ;
}
async function isNewVersionAvailable ( ) {
return new Promise ( async resolve => {
// gets tag of the latest version of youtubedl-material, compare to current version
const latest _tag = await getLatestVersion ( ) ;
const current _tag = CONSTS [ 'CURRENT_VERSION' ] ;
if ( latest _tag > current _tag ) {
resolve ( true ) ;
} else {
resolve ( false ) ;
}
} ) ;
}
async function getLatestVersion ( ) {
return new Promise ( resolve => {
fetch ( 'https://api.github.com/repos/tzahi12345/youtubedl-material/releases/latest' , { method : 'Get' } )
. then ( async res => res . json ( ) )
. then ( async ( json ) => {
if ( json [ 'message' ] ) {
// means there's an error in getting latest version
console . log ( ` ERROR: Received the following message from GitHub's API: ` ) ;
console . log ( json [ 'message' ] ) ;
if ( json [ 'documentation_url' ] ) console . log ( ` Associated URL: ${ json [ 'documentation_url' ] } ` )
}
resolve ( json [ 'tag_name' ] ) ;
return ;
} ) ;
} ) ;
}
async function setPortItemFromENV ( ) {
@ -143,7 +393,6 @@ async function setPortItemFromENV() {
async function setAndLoadConfig ( ) {
await setConfigFromEnv ( ) ;
await loadConfig ( ) ;
// console.log(backendUrl);
}
async function setConfigFromEnv ( ) {
@ -162,9 +411,6 @@ async function setConfigFromEnv() {
async function loadConfig ( ) {
return new Promise ( resolve => {
// get config library
// config = require('config');
url = ! debugMode ? config _api . getConfigItem ( 'ytdl_url' ) : 'http://localhost:4200' ;
backendPort = config _api . getConfigItem ( 'ytdl_port' ) ;
usingEncryption = config _api . getConfigItem ( 'ytdl_use_encryption' ) ;
@ -657,7 +903,7 @@ async function autoUpdateYoutubeDL() {
let current _app _details _path = 'node_modules/youtube-dl/bin/details' ;
let current _app _details _exists = fs . existsSync ( current _app _details _path ) ;
if ( ! current _app _details _exists ) {
console . log ( ` : ${ current _app _details _path } ` ) ;
console . log ( ` ERROR: Failed to get youtube-dl binary details at location ' ${ current _app _details _path } ' . Cancelling update check.` ) ;
resolve ( false ) ;
return ;
}
@ -665,9 +911,17 @@ async function autoUpdateYoutubeDL() {
let current _version = current _app _details [ 'version' ] ;
let stored _binary _path = current _app _details [ 'path' ] ;
if ( ! stored _binary _path || typeof stored _binary _path !== 'string' ) {
console . log ( ` Failed to get youtube-dl binary path at location: ${ current _app _details _path } . Cancelling update check. ` ) ;
resolve ( false ) ;
return ;
// console.log(`INFO: Failed to get youtube-dl binary path at location: ${current_app_details_path}, attempting to guess actual path...`);
const guessed _base _path = 'node_modules/youtube-dl/bin/' ;
const guessed _file _path = guessed _base _path + 'youtube-dl' + ( process . platform === 'win32' ? '.exe' : '' ) ;
if ( fs . existsSync ( guessed _file _path ) ) {
stored _binary _path = guessed _file _path ;
// console.log('INFO: Guess successful! Update process continuing...')
} else {
console . log ( ` ERROR: Guess ' ${ guessed _file _path } ' is not correct. Cancelling update check. Verify that your youtube-dl binaries exist by running npm install. ` ) ;
resolve ( false ) ;
return ;
}
}
// got version, now let's check the latest version from the youtube-dl API
@ -676,7 +930,11 @@ async function autoUpdateYoutubeDL() {
. then ( async res => res . json ( ) )
. then ( async ( json ) => {
// check if the versions are different
const latest _update _version = json [ 0 ] [ 'name' ] ;
if ( ! json || ! json [ 0 ] ) {
resolve ( false ) ;
return false ;
}
const latest _update _version = json [ 0 ] [ 'name' ] ;
if ( current _version !== latest _update _version ) {
let binary _path = 'node_modules/youtube-dl/bin' ;
// versions different, download new update
@ -729,6 +987,21 @@ async function checkExistsWithTimeout(filePath, timeout) {
} ) ;
}
// https://stackoverflow.com/a/32197381/8088021
const deleteFolderRecursive = function ( folder _to _delete ) {
if ( fs . existsSync ( folder _to _delete ) ) {
fs . readdirSync ( folder _to _delete ) . forEach ( ( file , index ) => {
const curPath = path . join ( folder _to _delete , file ) ;
if ( fs . lstatSync ( curPath ) . isDirectory ( ) ) { // recurse
deleteFolderRecursive ( curPath ) ;
} else { // delete file
fs . unlinkSync ( curPath ) ;
}
} ) ;
fs . rmdirSync ( folder _to _delete ) ;
}
} ;
app . use ( function ( req , res , next ) {
res . header ( "Access-Control-Allow-Origin" , getOrigin ( ) ) ;
res . header ( "Access-Control-Allow-Headers" , "Origin, X-Requested-With, Content-Type, Accept" ) ;
@ -1469,6 +1742,32 @@ app.post('/api/downloadArchive', async (req, res) => {
} ) ;
// Updater API calls
app . get ( '/api/updaterStatus' , async ( req , res ) => {
let status = updaterStatus ;
if ( status ) {
res . send ( updaterStatus ) ;
} else {
res . sendStatus ( 404 ) ;
}
} ) ;
app . post ( '/api/updateServer' , async ( req , res ) => {
let tag = req . body . tag ;
updateServer ( tag ) ;
res . send ( {
success : true
} ) ;
} ) ;
// Pin API calls
app . post ( '/api/isPinSet' , async ( req , res ) => {
let stored _pin = db . get ( 'pin_md5' ) . value ( ) ;
let is _set = false ;
Tzahi12345
6 years ago
committed by
GitHub