diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0f2f132 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:18.04 + +RUN apt-get update && apt-get install -y \ + nodejs \ + apache2 \ + npm \ + youtube-dl + +# Change directory so that our commands run inside this new directory +WORKDIR /var/www/html + +# Copy dependency definitions +COPY ./ /var/www/html/ + +# Change directory to backend +WORKDIR /var/www/html/backend + +# Install dependencies on backend +RUN npm install + +# Change back to original directory +WORKDIR /var/www/html + +# Expose the port the app runs in +EXPOSE 80 + +# Run the specified command within the container. +CMD ./docker_wrapper.sh \ No newline at end of file diff --git a/backend/app.js b/backend/app.js index 2db8dcf..452056c 100644 --- a/backend/app.js +++ b/backend/app.js @@ -2,7 +2,6 @@ var async = require('async'); var fs = require('fs'); var path = require('path'); var youtubedl = require('youtube-dl'); -var config = require('config'); var https = require('https'); var express = require("express"); var bodyParser = require("body-parser"); @@ -10,6 +9,7 @@ var archiver = require('archiver'); const low = require('lowdb') var URL = require('url').URL; const shortid = require('shortid') +var config_api = require('./config.js'); var app = express(); @@ -18,65 +18,56 @@ const adapter = new FileSync('db.json'); const db = low(adapter) // Set some defaults -db.defaults({ playlists: { - audio: [], - video: [] -}}).write(); - +db.defaults( + { + playlists: { + audio: [], + video: [] + }, + configWriteFlag: false +}).write(); + +// config values +var frontendUrl = null; +var backendUrl = null; +var backendPort = 17442; +var usingEncryption = null; +var basePath = null; +var audioFolderPath = null; +var videoFolderPath = null; +var downloadOnlyMode = null; +var useDefaultDownloadingAgent = null; +var customDownloadingAgent = null; + +// other needed values +var options = null; // encryption options +var url_domain = null; // check if debug mode let debugMode = process.env.YTDL_MODE === 'debug'; if (debugMode) console.log('YTDL-Material in debug mode!'); -var frontendUrl = !debugMode ? config.get("YoutubeDLMaterial.Host.frontendurl") : 'http://localhost:4200'; -var backendUrl = config.get("YoutubeDLMaterial.Host.backendurl") -var backendPort = 17442; -var usingEncryption = config.get("YoutubeDLMaterial.Encryption.use-encryption"); -var basePath = config.get("YoutubeDLMaterial.Downloader.path-base"); -var audioFolderPath = config.get("YoutubeDLMaterial.Downloader.path-audio"); -var videoFolderPath = config.get("YoutubeDLMaterial.Downloader.path-video"); -var downloadOnlyMode = config.get("YoutubeDLMaterial.Extra.download_only_mode") -var useDefaultDownloadingAgent = config.get("YoutubeDLMaterial.Advanced.use_default_downloading_agent"); -var customDownloadingAgent = config.get("YoutubeDLMaterial.Advanced.custom_downloading_agent"); var validDownloadingAgents = [ 'aria2c' ] -if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) { - console.log(`INFO: Using non-default downloading agent \'${customDownloadingAgent}\'`) -} - -var descriptors = {}; - - -if (usingEncryption) -{ - - var certFilePath = path.resolve(config.get("YoutubeDLMaterial.Encryption.cert-file-path")); - var keyFilePath = path.resolve(config.get("YoutubeDLMaterial.Encryption.key-file-path")); - var certKeyFile = fs.readFileSync(keyFilePath); - var certFile = fs.readFileSync(certFilePath); +// don't overwrite config if it already happened.. NOT +// let alreadyWritten = db.get('configWriteFlag').value(); +let writeConfigMode = process.env.write_ytdl_config; +var config = null; - var options = { - key: certKeyFile, - cert: certFile - }; +if (writeConfigMode) { + setAndLoadConfig(); +} else { + loadConfig(); } - +var descriptors = {}; app.use(bodyParser.urlencoded({ extended: false })); app.use(bodyParser.json()); -var url_domain = new URL(frontendUrl); - -app.use(function(req, res, next) { - res.header("Access-Control-Allow-Origin", url_domain.origin); - res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); - next(); -}); - app.get('/using-encryption', function(req, res) { res.send(usingEncryption); res.end("yes"); @@ -94,6 +85,109 @@ function File(id, title, thumbnailURL, isAudio, duration) { // actual functions +function startServer() { + if (usingEncryption) + { + https.createServer(options, app).listen(backendPort, function() { + console.log('HTTPS: Anchor set on 17442'); + }); + } + else + { + app.listen(backendPort,function(){ + console.log("HTTP: Started on PORT " + backendPort); + }); + } +} + +async function setAndLoadConfig() { + await setConfigFromEnv(); + await loadConfig(); + // console.log(backendUrl); +} + +async function setConfigFromEnv() { + return new Promise(resolve => { + let config_items = getEnvConfigItems(); + let success = config_api.setConfigItems(config_items); + if (success) { + console.log('Config items set using ENV variables.'); + setTimeout(() => resolve(true), 100); + } else { + console.log('ERROR: Failed to set config items using ENV variables.'); + resolve(false); + } + }); +} + +async function loadConfig() { + return new Promise(resolve => { + // get config library + // config = require('config'); + + frontendUrl = !debugMode ? config_api.getConfigItem('ytdl_frontend_url') : 'http://localhost:4200'; + backendUrl = config_api.getConfigItem('ytdl_backend_url') + backendPort = 17442; + usingEncryption = config_api.getConfigItem('ytdl_use_encryption'); + basePath = config_api.getConfigItem('ytdl_base_path'); + audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); + videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); + downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode'); + useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent'); + customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent'); + if (!useDefaultDownloadingAgent && validDownloadingAgents.indexOf(customDownloadingAgent) !== -1 ) { + console.log(`INFO: Using non-default downloading agent \'${customDownloadingAgent}\'`) + } + + if (usingEncryption) + { + var certFilePath = path.resolve(config_api.getConfigItem('ytdl_cert_file_path')); + var keyFilePath = path.resolve(config_api.getConfigItem('ytdl_key_file_path')); + + var certKeyFile = fs.readFileSync(keyFilePath); + var certFile = fs.readFileSync(certFilePath); + + options = { + key: certKeyFile, + cert: certFile + }; + } + + url_domain = new URL(frontendUrl); + + // start the server here + startServer(); + + resolve(true); + }); + +} + +function getOrigin() { + return url_domain.origin; +} + +// gets a list of config items that are stored as an environment variable +function getEnvConfigItems() { + let config_items = []; + + let config_item_keys = Object.keys(config_api.CONFIG_ITEMS); + for (let i = 0; i < config_item_keys.length; i++) { + let key = config_item_keys[i]; + if (process['env'][key]) { + const config_item = generateEnvVarConfigItem(key); + config_items.push(config_item); + } + } + + return config_items; +} + +// gets value of a config item and stores it in an object +function generateEnvVarConfigItem(key) { + return {key: key, value: process['env'][key]}; +} + function getThumbnailMp3(name) { var obj = getJSONMp3(name); @@ -382,6 +476,12 @@ async function getUrlInfos(urls) { }); } +app.use(function(req, res, next) { + res.header("Access-Control-Allow-Origin", getOrigin()); + res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); + next(); +}); + app.post('/tomp3', function(req, res) { var url = req.body.url; var date = Date.now(); @@ -879,19 +979,3 @@ app.get('/audio/:id', function(req , res){ success: !!result }) }); - - - - -if (usingEncryption) -{ - https.createServer(options, app).listen(backendPort, function() { - console.log('HTTPS: Anchor set on 17442'); - }); -} -else -{ - app.listen(backendPort,function(){ - console.log("HTTP: Started on PORT " + backendPort); - }); -} \ No newline at end of file diff --git a/backend/config.js b/backend/config.js new file mode 100644 index 0000000..0ba7dbe --- /dev/null +++ b/backend/config.js @@ -0,0 +1,112 @@ +const fs = require('fs'); + +let CONFIG_ITEMS = require('./consts.js')['CONFIG_ITEMS']; + +let configPath = 'config/default.json'; + +// https://stackoverflow.com/questions/6491463/accessing-nested-javascript-objects-with-string-key +Object.byString = function(o, s) { + s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties + s = s.replace(/^\./, ''); // strip a leading dot + var a = s.split('.'); + for (var i = 0, n = a.length; i < n; ++i) { + var k = a[i]; + if (k in o) { + o = o[k]; + } else { + return; + } + } + return o; +} + +function getParentPath(path) { + let elements = path.split('.'); + elements.splice(elements.length - 1, 1); + return elements.join('.'); +} + +function getElementNameInConfig(path) { + let elements = path.split('.'); + return elements[elements.length - 1]; +} + +/* +* Gets config file and returns as a json +*/ +function getConfigFile() { + let raw_data = fs.readFileSync(configPath); + try { + let parsed_data = JSON.parse(raw_data); + return parsed_data; + } catch(e) { + console.log('ERROR: Failed to get config file'); + return null; + } +} + +function setConfigFile(config) { + try { + fs.writeFileSync(configPath, JSON.stringify(config, null, 2)); + return true; + } catch(e) { + return false; + } +} + +function getConfigItem(key) { + let config_json = getConfigFile(); + if (!CONFIG_ITEMS[key]) console.log('cannot find config with key ' + key); + let path = CONFIG_ITEMS[key]['path']; + return Object.byString(config_json, path); +}; + +function setConfigItem(key, value) { + let success = false; + let config_json = getConfigFile(); + let path = CONFIG_ITEMS[key]['path']; + let parent_path = getParentPath(path); + let element_name = getElementNameInConfig(path); + + let parent_object = Object.byString(config_json, parent_path); + if (value === 'false' || value === 'true') { + parent_object[element_name] = (value === 'true'); + } else { + parent_object[element_name] = value; + } + + success = setConfigFile(config_json); + + return success; +}; + +function setConfigItems(items) { + let success = false; + let config_json = getConfigFile(); + for (let i = 0; i < items.length; i++) { + let key = items[i].key; + let value = items[i].value; + + // if boolean strings, set to booleans again + if (value === 'false' || value === 'true') { + value = (value === 'true'); + } + + let item_path = CONFIG_ITEMS[key]['path']; + let item_parent_path = getParentPath(item_path); + let item_element_name = getElementNameInConfig(item_path); + + let item_parent_object = Object.byString(config_json, item_parent_path); + item_parent_object[item_element_name] = value; + } + + success = setConfigFile(config_json); + return success; +} + +module.exports = { + getConfigItem: getConfigItem, + setConfigItem: setConfigItem, + setConfigItems: setConfigItems, + CONFIG_ITEMS: CONFIG_ITEMS +} \ No newline at end of file diff --git a/backend/consts.js b/backend/consts.js new file mode 100644 index 0000000..e645a53 --- /dev/null +++ b/backend/consts.js @@ -0,0 +1,91 @@ +var config = require('config'); + +let CONFIG_ITEMS = { + // Host + 'ytdl_frontend_url': { + 'key': 'ytdl_frontend_url', + 'path': 'YoutubeDLMaterial.Host.frontendurl' + }, + 'ytdl_backend_url': { + 'key': 'ytdl_backend_url', + 'path': 'YoutubeDLMaterial.Host.backendurl' + }, + + // Encryption + 'ytdl_use_encryption': { + 'key': 'ytdl_use_encryption', + 'path': 'YoutubeDLMaterial.Encryption.use-encryption' + }, + 'ytdl_cert_file_path': { + 'key': 'ytdl_cert_file_path', + 'path': 'YoutubeDLMaterial.Encryption.cert-file-path' + }, + 'ytdl_key_file_path': { + 'key': 'ytdl_key_file_path', + 'path': 'YoutubeDLMaterial.Encryption.key-file-path' + }, + + // Downloader + 'ytdl_base_path': { + 'key': 'ytdl_base_path', + 'path': 'YoutubeDLMaterial.Downloader.path-base' + }, + 'ytdl_audio_folder_path': { + 'key': 'ytdl_audio_folder_path', + 'path': 'YoutubeDLMaterial.Downloader.path-audio' + }, + 'ytdl_video_folder_path': { + 'key': 'ytdl_video_folder_path', + 'path': 'YoutubeDLMaterial.Downloader.path-video' + }, + + // Extra + 'ytdl_title_top': { + 'key': 'ytdl_title_top', + 'path': 'YoutubeDLMaterial.Extra.title_top' + }, + 'ytdl_file_manager_enabled': { + 'key': 'ytdl_file_manager_enabled', + 'path': 'YoutubeDLMaterial.Extra.file_manager_enabled' + }, + 'ytdl_allow_quality_select': { + 'key': 'ytdl_allow_quality_select', + 'path': 'YoutubeDLMaterial.Extra.allow_quality_select' + }, + 'ytdl_download_only_mode': { + 'key': 'ytdl_download_only_mode', + 'path': 'YoutubeDLMaterial.Extra.download_only_mode' + }, + + // API + 'ytdl_use_youtube_api': { + 'key': 'ytdl_use_youtube_api', + 'path': 'YoutubeDLMaterial.API.use_youtube_API' + }, + 'ytdl_youtube_api_key': { + 'key': 'ytdl_youtube_api_key', + 'path': 'YoutubeDLMaterial.API.youtube_API_key' + }, + + // Themes + 'ytdl_default_theme': { + 'key': 'ytdl_default_theme', + 'path': 'YoutubeDLMaterial.Themes.default_theme' + }, + 'ytdl_allow_theme_change': { + 'key': 'ytdl_allow_theme_change', + 'path': 'YoutubeDLMaterial.Themes.allow_theme_change' + }, + + // Advanced + 'ytdl_use_default_downloading_agent': { + 'key': 'ytdl_use_default_downloading_agent', + 'path': 'YoutubeDLMaterial.Advanced.use_default_downloading_agent' + }, + 'ytdl_custom_downloading_agent': { + 'key': 'ytdl_custom_downloading_agent', + 'path': 'YoutubeDLMaterial.Advanced.custom_downloading_agent' + }, +}; + +module.exports.CONFIG_ITEMS = CONFIG_ITEMS; \ No newline at end of file diff --git a/backend/package.json b/backend/package.json index be33409..a6e40b7 100644 --- a/backend/package.json +++ b/backend/package.json @@ -4,7 +4,8 @@ "description": "backend for YoutubeDL-Material", "main": "index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node app.js" }, "repository": { "type": "git", @@ -24,6 +25,6 @@ "express": "^4.17.1", "lowdb": "^1.0.0", "shortid": "^2.2.15", - "youtube-dl": "^2.3.0" + "youtube-dl": "^3.0.2" } } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..76bbf7e --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ + +version: "3.2" +services: + ytdl_material: + build: . + environment: + # config items + ytdl_frontend_url: http://localhost:8998 + ytdl_backend_url: http://localhost:17442/ + ytdl_use_encryption: 'false' + ytdl_cert_file_path: /etc/letsencrypt/live/example.com/fullchain.pem + ytdl_key_file_path: /etc/letsencrypt/live/example.com/privkey.pem + ytdl_base_path: http://localhost:17442/ + ytdl_audio_folder_path: audio/ + ytdl_video_folder_path: video/ + ytdl_title_top: Youtube Downloader + ytdl_file_manager_enabled: 'true' + ytdl_allow_quality_select: 'true' + ytdl_download_only_mode: 'false' + ytdl_use_youtube_api: 'false' + ytdl_youtube_api_key: 'false' + ytdl_default_theme: default + ytdl_allow_theme_change: 'true' + ytdl_use_default_downloading_agent: 'true' + ytdl_custom_downloading_agent: 'false' + write_ytdl_config: 'true' + # do not touch this + ALLOW_CONFIG_MUTATIONS: 'true' + restart: always + ports: + - "17442:17442" + - "8998:80" + image: tzahi12345/youtubedl-material \ No newline at end of file diff --git a/docker_wrapper.sh b/docker_wrapper.sh new file mode 100644 index 0000000..ed24447 --- /dev/null +++ b/docker_wrapper.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +cd backend + +# Start the first process +node app.js & +status=$? +if [ $status -ne 0 ]; then + echo "Failed to start my_first_process: $status" + exit $status +fi + +# Start the second process +apachectl -DFOREGROUND +status=$? +if [ $status -ne 0 ]; then + echo "Failed to start my_second_process: $status" + exit $status +fi + +# Naive check runs checks once a minute to see if either of the processes exited. +# This illustrates part of the heavy lifting you need to do if you want to run +# more than one service in a container. The container will exit with an error +# if it detects that either of the processes has exited. +# Otherwise it will loop forever, waking up every 60 seconds + +while /bin/true; do + ps aux |grep node\ app.js # |grep -q -v grep + PROCESS_1_STATUS=$? + ps aux |grep apache2 # |grep -q -v grep + PROCESS_2_STATUS=$? + # If the greps above find anything, they will exit with 0 status + # If they are not both 0, then something is wrong + if [ $PROCESS_1_STATUS -ne 0 -o $PROCESS_2_STATUS -ne 0 ]; then + echo "One of the processes has already exited." + exit -1 + fi + sleep 60 +done \ No newline at end of file diff --git a/package.json b/package.json index 4284f86..e3f6719 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,7 @@ "test": "ng test", "lint": "ng lint", "e2e": "ng e2e", - "electron": "ng build --base-href ./ && electron .", - "postinstall": "ng build --prod && mkdir dist/backend && mkdir dist/backend/config && mkdir dist/backend/audio && mkdir dist/backend/video && cp src/assets/default.json dist/backend/config/default.json && cp backend/app.js dist/backend/app.js && cp backend/package.json dist/backend/package.json && cd dist/backend && npm install" + "electron": "ng build --base-href ./ && electron ." }, "engines": { "node": "12.3.1",