diff --git a/backend/app.js b/backend/app.js index 97bc52e..b50dd99 100644 --- a/backend/app.js +++ b/backend/app.js @@ -8,6 +8,7 @@ var https = require('https'); var express = require("express"); var bodyParser = require("body-parser"); var archiver = require('archiver'); +var mergeFiles = require('merge-files'); const low = require('lowdb') var URL = require('url').URL; const shortid = require('shortid') @@ -40,6 +41,7 @@ var usingEncryption = null; var basePath = null; var audioFolderPath = null; var videoFolderPath = null; +var useYoutubeDLArchive = null; var downloadOnlyMode = null; var useDefaultDownloadingAgent = null; var customDownloadingAgent = null; @@ -132,6 +134,7 @@ async function loadConfig() { usingEncryption = config_api.getConfigItem('ytdl_use_encryption'); audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); + useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); downloadOnlyMode = config_api.getConfigItem('ytdl_download_only_mode'); useDefaultDownloadingAgent = config_api.getConfigItem('ytdl_use_default_downloading_agent'); customDownloadingAgent = config_api.getConfigItem('ytdl_custom_downloading_agent'); @@ -382,7 +385,7 @@ async function createPlaylistZipFile(fileNames, type, outputName) { } -function deleteAudioFile(name) { +async function deleteAudioFile(name, blacklistMode = false) { return new Promise(resolve => { // TODO: split descriptors into audio and video descriptors, as deleting an audio file will close all video file streams var jsonPath = path.join(audioFolderPath,name+'.mp3.info.json'); @@ -403,7 +406,19 @@ function deleteAudioFile(name) { } } - + if (useYoutubeDLArchive) { + const archive_path = audioFolderPath + 'archive.txt'; + + // get ID from JSON + + var jsonobj = getJSONMp3(name); + let id = null; + if (jsonobj) id = jsonobj.id; + + // use subscriptions API to remove video from the archive file, and write it to the blacklist + const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null; + if (blacklistMode && line) writeToBlacklist('audio', line); + } if (jsonExists) fs.unlinkSync(jsonPath); if (audioFileExists) { @@ -422,7 +437,7 @@ function deleteAudioFile(name) { }); } -async function deleteVideoFile(name, customPath = null) { +async function deleteVideoFile(name, customPath = null, blacklistMode = false) { return new Promise(resolve => { let filePath = customPath ? customPath : videoFolderPath; var jsonPath = path.join(filePath,name+'.info.json'); @@ -443,7 +458,19 @@ async function deleteVideoFile(name, customPath = null) { } } - + if (useYoutubeDLArchive) { + const archive_path = videoFolderPath + 'archive.txt'; + + // get ID from JSON + + var jsonobj = getJSONMp4(name); + let id = null; + if (jsonobj) id = jsonobj.id; + + // use subscriptions API to remove video from the archive file, and write it to the blacklist + const line = id ? subscriptions_api.removeIDFromArchive(archive_path, id) : null; + if (blacklistMode && line) writeToBlacklist('video', line); + } if (jsonExists) fs.unlinkSync(jsonPath); if (videoFileExists) { @@ -549,6 +576,13 @@ async function getUrlInfos(urls) { }); } +function writeToBlacklist(type, line) { + let blacklistBasePath = (type === 'audio') ? audioFolderPath : videoFolderPath; + // adds newline to the beginning of the line + line = '\n' + line; + fs.appendFileSync(blacklistBasePath + 'blacklist.txt', line); +} + app.use(function(req, res, next) { res.header("Access-Control-Allow-Origin", getOrigin()); res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept"); @@ -583,7 +617,7 @@ app.get('/api/using-encryption', function(req, res) { res.send(usingEncryption); }); -app.post('/api/tomp3', function(req, res) { +app.post('/api/tomp3', async function(req, res) { var url = req.body.url; var date = Date.now(); var audiopath = '%(title)s'; @@ -596,10 +630,11 @@ app.post('/api/tomp3', function(req, res) { var youtubeUsername = req.body.youtubeUsername; var youtubePassword = req.body.youtubePassword; - let downloadConfig = null; let qualityPath = '-f bestaudio'; + let merged_string = null; + if (customArgs) { downloadConfig = customArgs.split(' '); } else { @@ -628,6 +663,29 @@ app.post('/api/tomp3', function(req, res) { downloadConfig.splice(0, 0, '--external-downloader', 'aria2c'); } + if (useYoutubeDLArchive) { + let archive_path = audioFolderPath + 'archive.txt'; + // create archive file if it doesn't exist + if (!fs.existsSync(archive_path)) { + fs.closeSync(fs.openSync(archive_path, 'w')); + } + + let blacklist_path = audioFolderPath + 'blacklist.txt'; + // create blacklist file if it doesn't exist + if (!fs.existsSync(blacklist_path)) { + fs.closeSync(fs.openSync(blacklist_path, 'w')); + } + + let merged_path = audioFolderPath + 'merged.txt'; + // merges blacklist and regular archive + let inputPathList = [archive_path, blacklist_path]; + let status = await mergeFiles(inputPathList, merged_path); + + merged_string = fs.readFileSync(merged_path, "utf8"); + + downloadConfig.push('--download-archive', merged_path); + } + if (globalArgs && globalArgs !== '') { // adds global args downloadConfig = downloadConfig.concat(globalArgs.split(' ')); @@ -647,6 +705,10 @@ app.post('/api/tomp3', function(req, res) { throw err; } else if (output) { var file_names = []; + if (output.length === 0 || output[0].length === 0) { + res.sendStatus(500); + return; + } for (let i = 0; i < output.length; i++) { let output_json = null; try { @@ -660,13 +722,19 @@ app.post('/api/tomp3', function(req, res) { } var file_name = output_json['_filename'].replace(/^.*[\\\/]/, ''); var file_path = output_json['_filename'].substring(audioFolderPath.length, output_json['_filename'].length); - var alternate_file_path = file_path.substring(0, file_path.length-4); + // remove extension from file path + var alternate_file_path = file_path.replace(/\.[^/.]+$/, "") var alternate_file_name = file_name.substring(0, file_name.length-4); if (alternate_file_path) file_names.push(alternate_file_path); } let is_playlist = file_names.length > 1; - // if (!is_playlist) audiopath = file_names[0]; + + if (merged_string !== null) { + let current_merged_archive = fs.readFileSync(audioFolderPath + 'merged.txt', 'utf8'); + let diff = current_merged_archive.replace(merged_string, ''); + fs.appendFileSync(audioFolderPath + 'archive.txt', diff); + } var audiopathEncoded = encodeURIComponent(file_names[0]); res.send({ @@ -677,7 +745,7 @@ app.post('/api/tomp3', function(req, res) { }); }); -app.post('/api/tomp4', function(req, res) { +app.post('/api/tomp4', async function(req, res) { var url = req.body.url; var date = Date.now(); var path = videoFolderPath; @@ -691,6 +759,8 @@ app.post('/api/tomp4', function(req, res) { var youtubeUsername = req.body.youtubeUsername; var youtubePassword = req.body.youtubePassword; + let merged_string = null; + let downloadConfig = null; let qualityPath = 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4'; @@ -717,10 +787,34 @@ app.post('/api/tomp4', function(req, res) { downloadConfig.splice(0, 0, '--external-downloader', 'aria2c'); } + if (useYoutubeDLArchive) { + let archive_path = videoFolderPath + 'archive.txt'; + // create archive file if it doesn't exist + if (!fs.existsSync(archive_path)) { + fs.closeSync(fs.openSync(archive_path, 'w')); + } + + let blacklist_path = videoFolderPath + 'blacklist.txt'; + // create blacklist file if it doesn't exist + if (!fs.existsSync(blacklist_path)) { + fs.closeSync(fs.openSync(blacklist_path, 'w')); + } + + let merged_path = videoFolderPath + 'merged.txt'; + // merges blacklist and regular archive + let inputPathList = [archive_path, blacklist_path]; + let status = await mergeFiles(inputPathList, merged_path); + + merged_string = fs.readFileSync(merged_path, "utf8"); + + downloadConfig.push('--download-archive', merged_path); + } + if (globalArgs && globalArgs !== '') { // adds global args downloadConfig = downloadConfig.concat(globalArgs.split(' ')); } + } youtubedl.exec(url, downloadConfig, {}, function(err, output) { @@ -735,7 +829,11 @@ app.post('/api/tomp4', function(req, res) { res.sendStatus(500); throw err; } else if (output) { - var file_names = []; + if (output.length === 0 || output[0].length === 0) { + res.sendStatus(500); + return; + } + var file_names = []; for (let i = 0; i < output.length; i++) { let output_json = null; try { @@ -759,12 +857,19 @@ app.post('/api/tomp4', function(req, res) { } var alternate_file_name = file_name.substring(0, file_name.length-4); var file_path = output_json['_filename'].substring(audioFolderPath.length, output_json['_filename'].length); - var alternate_file_path = file_path.substring(0, file_path.length-4); + // remove extension from file path + var alternate_file_path = file_path.replace(/\.[^/.]+$/, "") if (alternate_file_name) file_names.push(alternate_file_path); } let is_playlist = file_names.length > 1; if (!is_playlist) audiopath = file_names[0]; + + if (merged_string !== null) { + let current_merged_archive = fs.readFileSync(videoFolderPath + 'merged.txt', 'utf8'); + let diff = current_merged_archive.replace(merged_string, ''); + fs.appendFileSync(videoFolderPath + 'archive.txt', diff); + } var videopathEncoded = encodeURIComponent(file_names[0]); res.send({ @@ -1091,11 +1196,12 @@ app.post('/api/deletePlaylist', async (req, res) => { // deletes mp3 file app.post('/api/deleteMp3', async (req, res) => { var name = req.body.name; + var blacklistMode = req.body.blacklistMode; var fullpath = audioFolderPath + name + ".mp3"; var wasDeleted = false; if (fs.existsSync(fullpath)) { - deleteAudioFile(name); + deleteAudioFile(name, blacklistMode); wasDeleted = true; res.send(wasDeleted); res.end("yes"); @@ -1111,11 +1217,12 @@ app.post('/api/deleteMp3', async (req, res) => { // deletes mp4 file app.post('/api/deleteMp4', async (req, res) => { var name = req.body.name; + var blacklistMode = req.body.blacklistMode; var fullpath = videoFolderPath + name + ".mp4"; var wasDeleted = false; if (fs.existsSync(fullpath)) { - wasDeleted = await deleteVideoFile(name); + wasDeleted = await deleteVideoFile(name, null, blacklistMode); // wasDeleted = true; res.send(wasDeleted); res.end("yes"); diff --git a/backend/config/default.json b/backend/config/default.json index 0eb440f..6cefae0 100644 --- a/backend/config/default.json +++ b/backend/config/default.json @@ -12,6 +12,7 @@ "Downloader": { "path-audio": "audio/", "path-video": "video/", + "use_youtubedl_archive": false, "custom_args": "" }, "Extra": { diff --git a/backend/config/encrypted.json b/backend/config/encrypted.json index a0fe6c2..df9c76c 100644 --- a/backend/config/encrypted.json +++ b/backend/config/encrypted.json @@ -12,6 +12,7 @@ "Downloader": { "path-audio": "audio/", "path-video": "video/", + "use_youtubedl_archive": false, "custom_args": "" }, "Extra": { diff --git a/backend/consts.js b/backend/consts.js index 10c82c7..73e4087 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -34,6 +34,10 @@ let CONFIG_ITEMS = { 'key': 'ytdl_video_folder_path', 'path': 'YoutubeDLMaterial.Downloader.path-video' }, + 'ytdl_use_youtubedl_archive': { + 'key': 'ytdl_use_youtubedl_archive', + 'path': 'YoutubeDLMaterial.Downloader.use_youtubedl_archive' + }, 'ytdl_custom_args': { 'key': 'ytdl_custom_args', 'path': 'YoutubeDLMaterial.Downloader.custom_args' diff --git a/backend/package.json b/backend/package.json index 6da5631..b67c591 100644 --- a/backend/package.json +++ b/backend/package.json @@ -25,6 +25,7 @@ "exe": "^1.0.2", "express": "^4.17.1", "lowdb": "^1.0.0", + "merge-files": "^0.1.2", "shortid": "^2.2.15", "uuidv4": "^6.0.6", "youtube-dl": "^3.0.2" diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 75a4d11..8fa9ce6 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -279,30 +279,30 @@ const deleteFolderRecursive = function(folder_to_delete) { }; function removeIDFromArchive(archive_path, id) { - fs.readFile(archive_path, {encoding: 'utf-8'}, function(err, data) { - if (err) throw error; - - let dataArray = data.split('\n'); // convert file data in an array - const searchKeyword = id; // we are looking for a line, contains, key word id in the file - let lastIndex = -1; // let say, we have not found the keyword - - for (let index=0; index { - if (err) throw err; - // console.log ('Successfully updated the file data'); - }); - - }); + } + + const line = dataArray.splice(lastIndex, 1); // remove the keyword id from the data Array + + // UPDATE FILE WITH NEW DATA + const updatedData = dataArray.join('\n'); + fs.writeFileSync(archive_path, updatedData); + if (line) return line; + if (err) throw err; } module.exports = { @@ -311,5 +311,6 @@ module.exports = { subscribe : subscribe, unsubscribe : unsubscribe, deleteSubscriptionFile : deleteSubscriptionFile, - getVideosForSub : getVideosForSub + getVideosForSub : getVideosForSub, + removeIDFromArchive : removeIDFromArchive } diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 8be4234..e6b6152 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -1,7 +1,6 @@ import { BrowserModule } from '@angular/platform-browser'; import { NgModule, LOCALE_ID } from '@angular/core'; import { registerLocaleData } from '@angular/common'; -import { LocaleService } from '@soluling/angular'; import { MatButtonModule } from '@angular/material/button'; import { MatButtonToggleModule } from '@angular/material/button-toggle'; import { MatCardModule } from '@angular/material/card'; @@ -111,9 +110,7 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible AppRoutingModule, ], providers: [ - PostsService, - LocaleService, - { provide: LOCALE_ID, deps: [LocaleService], useFactory: (service: LocaleService) => service.localeId }, + PostsService ], bootstrap: [AppComponent] }) diff --git a/src/app/file-card/file-card.component.html b/src/app/file-card/file-card.component.html index 39184ce..1c796c5 100644 --- a/src/app/file-card/file-card.component.html +++ b/src/app/file-card/file-card.component.html @@ -15,5 +15,10 @@ - + + + + + + diff --git a/src/app/file-card/file-card.component.ts b/src/app/file-card/file-card.component.ts index 7e39fc7..770976e 100644 --- a/src/app/file-card/file-card.component.ts +++ b/src/app/file-card/file-card.component.ts @@ -21,6 +21,7 @@ export class FileCardComponent implements OnInit { @Output() removeFile: EventEmitter = new EventEmitter(); @Input() isPlaylist = false; @Input() count = null; + @Input() use_youtubedl_archive = false; type; image_loaded = false; image_errored = false; @@ -40,9 +41,9 @@ export class FileCardComponent implements OnInit { this.type = this.isAudio ? 'audio' : 'video'; } - deleteFile() { + deleteFile(blacklistMode = false) { if (!this.isPlaylist) { - this.postsService.deleteFile(this.name, this.isAudio).subscribe(result => { + this.postsService.deleteFile(this.name, this.isAudio, blacklistMode).subscribe(result => { if (result === true) { this.openSnackBar('Delete success!', 'OK.'); this.removeFile.emit(this.name); diff --git a/src/app/main/main.component.html b/src/app/main/main.component.html index 110b1d0..dc11d79 100644 --- a/src/app/main/main.component.html +++ b/src/app/main/main.component.html @@ -204,7 +204,7 @@ + [length]="file.duration" [isAudio]="true" [use_youtubedl_archive]="use_youtubedl_archive"> @@ -215,7 +215,7 @@ + [length]="null" [isAudio]="true" [isPlaylist]="true" [count]="playlist.fileNames.length" [use_youtubedl_archive]="use_youtubedl_archive"> @@ -245,7 +245,7 @@ + [length]="file.duration" [isAudio]="false" [use_youtubedl_archive]="use_youtubedl_archive"> @@ -257,7 +257,7 @@ + [length]="null" [isAudio]="false" [isPlaylist]="true" [count]="playlist.fileNames.length" [use_youtubedl_archive]="use_youtubedl_archive"> diff --git a/src/app/main/main.component.ts b/src/app/main/main.component.ts index ba36282..9c356ce 100644 --- a/src/app/main/main.component.ts +++ b/src/app/main/main.component.ts @@ -72,6 +72,7 @@ export class MainComponent implements OnInit { allowMultiDownloadMode = false; audioFolderPath; videoFolderPath; + use_youtubedl_archive = false; globalCustomArgs = null; allowAdvancedDownload = false; useDefaultDownloadingAgent = true; @@ -241,6 +242,7 @@ export class MainComponent implements OnInit { this.allowMultiDownloadMode = result['YoutubeDLMaterial']['Extra']['allow_multi_download_mode']; this.audioFolderPath = result['YoutubeDLMaterial']['Downloader']['path-audio']; this.videoFolderPath = result['YoutubeDLMaterial']['Downloader']['path-video']; + this.use_youtubedl_archive = result['YoutubeDLMaterial']['Downloader']['use_youtubedl_archive']; this.globalCustomArgs = result['YoutubeDLMaterial']['Downloader']['custom_args']; this.youtubeSearchEnabled = result['YoutubeDLMaterial']['API'] && result['YoutubeDLMaterial']['API']['use_youtube_API'] && result['YoutubeDLMaterial']['API']['youtube_API_key']; @@ -594,6 +596,8 @@ export class MainComponent implements OnInit { } }, error => { // can't access server this.downloadingfile = false; + this.current_download = null; + new_download['downloading'] = false; this.openSnackBar('Download failed!', 'OK.'); }); } else { @@ -626,6 +630,8 @@ export class MainComponent implements OnInit { } }, error => { // can't access server this.downloadingfile = false; + this.current_download = null; + new_download['downloading'] = false; this.openSnackBar('Download failed!', 'OK.'); }); } @@ -879,6 +885,10 @@ export class MainComponent implements OnInit { full_string_array.push(...additional_params); } + if (this.use_youtubedl_archive) { + full_string_array.push('--download-archive', 'archive.txt'); + } + if (globalArgsExists) { full_string_array = full_string_array.concat(this.globalCustomArgs.split(' ')); } diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index f618f39..6ea4133 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -98,11 +98,11 @@ export class PostsService { return this.http.post(this.path + 'setConfig', {new_config_file: config}); } - deleteFile(name: string, isAudio: boolean) { + deleteFile(name: string, isAudio: boolean, blacklistMode = false) { if (isAudio) { - return this.http.post(this.path + 'deleteMp3', {name: name}); + return this.http.post(this.path + 'deleteMp3', {name: name, blacklistMode: blacklistMode}); } else { - return this.http.post(this.path + 'deleteMp4', {name: name}); + return this.http.post(this.path + 'deleteMp4', {name: name, blacklistMode: blacklistMode}); } } diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index 8d03462..1b0dd3b 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -93,6 +93,11 @@ Global custom args for downloads on the home page. + +
+ Use youtube-dl archive +

Note: This setting only applies to downloads on the Home page. If you would like to use youtube-dl archive functionality in subscriptions, head down to the Subscriptions section.

+
diff --git a/src/app/subscription/subscription-file-card/subscription-file-card.component.html b/src/app/subscription/subscription-file-card/subscription-file-card.component.html index 4537dd6..a2972a9 100644 --- a/src/app/subscription/subscription-file-card/subscription-file-card.component.html +++ b/src/app/subscription/subscription-file-card/subscription-file-card.component.html @@ -3,7 +3,7 @@ Length: {{formattedDuration}} - + diff --git a/src/assets/default.json b/src/assets/default.json index 94aad55..6a7e5b8 100644 --- a/src/assets/default.json +++ b/src/assets/default.json @@ -1,44 +1,45 @@ { - "YoutubeDLMaterial": { - "Host": { - "url": "http://localhost", - "port": "17442" - }, - "Encryption": { - "use-encryption": false, - "cert-file-path": "/etc/letsencrypt/live/example.com/fullchain.pem", - "key-file-path": "/etc/letsencrypt/live/example.com/privkey.pem" - }, - "Downloader": { - "path-audio": "audio/", - "path-video": "video/", - "custom_args": "" - }, - "Extra": { - "title_top": "Youtube Downloader", - "file_manager_enabled": true, - "allow_quality_select": true, - "download_only_mode": false, - "allow_multi_download_mode": true - }, - "API": { - "use_youtube_API": false, - "youtube_API_key": "" - }, - "Themes": { - "default_theme": "default", - "allow_theme_change": true - }, - "Subscriptions": { - "allow_subscriptions": true, - "subscriptions_base_path": "subscriptions/", - "subscriptions_check_interval": "300", - "subscriptions_use_youtubedl_archive": true - }, - "Advanced": { - "use_default_downloading_agent": true, - "custom_downloading_agent": "", - "allow_advanced_download": true - } + "YoutubeDLMaterial": { + "Host": { + "url": "http://localhost", + "port": "17442" + }, + "Encryption": { + "use-encryption": false, + "cert-file-path": "/etc/letsencrypt/live/example.com/fullchain.pem", + "key-file-path": "/etc/letsencrypt/live/example.com/privkey.pem" + }, + "Downloader": { + "path-audio": "audio/", + "path-video": "video/", + "use_youtubedl_archive": false, + "custom_args": "" + }, + "Extra": { + "title_top": "Youtube Downloader", + "file_manager_enabled": true, + "allow_quality_select": true, + "download_only_mode": false, + "allow_multi_download_mode": true + }, + "API": { + "use_youtube_API": false, + "youtube_API_key": "" + }, + "Themes": { + "default_theme": "default", + "allow_theme_change": true + }, + "Subscriptions": { + "allow_subscriptions": true, + "subscriptions_base_path": "subscriptions/", + "subscriptions_check_interval": "300", + "subscriptions_use_youtubedl_archive": true + }, + "Advanced": { + "use_default_downloading_agent": true, + "custom_downloading_agent": "", + "allow_advanced_download": true } } +} \ No newline at end of file