From 8a7409478af34582adcc86454505fa47eb9ce9ca Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sat, 29 Aug 2020 23:05:37 -0400 Subject: [PATCH] Added the ability to download videos at higher resolutions than the highest mp4 (fixes #76) Deprecates normal downloading method. The "safe" method is now always used, and download progress is now estimated using the predicted end file size Thumbnails are now auto downloaded along with the other metadata --- backend/app.js | 136 ++++++++++++------ backend/appdata/default.json | 4 +- backend/config.js | 4 +- backend/consts.js | 8 ++ backend/db.js | 4 +- backend/package.json | 1 + backend/subscriptions.js | 4 + backend/utils.js | 37 +++++ .../unified-file-card.component.html | 2 +- .../unified-file-card.component.ts | 27 +++- 10 files changed, 175 insertions(+), 52 deletions(-) diff --git a/backend/app.js b/backend/app.js index 9058e51..50c1248 100644 --- a/backend/app.js +++ b/backend/app.js @@ -7,7 +7,7 @@ var path = require('path'); var youtubedl = require('youtube-dl'); var ffmpeg = require('fluent-ffmpeg'); var compression = require('compression'); -var https = require('https'); +var glob = require("glob") var multer = require('multer'); var express = require("express"); var bodyParser = require("body-parser"); @@ -1146,12 +1146,29 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { type: type, percent_complete: 0, is_playlist: url.includes('playlist'), - timestamp_start: Date.now() + timestamp_start: Date.now(), + filesize: null }; const download = downloads[session][download_uid]; updateDownloads(); + // get video info prior to download + const info = await getVideoInfoByURL(url, downloadConfig, download); + if (!info) { + resolve(false); + return; + } else { + // store info in download for future use + download['_filename'] = info['_filename']; // .substring(fileFolderPath.length, info['_filename'].length-4); + download['filesize'] = utils.getExpectedFileSize(info); + } + + const download_checker = setInterval(() => checkDownloadPercent(download), 1000); + + // download file youtubedl.exec(url, downloadConfig, {}, function(err, output) { + clearInterval(download_checker); // stops the download checker from running as the download finished (or errored) + download['downloading'] = false; download['timestamp_end'] = Date.now(); var file_uid = null; @@ -1164,7 +1181,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { download['error'] = err.stderr; updateDownloads(); resolve(false); - throw err; + return; } else if (output) { if (output.length === 0 || output[0].length === 0) { download['error'] = 'No output. Check if video already exists in your archive.'; @@ -1407,7 +1424,7 @@ async function generateArgs(url, type, options) { var youtubePassword = options.youtubePassword; let downloadConfig = null; - let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'best[ext=mp4]']; + let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4']; const is_youtube = url.includes('youtu'); if (!is_audio && !is_youtube) { // tiktok videos fail when using the default format @@ -1485,6 +1502,10 @@ async function generateArgs(url, type, options) { downloadConfig.push('--download-archive', merged_path); } + if (config_api.getConfigItem('ytdl_include_thumbnail')) { + downloadConfig.push('--write-thumbnail'); + } + if (globalArgs && globalArgs !== '') { // adds global args if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) { @@ -1497,11 +1518,36 @@ async function generateArgs(url, type, options) { } logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`); - // downloadConfig.map((arg) => `"${arg}"`); resolve(downloadConfig); }); } +async function getVideoInfoByURL(url, args = [], download = null) { + return new Promise(resolve => { + // remove bad args + const new_args = [...args]; + + const archiveArgIndex = new_args.indexOf('--download-archive'); + if (archiveArgIndex !== -1) { + new_args.splice(archiveArgIndex, 2); + } + + // actually get info + youtubedl.getInfo(url, new_args, (err, output) => { + if (output) { + resolve(output); + } else { + logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`); + if (download) { + download['error'] = `Failed pre-check for video info: ${err}`; + updateDownloads(); + } + resolve(null); + } + }); + }); +} + // currently only works for single urls async function getUrlInfos(urls) { let startDate = Date.now(); @@ -1559,47 +1605,26 @@ function updateDownloads() { db.assign({downloads: downloads}).write(); } -/* -function checkDownloads() { - for (let [session_id, session_downloads] of Object.entries(downloads)) { - for (let [download_uid, download_obj] of Object.entries(session_downloads)) { - if (download_obj && !download_obj['complete'] && !download_obj['error'] - && download_obj.timestamp_start > timestamp_server_start) { - // download is still running (presumably) - download_obj.percent_complete = getDownloadPercent(download_obj); - } - } - } -} -*/ - -function getDownloadPercent(download_obj) { - if (!download_obj.final_size) { - if (fs.existsSync(download_obj.expected_json_path)) { - const file_json = JSON.parse(fs.readFileSync(download_obj.expected_json_path, 'utf8')); - let calculated_filesize = null; - if (file_json['format_id']) { - calculated_filesize = 0; - const formats_used = file_json['format_id'].split('+'); - for (let i = 0; i < file_json['formats'].length; i++) { - if (formats_used.includes(file_json['formats'][i]['format_id'])) { - calculated_filesize += file_json['formats'][i]['filesize']; - } +function checkDownloadPercent(download) { + const file_id = download['file_id']; + const filename = path.format(path.parse(download['_filename'].substring(0, download['_filename'].length-4))); + const resulting_file_size = download['filesize']; + + glob(`${filename}*`, (err, files) => { + let sum_size = 0; + files.forEach(file => { + try { + const file_stats = fs.statSync(file); + if (file_stats && file_stats.size) { + sum_size += file_stats.size; } + } catch (e) { + } - download_obj.final_size = calculated_filesize; - } else { - console.log('could not find json file'); - } - } - if (fs.existsSync(download_obj.expected_path)) { - const stats = fs.statSync(download_obj.expected_path); - const size = stats.size; - return (size / download_obj.final_size)*100; - } else { - console.log('could not find file'); - return 0; - } + }); + download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2); + updateDownloads(); + }); } // youtube-dl functions @@ -1821,7 +1846,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) { const is_playlist = url.includes('playlist'); let result_obj = null; - if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate) + if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate) result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID); else result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID); @@ -1833,6 +1858,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) { }); app.post('/api/tomp4', optionalJwt, async function(req, res) { + req.setTimeout(0); // remove timeout in case of long videos var url = req.body.url; var options = { customArgs: req.body.customArgs, @@ -1850,7 +1876,7 @@ app.post('/api/tomp4', optionalJwt, async function(req, res) { const is_playlist = url.includes('playlist'); let result_obj = null; - if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu')) + if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu')) result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID); else result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID); @@ -1878,6 +1904,12 @@ app.get('/api/getMp3s', optionalJwt, function(req, res) { playlists = auth_api.getUserPlaylists(req.user.uid, 'audio'); } + // add thumbnails if present + mp3s.forEach(mp3 => { + if (mp3['thumbnailPath'] && fs.existsSync(mp3['thumbnailPath'])) + mp3['thumbnailBlob'] = fs.readFileSync(mp3['thumbnailPath']); + }); + res.send({ mp3s: mp3s, playlists: playlists @@ -1897,6 +1929,12 @@ app.get('/api/getMp4s', optionalJwt, function(req, res) { playlists = auth_api.getUserPlaylists(req.user.uid, 'video'); } + // add thumbnails if present + mp4s.forEach(mp4 => { + if (mp4['thumbnailPath'] && fs.existsSync(mp4['thumbnailPath'])) + mp4['thumbnailBlob'] = fs.readFileSync(mp4['thumbnailPath']); + }); + res.send({ mp4s: mp4s, playlists: playlists @@ -1981,6 +2019,12 @@ app.post('/api/getAllFiles', optionalJwt, function (req, res) { files = files.concat(sub.videos); } + // add thumbnails if present + files.forEach(file => { + if (file['thumbnailPath'] && fs.existsSync(file['thumbnailPath'])) + file['thumbnailBlob'] = fs.readFileSync(file['thumbnailPath']); + }); + res.send({ files: files, playlists: playlists diff --git a/backend/appdata/default.json b/backend/appdata/default.json index 6646e71..745b038 100644 --- a/backend/appdata/default.json +++ b/backend/appdata/default.json @@ -9,7 +9,9 @@ "path-video": "video/", "use_youtubedl_archive": false, "custom_args": "", - "safe_download_override": false + "safe_download_override": false, + "include_thumbnail": true, + "include_metadata": true }, "Extra": { "title_top": "YoutubeDL-Material", diff --git a/backend/config.js b/backend/config.js index 6a1a3c1..8737785 100644 --- a/backend/config.js +++ b/backend/config.js @@ -186,7 +186,9 @@ DEFAULT_CONFIG = { "path-video": "video/", "use_youtubedl_archive": false, "custom_args": "", - "safe_download_override": false + "safe_download_override": false, + "include_thumbnail": true, + "include_metadata": true }, "Extra": { "title_top": "YoutubeDL-Material", diff --git a/backend/consts.js b/backend/consts.js index 950a1d0..11ee67b 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -30,6 +30,14 @@ let CONFIG_ITEMS = { 'key': 'ytdl_safe_download_override', 'path': 'YoutubeDLMaterial.Downloader.safe_download_override' }, + 'ytdl_include_thumbnail': { + 'key': 'ytdl_include_thumbnail', + 'path': 'YoutubeDLMaterial.Downloader.include_thumbnail' + }, + 'ytdl_include_metadata': { + 'key': 'ytdl_include_metadata', + 'path': 'YoutubeDLMaterial.Downloader.include_metadata' + }, // Extra 'ytdl_title_top': { diff --git a/backend/db.js b/backend/db.js index 2c71ca1..dd0d094 100644 --- a/backend/db.js +++ b/backend/db.js @@ -29,8 +29,8 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) { // add additional info file_object['uid'] = uuid(); file_object['registered'] = Date.now(); - path_object = path.parse(file_object['path']); - file_object['path'] = path.format(path_object); + file_object['path'] = path.format(path.parse(file_object['path'])); + file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, multiUserMode && multiUserMode.file_path); if (!sub) { if (multiUserMode) { diff --git a/backend/package.json b/backend/package.json index 6e70aa0..b153a5d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -37,6 +37,7 @@ "express": "^4.17.1", "fluent-ffmpeg": "^2.1.2", "fs-extra": "^9.0.0", + "glob": "^7.1.6", "jsonwebtoken": "^8.5.1", "lowdb": "^1.0.0", "md5": "^2.2.1", diff --git a/backend/subscriptions.js b/backend/subscriptions.js index d14169b..e65043e 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -345,6 +345,10 @@ async function getVideosForSub(sub, user_uid = null) { } } + if (config_api.getConfigItem('ytdl_include_thumbnail')) { + downloadConfig.push('--write-thumbnail'); + } + // get videos logger.verbose('Subscription: getting videos for subscription ' + sub.name); youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) { diff --git a/backend/utils.js b/backend/utils.js index 034ce3c..d2e2a5e 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -88,6 +88,41 @@ function getJSONByType(type, name, customPath, openReadPerms = false) { return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms) } +function getDownloadedThumbnail(name, type, customPath = null) { + if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path'); + + let jpgPath = path.join(customPath, name + '.jpg'); + let webpPath = path.join(customPath, name + '.webp'); + let pngPath = path.join(customPath, name + '.png'); + + if (fs.existsSync(jpgPath)) + return jpgPath; + else if (fs.existsSync(webpPath)) + return webpPath; + else if (fs.existsSync(pngPath)) + return pngPath; + else + return null; +} + +function getExpectedFileSize(info_json) { + if (info_json['filesize']) { + return info_json['filesize']; + } + + const formats = info_json['format_id'].split('+'); + let expected_filesize = 0; + formats.forEach(format_id => { + info_json.formats.forEach(available_format => { + if (available_format.format_id === format_id && available_format.filesize) { + expected_filesize += available_format.filesize; + } + }); + }); + + return expected_filesize; +} + function fixVideoMetadataPerms(name, type, customPath = null) { if (is_windows) return; if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') @@ -153,6 +188,8 @@ module.exports = { getJSONMp3: getJSONMp3, getJSONMp4: getJSONMp4, getTrueFileName: getTrueFileName, + getDownloadedThumbnail: getDownloadedThumbnail, + getExpectedFileSize: getExpectedFileSize, fixVideoMetadataPerms: fixVideoMetadataPerms, getDownloadedFilesByType: getDownloadedFilesByType, recFindByExt: recFindByExt, diff --git a/src/app/components/unified-file-card/unified-file-card.component.html b/src/app/components/unified-file-card/unified-file-card.component.html index 27a1b37..3624502 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.html +++ b/src/app/components/unified-file-card/unified-file-card.component.html @@ -29,7 +29,7 @@
- Thumbnail + Thumbnail
{{file_length}}
diff --git a/src/app/components/unified-file-card/unified-file-card.component.ts b/src/app/components/unified-file-card/unified-file-card.component.ts index aa4166d..54f56ba 100644 --- a/src/app/components/unified-file-card/unified-file-card.component.ts +++ b/src/app/components/unified-file-card/unified-file-card.component.ts @@ -1,6 +1,7 @@ import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core'; import { MatDialog } from '@angular/material/dialog'; import { VideoInfoDialogComponent } from 'app/dialogs/video-info-dialog/video-info-dialog.component'; +import { DomSanitizer } from '@angular/platform-browser'; @Component({ selector: 'app-unified-file-card', @@ -16,6 +17,10 @@ export class UnifiedFileCardComponent implements OnInit { type = null; elevated = false; + // optional vars + thumbnailBlobURL = null; + + // input/output @Input() loading = true; @Input() theme = null; @Input() file_obj = null; @@ -35,12 +40,19 @@ export class UnifiedFileCardComponent implements OnInit { big: 250x200 */ - constructor(private dialog: MatDialog) { } + constructor(private dialog: MatDialog, private sanitizer: DomSanitizer) { } ngOnInit(): void { if (!this.loading) { this.file_length = fancyTimeFormat(this.file_obj.duration); } + + if (this.file_obj && this.file_obj.thumbnailBlob) { + const mime = getMimeByFilename(this.file_obj.thumbnailPath); + const blob = new Blob([new Uint8Array(this.file_obj.thumbnailBlob.data)], {type: mime}); + const bloburl = URL.createObjectURL(blob); + this.thumbnailBlobURL = this.sanitizer.bypassSecurityTrustUrl(bloburl); + } } emitDeleteFile(blacklistMode = false) { @@ -97,3 +109,16 @@ function fancyTimeFormat(time) { ret += '' + secs; return ret; } + +function getMimeByFilename(name) { + switch (name.substring(name.length-4, name.length)) { + case '.jpg': + return 'image/jpeg'; + case '.png': + return 'image/png'; + case 'webp': + return 'image/webp'; + default: + return null; + } +} \ No newline at end of file