Added the ability to download videos at higher resolutions than the highest mp4 (fixes #76)

Deprecates normal downloading method. The "safe" method is now always used, and download progress is now estimated using the predicted end file size

Thumbnails are now auto downloaded along with the other metadata

backend/app.js

@@ -7,7 +7,7 @@ var path = require('path');
 var youtubedl = require('youtube-dl');
 var ffmpeg = require('fluent-ffmpeg');
 var compression = require('compression');
-var https = require('https');
+var glob = require("glob")
 var multer  = require('multer');
 var express = require("express");
 var bodyParser = require("body-parser");
@@ -1146,12 +1146,29 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
             type: type,
             percent_complete: 0,
             is_playlist: url.includes('playlist'),
-            timestamp_start: Date.now()
+            timestamp_start: Date.now(),
+            filesize: null
         };
         const download = downloads[session][download_uid];
         updateDownloads();
 
+        // get video info prior to download
+        const info = await getVideoInfoByURL(url, downloadConfig, download);
+        if (!info) {
+            resolve(false);
+            return;
+        } else {
+            // store info in download for future use
+            download['_filename'] = info['_filename']; // .substring(fileFolderPath.length, info['_filename'].length-4);
+            download['filesize'] = utils.getExpectedFileSize(info);
+        }
+
+        const download_checker = setInterval(() => checkDownloadPercent(download), 1000);
+
+        // download file
         youtubedl.exec(url, downloadConfig, {}, function(err, output) {
+            clearInterval(download_checker); // stops the download checker from running as the download finished (or errored)
+
             download['downloading'] = false;
             download['timestamp_end'] = Date.now();
             var file_uid = null;
@@ -1164,7 +1181,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
                 download['error'] = err.stderr;
                 updateDownloads();
                 resolve(false);
-                throw err;
+                return;
             } else if (output) {
                 if (output.length === 0 || output[0].length === 0) {
                     download['error'] = 'No output. Check if video already exists in your archive.';
@@ -1407,7 +1424,7 @@ async function generateArgs(url, type, options) {
         var youtubePassword = options.youtubePassword;
 
         let downloadConfig = null;
-        let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'best[ext=mp4]'];
+        let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
         const is_youtube = url.includes('youtu');
         if (!is_audio && !is_youtube) {
             // tiktok videos fail when using the default format
@@ -1485,6 +1502,10 @@ async function generateArgs(url, type, options) {
                 downloadConfig.push('--download-archive', merged_path);
             }
 
+            if (config_api.getConfigItem('ytdl_include_thumbnail')) {
+                downloadConfig.push('--write-thumbnail');
+            }
+
             if (globalArgs && globalArgs !== '') {
                 // adds global args
                 if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
@@ -1497,11 +1518,36 @@ async function generateArgs(url, type, options) {
 
         }
         logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
-        // downloadConfig.map((arg) => `"${arg}"`);
         resolve(downloadConfig);
     });
 }
 
+async function getVideoInfoByURL(url, args = [], download = null) {
+    return new Promise(resolve => {
+        // remove bad args
+        const new_args = [...args];
+        
+        const archiveArgIndex = new_args.indexOf('--download-archive');
+        if (archiveArgIndex !== -1) {
+            new_args.splice(archiveArgIndex, 2);
+        }
+
+        // actually get info
+        youtubedl.getInfo(url, new_args, (err, output) => {
+            if (output) {
+                resolve(output);
+            } else {
+                logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`);
+                if (download) {
+                    download['error'] = `Failed pre-check for video info: ${err}`;
+                    updateDownloads();
+                }
+                resolve(null);
+            }
+        });
+    });
+}
+
 // currently only works for single urls
 async function getUrlInfos(urls) {
     let startDate = Date.now();
@@ -1559,47 +1605,26 @@ function updateDownloads() {
     db.assign({downloads: downloads}).write();
 }
 
-/*
-function checkDownloads() {
-    for (let [session_id, session_downloads] of Object.entries(downloads)) {
-        for (let [download_uid, download_obj] of Object.entries(session_downloads)) {
-            if (download_obj && !download_obj['complete'] && !download_obj['error']
-                             && download_obj.timestamp_start > timestamp_server_start) {
-                // download is still running (presumably)
-                download_obj.percent_complete = getDownloadPercent(download_obj);
-            }
-        }
-    }
-}
-*/
+function checkDownloadPercent(download) {
+    const file_id = download['file_id'];
+    const filename = path.format(path.parse(download['_filename'].substring(0, download['_filename'].length-4)));
+    const resulting_file_size = download['filesize'];
 
-function getDownloadPercent(download_obj) {
-    if (!download_obj.final_size) {
-        if (fs.existsSync(download_obj.expected_json_path)) {
-            const file_json = JSON.parse(fs.readFileSync(download_obj.expected_json_path, 'utf8'));
-            let calculated_filesize = null;
-            if (file_json['format_id']) {
-                calculated_filesize = 0;
-                const formats_used = file_json['format_id'].split('+');
-                for (let i = 0; i < file_json['formats'].length; i++) {
-                    if (formats_used.includes(file_json['formats'][i]['format_id'])) {
-                        calculated_filesize += file_json['formats'][i]['filesize'];
-                    }
-                }
-            }
-            download_obj.final_size = calculated_filesize;
-        } else {
-            console.log('could not find json file');
-        }
+    glob(`${filename}*`, (err, files) => {
+        let sum_size = 0;
+        files.forEach(file => {
+            try {
+                const file_stats = fs.statSync(file);
+                if (file_stats && file_stats.size) {
+                    sum_size += file_stats.size;
                 }
-    if (fs.existsSync(download_obj.expected_path)) {
-        const stats = fs.statSync(download_obj.expected_path);
-        const size = stats.size;
-        return (size / download_obj.final_size)*100;
-    } else {
-        console.log('could not find file');
-        return 0;
+            } catch (e) {
+
             }
+        });
+        download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2);
+        updateDownloads();
+    });
 }
 
 // youtube-dl functions
@@ -1821,7 +1846,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
     const is_playlist = url.includes('playlist');
 
     let result_obj = null;
-    if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
+    if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
         result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID);
     else
         result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID);
@@ -1833,6 +1858,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
 });
 
 app.post('/api/tomp4', optionalJwt, async function(req, res) {
+    req.setTimeout(0); // remove timeout in case of long videos
     var url = req.body.url;
     var options = {
         customArgs: req.body.customArgs,
@@ -1850,7 +1876,7 @@ app.post('/api/tomp4', optionalJwt, async function(req, res) {
     const is_playlist = url.includes('playlist');
 
     let result_obj = null;
-    if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
+    if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
         result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID);
     else
         result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID);
@@ -1878,6 +1904,12 @@ app.get('/api/getMp3s', optionalJwt, function(req, res) {
         playlists = auth_api.getUserPlaylists(req.user.uid, 'audio');
     }
 
+    // add thumbnails if present
+    mp3s.forEach(mp3 => {
+        if (mp3['thumbnailPath'] && fs.existsSync(mp3['thumbnailPath']))
+            mp3['thumbnailBlob'] = fs.readFileSync(mp3['thumbnailPath']);
+    });
+
     res.send({
         mp3s: mp3s,
         playlists: playlists
@@ -1897,6 +1929,12 @@ app.get('/api/getMp4s', optionalJwt, function(req, res) {
         playlists = auth_api.getUserPlaylists(req.user.uid, 'video');
     }
 
+    // add thumbnails if present
+    mp4s.forEach(mp4 => {
+        if (mp4['thumbnailPath'] && fs.existsSync(mp4['thumbnailPath']))
+            mp4['thumbnailBlob'] = fs.readFileSync(mp4['thumbnailPath']);
+    });
+
     res.send({
         mp4s: mp4s,
         playlists: playlists
@@ -1981,6 +2019,12 @@ app.post('/api/getAllFiles', optionalJwt, function (req, res) {
         files = files.concat(sub.videos);
     }
 
+    // add thumbnails if present
+    files.forEach(file => {
+        if (file['thumbnailPath'] && fs.existsSync(file['thumbnailPath']))
+            file['thumbnailBlob'] = fs.readFileSync(file['thumbnailPath']);
+    });
+
     res.send({
         files: files,
         playlists: playlists

backend/appdata/default.json

@@ -9,7 +9,9 @@
       "path-video": "video/",
       "use_youtubedl_archive": false,
       "custom_args": "",
-      "safe_download_override": false
+      "safe_download_override": false,
+      "include_thumbnail": true,
+      "include_metadata": true
     },
     "Extra": {
       "title_top": "YoutubeDL-Material",

backend/config.js

@@ -186,7 +186,9 @@ DEFAULT_CONFIG = {
         "path-video": "video/",
         "use_youtubedl_archive": false,
         "custom_args": "",
-        "safe_download_override": false
+        "safe_download_override": false,
+        "include_thumbnail": true,
+        "include_metadata": true
       },
       "Extra": {
         "title_top": "YoutubeDL-Material",

backend/consts.js

@@ -30,6 +30,14 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_safe_download_override',
         'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
     },
+    'ytdl_include_thumbnail': {
+        'key': 'ytdl_include_thumbnail',
+        'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
+    },
+    'ytdl_include_metadata': {
+        'key': 'ytdl_include_metadata',
+        'path': 'YoutubeDLMaterial.Downloader.include_metadata'
+    },
 
     // Extra
     'ytdl_title_top': {

backend/db.js

@@ -29,8 +29,8 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) {
     // add additional info
     file_object['uid'] = uuid();
     file_object['registered'] = Date.now();
-    path_object = path.parse(file_object['path']);
-    file_object['path'] = path.format(path_object);
+    file_object['path'] = path.format(path.parse(file_object['path']));
+    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, multiUserMode && multiUserMode.file_path);
 
     if (!sub) {
         if (multiUserMode) {

backend/package.json

@@ -37,6 +37,7 @@
     "express": "^4.17.1",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
+    "glob": "^7.1.6",
     "jsonwebtoken": "^8.5.1",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",

backend/subscriptions.js

@@ -345,6 +345,10 @@ async function getVideosForSub(sub, user_uid = null) {
             }
         }
 
+        if (config_api.getConfigItem('ytdl_include_thumbnail')) {
+            downloadConfig.push('--write-thumbnail');
+        }
+
         // get videos
         logger.verbose('Subscription: getting videos for subscription ' + sub.name);
         youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {

backend/utils.js

@@ -88,6 +88,41 @@ function getJSONByType(type, name, customPath, openReadPerms = false) {
     return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
 }
 
+function getDownloadedThumbnail(name, type, customPath = null) {
+    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
+
+    let jpgPath = path.join(customPath, name + '.jpg');
+    let webpPath = path.join(customPath, name + '.webp');
+    let pngPath = path.join(customPath, name + '.png');
+
+    if (fs.existsSync(jpgPath))
+        return jpgPath;
+    else if (fs.existsSync(webpPath))
+        return webpPath;
+    else if (fs.existsSync(pngPath))
+        return pngPath;
+    else
+        return null;
+}
+
+function getExpectedFileSize(info_json) {
+    if (info_json['filesize']) {
+        return info_json['filesize'];
+    }
+
+    const formats = info_json['format_id'].split('+');
+    let expected_filesize = 0;
+    formats.forEach(format_id => {
+        info_json.formats.forEach(available_format => {
+            if (available_format.format_id === format_id && available_format.filesize) {
+                expected_filesize += available_format.filesize;
+            }
+        });
+    });
+    
+    return expected_filesize;
+}
+
 function fixVideoMetadataPerms(name, type, customPath = null) {
     if (is_windows) return;
     if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
@@ -153,6 +188,8 @@ module.exports = {
     getJSONMp3: getJSONMp3,
     getJSONMp4: getJSONMp4,
     getTrueFileName: getTrueFileName,
+    getDownloadedThumbnail: getDownloadedThumbnail,
+    getExpectedFileSize: getExpectedFileSize,
     fixVideoMetadataPerms: fixVideoMetadataPerms,
     getDownloadedFilesByType: getDownloadedFilesByType,
     recFindByExt: recFindByExt,

src/app/components/unified-file-card/unified-file-card.component.html

@@ -29,7 +29,7 @@
         <div style="padding:5px">
             <div *ngIf="!loading && file_obj.thumbnailURL" class="img-div">
                 <div style="position: relative">
-                    <img [ngClass]="{'image-small': card_size === 'small', 'image': card_size === 'medium', 'image-large': card_size === 'large'}" [src]="file_obj.thumbnailURL" alt="Thumbnail">
+                    <img [ngClass]="{'image-small': card_size === 'small', 'image': card_size === 'medium', 'image-large': card_size === 'large'}" [src]="file_obj.thumbnailBlob ? thumbnailBlobURL : file_obj.thumbnailURL" alt="Thumbnail">
                     <div class="duration-time">
                         {{file_length}}
                     </div>

src/app/components/unified-file-card/unified-file-card.component.ts

@@ -1,6 +1,7 @@
 import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core';
 import { MatDialog } from '@angular/material/dialog';
 import { VideoInfoDialogComponent } from 'app/dialogs/video-info-dialog/video-info-dialog.component';
+import { DomSanitizer } from '@angular/platform-browser';
 
 @Component({
   selector: 'app-unified-file-card',
@@ -16,6 +17,10 @@ export class UnifiedFileCardComponent implements OnInit {
   type = null;
   elevated = false;
 
+  // optional vars
+  thumbnailBlobURL = null;
+
+  // input/output
   @Input() loading = true;
   @Input() theme = null;
   @Input() file_obj = null;
@@ -35,12 +40,19 @@ export class UnifiedFileCardComponent implements OnInit {
     big: 250x200
   */
 
-  constructor(private dialog: MatDialog) { }
+  constructor(private dialog: MatDialog, private sanitizer: DomSanitizer) { }
 
   ngOnInit(): void {
     if (!this.loading) {
       this.file_length = fancyTimeFormat(this.file_obj.duration);
     }
+
+    if (this.file_obj && this.file_obj.thumbnailBlob) {
+      const mime = getMimeByFilename(this.file_obj.thumbnailPath);
+      const blob = new Blob([new Uint8Array(this.file_obj.thumbnailBlob.data)], {type: mime});
+      const bloburl = URL.createObjectURL(blob);
+      this.thumbnailBlobURL = this.sanitizer.bypassSecurityTrustUrl(bloburl);
+    }
   }
 
   emitDeleteFile(blacklistMode = false) {
@@ -97,3 +109,16 @@ function fancyTimeFormat(time) {
   ret += '' + secs;
   return ret;
 }
+
+function getMimeByFilename(name) {
+  switch (name.substring(name.length-4, name.length)) {
+    case '.jpg':
+      return 'image/jpeg';
+    case '.png':
+      return 'image/png';
+    case 'webp':
+      return 'image/webp';
+    default:
+      return null;
+  }
+}