Merge pull request #206 from Tzahi12345/downloader-improvements

Downloader improvements - updated system and bug fixes

backend/app.js

@@ -7,7 +7,7 @@ var path = require('path');
 var youtubedl = require('youtube-dl');
 var ffmpeg = require('fluent-ffmpeg');
 var compression = require('compression');
-var https = require('https');
+var glob = require("glob")
 var multer  = require('multer');
 var express = require("express");
 var bodyParser = require("body-parser");
@@ -1146,12 +1146,29 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
             type: type,
             percent_complete: 0,
             is_playlist: url.includes('playlist'),
-            timestamp_start: Date.now()
+            timestamp_start: Date.now(),
+            filesize: null
         };
         const download = downloads[session][download_uid];
         updateDownloads();
 
+        // get video info prior to download
+        const info = await getVideoInfoByURL(url, downloadConfig, download);
+        if (!info) {
+            resolve(false);
+            return;
+        } else {
+            // store info in download for future use
+            download['_filename'] = info['_filename'];
+            download['filesize'] = utils.getExpectedFileSize(info);
+        }
+
+        const download_checker = setInterval(() => checkDownloadPercent(download), 1000);
+
+        // download file
         youtubedl.exec(url, downloadConfig, {}, function(err, output) {
+            clearInterval(download_checker); // stops the download checker from running as the download finished (or errored)
+
             download['downloading'] = false;
             download['timestamp_end'] = Date.now();
             var file_uid = null;
@@ -1164,7 +1181,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) {
                 download['error'] = err.stderr;
                 updateDownloads();
                 resolve(false);
-                throw err;
+                return;
             } else if (output) {
                 if (output.length === 0 || output[0].length === 0) {
                     download['error'] = 'No output. Check if video already exists in your archive.';
@@ -1407,7 +1424,7 @@ async function generateArgs(url, type, options) {
         var youtubePassword = options.youtubePassword;
 
         let downloadConfig = null;
-        let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'best[ext=mp4]'];
+        let qualityPath = (is_audio && !options.skip_audio_args) ? ['-f', 'bestaudio'] : ['-f', 'bestvideo+bestaudio', '--merge-output-format', 'mp4'];
         const is_youtube = url.includes('youtu');
         if (!is_audio && !is_youtube) {
             // tiktok videos fail when using the default format
@@ -1485,6 +1502,10 @@ async function generateArgs(url, type, options) {
                 downloadConfig.push('--download-archive', merged_path);
             }
 
+            if (config_api.getConfigItem('ytdl_include_thumbnail')) {
+                downloadConfig.push('--write-thumbnail');
+            }
+
             if (globalArgs && globalArgs !== '') {
                 // adds global args
                 if (downloadConfig.indexOf('-o') !== -1 && globalArgs.split(',,').indexOf('-o') !== -1) {
@@ -1497,11 +1518,36 @@ async function generateArgs(url, type, options) {
 
         }
         logger.verbose(`youtube-dl args being used: ${downloadConfig.join(',')}`);
-        // downloadConfig.map((arg) => `"${arg}"`);
         resolve(downloadConfig);
     });
 }
 
+async function getVideoInfoByURL(url, args = [], download = null) {
+    return new Promise(resolve => {
+        // remove bad args
+        const new_args = [...args];
+        
+        const archiveArgIndex = new_args.indexOf('--download-archive');
+        if (archiveArgIndex !== -1) {
+            new_args.splice(archiveArgIndex, 2);
+        }
+
+        // actually get info
+        youtubedl.getInfo(url, new_args, (err, output) => {
+            if (output) {
+                resolve(output);
+            } else {
+                logger.error(`Error while retrieving info on video with URL ${url} with the following message: ${err}`);
+                if (download) {
+                    download['error'] = `Failed pre-check for video info: ${err}`;
+                    updateDownloads();
+                }
+                resolve(null);
+            }
+        });
+    });
+}
+
 // currently only works for single urls
 async function getUrlInfos(urls) {
     let startDate = Date.now();
@@ -1559,47 +1605,33 @@ function updateDownloads() {
     db.assign({downloads: downloads}).write();
 }
 
-/*
-function checkDownloads() {
-    for (let [session_id, session_downloads] of Object.entries(downloads)) {
-        for (let [download_uid, download_obj] of Object.entries(session_downloads)) {
-            if (download_obj && !download_obj['complete'] && !download_obj['error']
-                             && download_obj.timestamp_start > timestamp_server_start) {
-                // download is still running (presumably)
-                download_obj.percent_complete = getDownloadPercent(download_obj);
-            }
-        }
-    }
-}
-*/
-
-function getDownloadPercent(download_obj) {
-    if (!download_obj.final_size) {
-        if (fs.existsSync(download_obj.expected_json_path)) {
-            const file_json = JSON.parse(fs.readFileSync(download_obj.expected_json_path, 'utf8'));
-            let calculated_filesize = null;
-            if (file_json['format_id']) {
-                calculated_filesize = 0;
-                const formats_used = file_json['format_id'].split('+');
-                for (let i = 0; i < file_json['formats'].length; i++) {
-                    if (formats_used.includes(file_json['formats'][i]['format_id'])) {
-                        calculated_filesize += file_json['formats'][i]['filesize'];
-                    }
+function checkDownloadPercent(download) {
+    /*
+    This is more of an art than a science, we're just selecting files that start with the file name,
+    thus capturing the parts being downloaded in files named like so: '<video title>.<format>.<ext>.part'.
+
+    Any file that starts with <video title> will be counted as part of the "bytes downloaded", which will
+    be divided by the "total expected bytes."
+    */
+    const file_id = download['file_id'];
+    const filename = path.format(path.parse(download['_filename'].substring(0, download['_filename'].length-4)));
+    const resulting_file_size = download['filesize'];
+
+    glob(`${filename}*`, (err, files) => {
+        let sum_size = 0;
+        files.forEach(file => {
+            try {
+                const file_stats = fs.statSync(file);
+                if (file_stats && file_stats.size) {
+                    sum_size += file_stats.size;
                 }
+            } catch (e) {
+
             }
-            download_obj.final_size = calculated_filesize;
-        } else {
-            console.log('could not find json file');
-        }
-    }
-    if (fs.existsSync(download_obj.expected_path)) {
-        const stats = fs.statSync(download_obj.expected_path);
-        const size = stats.size;
-        return (size / download_obj.final_size)*100;
-    } else {
-        console.log('could not find file');
-        return 0;
-    }
+        });
+        download['percent_complete'] = (sum_size/resulting_file_size * 100).toFixed(2);
+        updateDownloads();
+    });
 }
 
 // youtube-dl functions
@@ -1821,7 +1853,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
     const is_playlist = url.includes('playlist');
 
     let result_obj = null;
-    if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
+    if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.maxBitrate)
         result_obj = await downloadFileByURL_exec(url, 'audio', options, req.query.sessionID);
     else
         result_obj = await downloadFileByURL_normal(url, 'audio', options, req.query.sessionID);
@@ -1833,6 +1865,7 @@ app.post('/api/tomp3', optionalJwt, async function(req, res) {
 });
 
 app.post('/api/tomp4', optionalJwt, async function(req, res) {
+    req.setTimeout(0); // remove timeout in case of long videos
     var url = req.body.url;
     var options = {
         customArgs: req.body.customArgs,
@@ -1850,7 +1883,7 @@ app.post('/api/tomp4', optionalJwt, async function(req, res) {
     const is_playlist = url.includes('playlist');
 
     let result_obj = null;
-    if (safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
+    if (true || safeDownloadOverride || is_playlist || options.customQualityConfiguration || options.customArgs || options.selectedHeight || !url.includes('youtu'))
         result_obj = await downloadFileByURL_exec(url, 'video', options, req.query.sessionID);
     else
         result_obj = await downloadFileByURL_normal(url, 'video', options, req.query.sessionID);
@@ -1878,6 +1911,14 @@ app.get('/api/getMp3s', optionalJwt, function(req, res) {
         playlists = auth_api.getUserPlaylists(req.user.uid, 'audio');
     }
 
+    mp3s = JSON.parse(JSON.stringify(mp3s));
+
+    // add thumbnails if present
+    mp3s.forEach(mp3 => {
+        if (mp3['thumbnailPath'] && fs.existsSync(mp3['thumbnailPath']))
+            mp3['thumbnailBlob'] = fs.readFileSync(mp3['thumbnailPath']);
+    });
+
     res.send({
         mp3s: mp3s,
         playlists: playlists
@@ -1897,6 +1938,14 @@ app.get('/api/getMp4s', optionalJwt, function(req, res) {
         playlists = auth_api.getUserPlaylists(req.user.uid, 'video');
     }
 
+    mp4s = JSON.parse(JSON.stringify(mp4s));
+
+    // add thumbnails if present
+    mp4s.forEach(mp4 => {
+        if (mp4['thumbnailPath'] && fs.existsSync(mp4['thumbnailPath']))
+            mp4['thumbnailBlob'] = fs.readFileSync(mp4['thumbnailPath']);
+    });
+
     res.send({
         mp4s: mp4s,
         playlists: playlists
@@ -1981,6 +2030,14 @@ app.post('/api/getAllFiles', optionalJwt, function (req, res) {
         files = files.concat(sub.videos);
     }
 
+    files = JSON.parse(JSON.stringify(files));
+
+    // add thumbnails if present
+    files.forEach(file => {
+        if (file['thumbnailPath'] && fs.existsSync(file['thumbnailPath']))
+            file['thumbnailBlob'] = fs.readFileSync(file['thumbnailPath']);
+    });
+
     res.send({
         files: files,
         playlists: playlists

backend/appdata/default.json

@@ -9,7 +9,9 @@
       "path-video": "video/",
       "use_youtubedl_archive": false,
       "custom_args": "",
-      "safe_download_override": false
+      "safe_download_override": false,
+      "include_thumbnail": true,
+      "include_metadata": true
     },
     "Extra": {
       "title_top": "YoutubeDL-Material",

backend/config.js

@@ -186,7 +186,9 @@ DEFAULT_CONFIG = {
         "path-video": "video/",
         "use_youtubedl_archive": false,
         "custom_args": "",
-        "safe_download_override": false
+        "safe_download_override": false,
+        "include_thumbnail": true,
+        "include_metadata": true
       },
       "Extra": {
         "title_top": "YoutubeDL-Material",

backend/consts.js

@@ -30,6 +30,14 @@ let CONFIG_ITEMS = {
         'key': 'ytdl_safe_download_override',
         'path': 'YoutubeDLMaterial.Downloader.safe_download_override'
     },
+    'ytdl_include_thumbnail': {
+        'key': 'ytdl_include_thumbnail',
+        'path': 'YoutubeDLMaterial.Downloader.include_thumbnail'
+    },
+    'ytdl_include_metadata': {
+        'key': 'ytdl_include_metadata',
+        'path': 'YoutubeDLMaterial.Downloader.include_metadata'
+    },
 
     // Extra
     'ytdl_title_top': {

backend/db.js

@@ -26,11 +26,8 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) {
 
     utils.fixVideoMetadataPerms(file_id, type, multiUserMode && multiUserMode.file_path);
 
-    // add additional info
-    file_object['uid'] = uuid();
-    file_object['registered'] = Date.now();
-    path_object = path.parse(file_object['path']);
-    file_object['path'] = path.format(path_object);
+    // add thumbnail path
+    file_object['thumbnailPath'] = utils.getDownloadedThumbnail(file_id, type, multiUserMode && multiUserMode.file_path);
 
     if (!sub) {
         if (multiUserMode) {
@@ -48,7 +45,13 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) {
         }
     }
 
-    const file_uid = registerFileDBManual(db_path, file_object)
+    const file_uid = registerFileDBManual(db_path, file_object);
+
+    // remove metadata JSON if needed
+    if (!config_api.getConfigItem('ytdl_include_metadata')) {
+        utils.deleteJSONFile(file_id, type, multiUserMode && multiUserMode.file_path)
+    }
+
     return file_uid;
 }
 

backend/package.json

@@ -37,6 +37,7 @@
     "express": "^4.17.1",
     "fluent-ffmpeg": "^2.1.2",
     "fs-extra": "^9.0.0",
+    "glob": "^7.1.6",
     "jsonwebtoken": "^8.5.1",
     "lowdb": "^1.0.0",
     "md5": "^2.2.1",

backend/subscriptions.js

@@ -345,6 +345,10 @@ async function getVideosForSub(sub, user_uid = null) {
             }
         }
 
+        if (config_api.getConfigItem('ytdl_include_thumbnail')) {
+            downloadConfig.push('--write-thumbnail');
+        }
+
         // get videos
         logger.verbose('Subscription: getting videos for subscription ' + sub.name);
         youtubedl.exec(sub.url, downloadConfig, {}, function(err, output) {

backend/utils.js

@@ -88,6 +88,41 @@ function getJSONByType(type, name, customPath, openReadPerms = false) {
     return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms)
 }
 
+function getDownloadedThumbnail(name, type, customPath = null) {
+    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') : config_api.getConfigItem('ytdl_video_folder_path');
+
+    let jpgPath = path.join(customPath, name + '.jpg');
+    let webpPath = path.join(customPath, name + '.webp');
+    let pngPath = path.join(customPath, name + '.png');
+
+    if (fs.existsSync(jpgPath))
+        return jpgPath;
+    else if (fs.existsSync(webpPath))
+        return webpPath;
+    else if (fs.existsSync(pngPath))
+        return pngPath;
+    else
+        return null;
+}
+
+function getExpectedFileSize(info_json) {
+    if (info_json['filesize']) {
+        return info_json['filesize'];
+    }
+
+    const formats = info_json['format_id'].split('+');
+    let expected_filesize = 0;
+    formats.forEach(format_id => {
+        info_json.formats.forEach(available_format => {
+            if (available_format.format_id === format_id && available_format.filesize) {
+                expected_filesize += available_format.filesize;
+            }
+        });
+    });
+
+    return expected_filesize;
+}
+
 function fixVideoMetadataPerms(name, type, customPath = null) {
     if (is_windows) return;
     if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
@@ -110,6 +145,19 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
     }
 }
 
+function deleteJSONFile(name, type, customPath = null) {
+    if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path')
+                                                   : config_api.getConfigItem('ytdl_video_folder_path');
+
+    const ext = type === 'audio' ? '.mp3' : '.mp4';
+    let json_path = path.join(customPath, name + '.info.json');
+    let alternate_json_path = path.join(customPath, name + ext + '.info.json');
+
+    if (fs.existsSync(json_path)) fs.unlinkSync(json_path);
+    if (fs.existsSync(alternate_json_path)) fs.unlinkSync(alternate_json_path);
+}
+
+
 function recFindByExt(base,ext,files,result)
 {
     files = files || fs.readdirSync(base)
@@ -153,7 +201,10 @@ module.exports = {
     getJSONMp3: getJSONMp3,
     getJSONMp4: getJSONMp4,
     getTrueFileName: getTrueFileName,
+    getDownloadedThumbnail: getDownloadedThumbnail,
+    getExpectedFileSize: getExpectedFileSize,
     fixVideoMetadataPerms: fixVideoMetadataPerms,
+    deleteJSONFile: deleteJSONFile,
     getDownloadedFilesByType: getDownloadedFilesByType,
     recFindByExt: recFindByExt,
     File: File

src/app/components/unified-file-card/unified-file-card.component.html

@@ -29,7 +29,7 @@
         <div style="padding:5px">
             <div *ngIf="!loading && file_obj.thumbnailURL" class="img-div">
                 <div style="position: relative">
-                    <img [ngClass]="{'image-small': card_size === 'small', 'image': card_size === 'medium', 'image-large': card_size === 'large'}" [src]="file_obj.thumbnailURL" alt="Thumbnail">
+                    <img [ngClass]="{'image-small': card_size === 'small', 'image': card_size === 'medium', 'image-large': card_size === 'large'}" [src]="file_obj.thumbnailBlob ? thumbnailBlobURL : file_obj.thumbnailURL" alt="Thumbnail">
                     <div class="duration-time">
                         {{file_length}}
                     </div>

src/app/components/unified-file-card/unified-file-card.component.ts

@@ -1,6 +1,7 @@
 import { Component, OnInit, Input, Output, EventEmitter } from '@angular/core';
 import { MatDialog } from '@angular/material/dialog';
 import { VideoInfoDialogComponent } from 'app/dialogs/video-info-dialog/video-info-dialog.component';
+import { DomSanitizer } from '@angular/platform-browser';
 
 @Component({
   selector: 'app-unified-file-card',
@@ -16,6 +17,10 @@ export class UnifiedFileCardComponent implements OnInit {
   type = null;
   elevated = false;
 
+  // optional vars
+  thumbnailBlobURL = null;
+
+  // input/output
   @Input() loading = true;
   @Input() theme = null;
   @Input() file_obj = null;
@@ -35,12 +40,19 @@ export class UnifiedFileCardComponent implements OnInit {
     big: 250x200
   */
 
-  constructor(private dialog: MatDialog) { }
+  constructor(private dialog: MatDialog, private sanitizer: DomSanitizer) { }
 
   ngOnInit(): void {
     if (!this.loading) {
       this.file_length = fancyTimeFormat(this.file_obj.duration);
     }
+
+    if (this.file_obj && this.file_obj.thumbnailBlob) {
+      const mime = getMimeByFilename(this.file_obj.thumbnailPath);
+      const blob = new Blob([new Uint8Array(this.file_obj.thumbnailBlob.data)], {type: mime});
+      const bloburl = URL.createObjectURL(blob);
+      this.thumbnailBlobURL = this.sanitizer.bypassSecurityTrustUrl(bloburl);
+    }
   }
 
   emitDeleteFile(blacklistMode = false) {
@@ -97,3 +109,16 @@ function fancyTimeFormat(time) {
   ret += '' + secs;
   return ret;
 }
+
+function getMimeByFilename(name) {
+  switch (name.substring(name.length-4, name.length)) {
+    case '.jpg':
+      return 'image/jpeg';
+    case '.png':
+      return 'image/png';
+    case 'webp':
+      return 'image/webp';
+    default:
+      return null;
+  }
+}

src/app/main/main.component.html

@@ -164,7 +164,7 @@
 <br/>
 <div class="centered big" id="bar_div" *ngIf="current_download && current_download.downloading; else nofile">
     <div class="margined">
-        <div [ngClass]="(+percentDownloaded > 99)?'make-room-for-spinner':'equal-sizes'" style="display: inline-block; width: 100%; padding-left: 20px" *ngIf="current_download.percent_complete && current_download.percent_complete > 15;else indeterminateprogress">
+        <div [ngClass]="(+percentDownloaded > 99)?'make-room-for-spinner':'equal-sizes'" style="display: inline-block; width: 100%; padding-left: 20px" *ngIf="current_download.percent_complete && current_download.percent_complete > 1;else indeterminateprogress">
             <mat-progress-bar style="border-radius: 5px;" mode="determinate" value="{{percentDownloaded}}"></mat-progress-bar>
             <br/>
         </div>

src/app/main/main.component.ts

@@ -1033,8 +1033,8 @@ export class MainComponent implements OnInit {
         }
       } else if (format_obj.type === 'video') {
         // check if video format is mp4
-        const key = format.height.toString();
-        if (format.ext === 'mp4') {
+        const key = format.format_note.replace('p', '');
+        if (format.ext === 'mp4' || format.ext === 'mkv' || format.ext === 'webm') {
           format_obj['height'] = format.height;
           format_obj['acodec'] = format.acodec;
           format_obj['format_id'] = format.format_id;

src/app/settings/settings.component.html

@@ -128,7 +128,11 @@
                     </div>
 
                     <div class="col-12 mt-2">
-                        <mat-checkbox color="accent" [(ngModel)]="new_config['Downloader']['safe_download_override']"><ng-container i18n="Safe download override setting">Safe download override</ng-container></mat-checkbox>
+                        <mat-checkbox color="accent" [(ngModel)]="new_config['Downloader']['include_thumbnail']"><ng-container i18n="Include thumbnail setting">Include thumbnail</ng-container></mat-checkbox>
+                    </div>
+
+                    <div class="col-12 mt-2">
+                        <mat-checkbox color="accent" [(ngModel)]="new_config['Downloader']['include_metadata']"><ng-container i18n="Include metadata setting">Include metadata</ng-container></mat-checkbox>
                     </div>
 
                     <div class="col-12 mt-2">