aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implemented basic db-based archive functionality, converted old archive functionality to new system

Browse Source
pull/809/head
Tzahi12345 3 years ago
parent 54208ce6ce
commit 840e12db71
9 changed files with 121 additions and 116 deletions
Show Stats Download Patch File Download Diff File

14
Public API v1.yaml
Unescape Escape View File

@ -1632,6 +1632,9 @@ components:
$ref: '#/components/schemas/FileType' $ref: '#/components/schemas/FileType'
cropFileSettings: cropFileSettings:
$ref: '#/components/schemas/CropFileSettings' $ref: '#/components/schemas/CropFileSettings'
ignoreArchive:
type: boolean
description: If using youtube-dl archive, download will ignore it
DownloadResponse: DownloadResponse:
type: object type: object
properties: properties:
@ -2080,16 +2083,11 @@ components:
type: type:
$ref: '#/components/schemas/FileType' $ref: '#/components/schemas/FileType'
DownloadArchiveRequest: DownloadArchiveRequest:
required:
- sub
type: object
properties:
sub:
required:
- archive_dir
type: object type: object
properties: properties:
archive_dir: type:
$ref: '#/components/schemas/FileType'
sub_id:
type: string type: string
UpdaterStatus: UpdaterStatus:
required: required:

21
backend/app.js
Unescape Escape View File

@ -522,9 +522,6 @@ async function loadConfig() {
db_api.database_initialized = true; db_api.database_initialized = true;
db_api.database_initialized_bs.next(true); db_api.database_initialized_bs.next(true);
// creates archive path if missing
await fs.ensureDir(utils.getArchiveFolder());
// check migrations // check migrations
await checkMigrations(); await checkMigrations();
@ -830,7 +827,8 @@ app.post('/api/downloadFile', optionalJwt, async function(req, res) {
youtubeUsername: req.body.youtubeUsername, youtubeUsername: req.body.youtubeUsername,
youtubePassword: req.body.youtubePassword, youtubePassword: req.body.youtubePassword,
ui_uid: req.body.ui_uid, ui_uid: req.body.ui_uid,
cropFileSettings: req.body.cropFileSettings cropFileSettings: req.body.cropFileSettings,
ignoreArchive: req.body.ignoreArchive
}; };
const download = await downloader_api.createDownload(url, type, options, user_uid); const download = await downloader_api.createDownload(url, type, options, user_uid);
@ -1517,15 +1515,18 @@ app.post('/api/downloadFileFromServer', optionalJwt, async (req, res) => {
}); });
app.post('/api/downloadArchive', optionalJwt, async (req, res) => { app.post('/api/downloadArchive', optionalJwt, async (req, res) => {
let sub = req.body.sub; const uuid = req.isAuthenticated() ? req.user.uid : null;
let archive_dir = sub.archive; const sub_id = req.body.sub_id;
const type = req.body.type;
let full_archive_path = path.join(archive_dir, 'archive.txt'); const archive_text = await archive_api.generateArchive(type, uuid, sub_id);
if (await fs.pathExists(full_archive_path)) { if (archive_text !== null && archive_text !== undefined) {
res.sendFile(full_archive_path); res.setHeader('Content-type', "application/octet-stream");
res.setHeader('Content-disposition', 'attachment; filename=archive.txt');
res.send(archive_text);
} else { } else {
res.sendStatus(404); res.sendStatus(400);
} }
}); });

21
backend/archive.js
Unescape Escape View File

@ -3,20 +3,27 @@ const fs = require('fs-extra');
const db_api = require('./db'); const db_api = require('./db');
exports.generateArchive = async (user_uid = null, sub_id = null) => { exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => {
const archive_items = await db_api.getRecords('archives', {user_uid: user_uid, sub_id: sub_id}); const filter = {user_uid: user_uid, sub_id: sub_id};
if (type) filter['type'] = type;
const archive_items = await db_api.getRecords('archives', filter);
const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`); const archive_item_lines = archive_items.map(archive_item => `${archive_item['extractor']} ${archive_item['id']}`);
return archive_item_lines.join('\n'); return archive_item_lines.join('\n');
} }
exports.addToArchive = async (extractor, id, type, user_uid = null, sub_id = null) => { exports.addToArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id); const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id);
const success = await db_api.insertRecordIntoTable('archives', archive_item, {key: {extractor: extractor, id: id}, type: type}); const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type});
return success;
}
exports.removeFromArchive = async (extractor, id, type, user_uid = null, sub_id = null) => {
const success = await db_api.removeAllRecords('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
return success; return success;
} }
exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => { exports.existsInArchive = async (extractor, id, type, user_uid, sub_id) => {
const archive_item = await db_api.getRecord('archives', {'key.extractor': extractor, 'key.id': id, type: type, user_uid: user_uid, sub_id: sub_id}); const archive_item = await db_api.getRecord('archives', {extractor: extractor, id: id, type: type, user_uid: user_uid, sub_id: sub_id});
return !!archive_item; return !!archive_item;
} }
@ -37,7 +44,7 @@ exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id =
// we can't do a bulk write because we need to avoid duplicate archive items existing in db // we can't do a bulk write because we need to avoid duplicate archive items existing in db
const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id); const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id);
await db_api.insertRecordIntoTable('archives', archive_item, {key: {extractor: extractor, id: id}}); await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id});
archive_import_count++; archive_import_count++;
} }
return archive_import_count; return archive_import_count;
@ -71,10 +78,8 @@ exports.importArchives = async () => {
const createArchiveItem = (extractor, id, type, user_uid = null, sub_id = null) => { const createArchiveItem = (extractor, id, type, user_uid = null, sub_id = null) => {
return { return {
key: {
extractor: extractor, extractor: extractor,
id: id id: id,
},
type: type, type: type,
user_uid: user_uid ? user_uid : null, user_uid: user_uid ? user_uid : null,
sub_id: sub_id ? sub_id : null sub_id: sub_id ? sub_id : null

23
backend/db.js
Unescape Escape View File

@ -64,8 +64,7 @@ const tables = {
primary_key: 'uid' primary_key: 'uid'
}, },
archives: { archives: {
name: 'archives', name: 'archives'
primary_key: 'key'
}, },
test: { test: {
name: 'test' name: 'test'
@ -510,16 +509,22 @@ exports.deleteFile = async (uid, blacklistMode = false) => {
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) { if (useYoutubeDLArchive) {
const archive_path = utils.getArchiveFolder(type, file_obj.user_uid, file_obj.sub_id ? (await exports.getRecord('subscriptions', {id: file_obj.sub_id})) : null); // get id/extractor from JSON
// get ID from JSON
var jsonobj = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath)); const info_json = await (type === 'audio' ? utils.getJSONMp3(name, folderPath) : utils.getJSONMp4(name, folderPath));
let id = null; let retrievedID = null;
if (jsonobj) id = jsonobj.id; let retrievedExtractor = null;
if (info_json) {
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
}
// Remove file ID from the archive file, and write it to the blacklist (if enabled) // Remove file ID from the archive file, and write it to the blacklist (if enabled)
await utils.deleteFileFromArchive(uid, type, archive_path, id, blacklistMode); if (!blacklistMode) {
// workaround until a files_api is created (using archive_api would make a circular dependency)
await exports.removeAllRecords('archives', {extractor: retrievedExtractor, id: retrievedID, type: type, user_uid: file_obj.user_uid, sub_id: file_obj.sub_id});
// await archive_api.removeFromArchive(retrievedExtractor, retrievedID, type, file_obj.user_uid, file_obj.sub_id);
}
} }
if (jsonExists) await fs.unlink(jsonPath); if (jsonExists) await fs.unlink(jsonPath);

58
backend/downloader.js
Unescape Escape View File

@ -15,6 +15,7 @@ const categories_api = require('./categories');
const utils = require('./utils'); const utils = require('./utils');
const db_api = require('./db'); const db_api = require('./db');
const notifications_api = require('./notifications'); const notifications_api = require('./notifications');
const archive_api = require('./archive');
const mutex = new Mutex(); const mutex = new Mutex();
let should_check_downloads = true; let should_check_downloads = true;
@ -202,6 +203,20 @@ async function collectInfo(download_uid) {
return; return;
} }
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) {
const exists_in_archive = await archive_api.existsInArchive(info['extractor'], info['id'], type, download['user_uid'], download['sub_id']);
if (exists_in_archive) {
const error = `File '${info['title']}' already exists in archive! Disable the archive or override to continue downloading.`;
logger.warn(error);
if (download_uid) {
const download = await db_api.getRecord('download_queue', {uid: download_uid});
await handleDownloadError(download, error);
return;
}
}
}
let category = null; let category = null;
// check if it fits into a category. If so, then get info again using new args // check if it fits into a category. If so, then get info again using new args
@ -353,19 +368,14 @@ async function downloadQueuedFile(download_uid) {
// registers file in DB // registers file in DB
const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings);
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, download['user_uid'], download['sub_id']);
notifications_api.sendDownloadNotification(file_obj, download['user_uid']); notifications_api.sendDownloadNotification(file_obj, download['user_uid']);
file_objs.push(file_obj); file_objs.push(file_obj);
} }
if (options.merged_string !== null && options.merged_string !== undefined) {
const archive_folder = getArchiveFolder(fileFolderPath, options, download['user_uid']);
const current_merged_archive = fs.readFileSync(path.join(archive_folder, `merged_${type}.txt`), 'utf8');
const diff = current_merged_archive.replace(options.merged_string, '');
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
fs.appendFileSync(archive_path, diff);
}
let container = null; let container = null;
if (file_objs.length > 1) { if (file_objs.length > 1) {
@ -478,28 +488,6 @@ exports.generateArgs = async (url, type, options, user_uid = null, simulated = f
downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent); downloadConfig.splice(0, 0, '--external-downloader', customDownloadingAgent);
} }
let useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const archive_folder = getArchiveFolder(fileFolderPath, options, user_uid);
const archive_path = path.join(archive_folder, `archive_${type}.txt`);
await fs.ensureDir(archive_folder);
await fs.ensureFile(archive_path);
const blacklist_path = path.join(archive_folder, `blacklist_${type}.txt`);
await fs.ensureFile(blacklist_path);
const merged_path = path.join(archive_folder, `merged_${type}.txt`);
await fs.ensureFile(merged_path);
// merges blacklist and regular archive
let inputPathList = [archive_path, blacklist_path];
await mergeFiles(inputPathList, merged_path);
options.merged_string = await fs.readFile(merged_path, "utf8");
downloadConfig.push('--download-archive', merged_path);
}
if (config_api.getConfigItem('ytdl_include_thumbnail')) { if (config_api.getConfigItem('ytdl_include_thumbnail')) {
downloadConfig.push('--write-thumbnail'); downloadConfig.push('--write-thumbnail');
} }
@ -651,13 +639,3 @@ exports.generateNFOFile = (info, output_path) => {
const xml = doc.end({ prettyPrint: true }); const xml = doc.end({ prettyPrint: true });
fs.writeFileSync(output_path, xml); fs.writeFileSync(output_path, xml);
} }
function getArchiveFolder(fileFolderPath, options, user_uid) {
if (options.customArchivePath) {
return path.join(options.customArchivePath);
} else if (user_uid) {
return path.join(fileFolderPath, 'archives');
} else {
return path.join('appdata', 'archives');
}
}

39
backend/subscriptions.js
Unescape Escape View File

@ -3,6 +3,7 @@ const path = require('path');
const youtubedl = require('youtube-dl'); const youtubedl = require('youtube-dl');
const config_api = require('./config'); const config_api = require('./config');
const archive_api = require('./archive');
const utils = require('./utils'); const utils = require('./utils');
const logger = require('./logger'); const logger = require('./logger');
@ -160,10 +161,10 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
let basePath = null; let basePath = null;
basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions') basePath = user_uid ? path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions')
: config_api.getConfigItem('ytdl_subscriptions_base_path'); : config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const appendedBasePath = getAppendedBasePath(sub, basePath); const appendedBasePath = getAppendedBasePath(sub, basePath);
const name = file; const name = file;
let retrievedID = null; let retrievedID = null;
let retrievedExtractor = null;
await db_api.removeRecord('files', {uid: file_uid}); await db_api.removeRecord('files', {uid: file_uid});
@ -182,7 +183,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
]); ]);
if (jsonExists) { if (jsonExists) {
retrievedID = fs.readJSONSync(jsonPath)['id']; const info_json = fs.readJSONSync(jsonPath);
retrievedID = info_json['id'];
retrievedExtractor = info_json['extractor'];
await fs.unlink(jsonPath); await fs.unlink(jsonPath);
} }
@ -200,11 +203,9 @@ async function deleteSubscriptionFile(sub, file, deleteForever, file_uid = null,
return false; return false;
} else { } else {
// check if the user wants the video to be redownloaded (deleteForever === false) // check if the user wants the video to be redownloaded (deleteForever === false)
if (useArchive && retrievedID) { const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
const archive_path = utils.getArchiveFolder(sub.type, user_uid, sub); if (useArchive && !deleteForever) {
await archive_api.removeFromArchive(retrievedExtractor, retrievedID, sub.type, user_uid, sub.id);
// Remove file ID from the archive file, and write it to the blacklist (if enabled)
await utils.deleteFileFromArchive(file_uid, sub.type, archive_path, retrievedID, deleteForever);
} }
return true; return true;
} }
@ -335,8 +336,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
else else
basePath = config_api.getConfigItem('ytdl_subscriptions_base_path'); basePath = config_api.getConfigItem('ytdl_subscriptions_base_path');
const useArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
let appendedBasePath = getAppendedBasePath(sub, basePath); let appendedBasePath = getAppendedBasePath(sub, basePath);
const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s'; const file_output = config_api.getConfigItem('ytdl_default_file_output') ? config_api.getConfigItem('ytdl_default_file_output') : '%(title)s';
@ -372,21 +371,6 @@ async function generateArgsForSubscription(sub, user_uid, redownload = false, de
downloadConfig.push(...customArgsArray); downloadConfig.push(...customArgsArray);
} }
let archive_dir = null;
let archive_path = null;
if (useArchive && !redownload) {
if (sub.archive) {
archive_dir = sub.archive;
if (sub.type && sub.type === 'audio') {
archive_path = path.join(archive_dir, 'merged_audio.txt');
} else {
archive_path = path.join(archive_dir, 'merged_video.txt');
}
}
downloadConfig.push('--download-archive', archive_path);
}
if (sub.timerange && !redownload) { if (sub.timerange && !redownload) {
downloadConfig.push('--dateafter', sub.timerange); downloadConfig.push('--dateafter', sub.timerange);
} }
@ -429,7 +413,14 @@ async function getFilesToDownload(sub, output_jsons) {
if (file_with_path_exists) { if (file_with_path_exists) {
// or maybe just overwrite??? // or maybe just overwrite???
logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`) logger.info(`Skipping adding file ${output_json['_filename']} for subscription ${sub.name} as a file with that path already exists.`)
continue;
}
const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive');
if (useYoutubeDLArchive) {
const exists_in_archive = await archive_api.existsInArchive(output_json['extractor'], output_json['id'], sub.type, sub.user_uid, sub.id);
if (exists_in_archive) continue;
} }
files_to_download.push(output_json); files_to_download.push(output_json);
} }
} }

44
backend/test/tests.js
Unescape Escape View File

@ -625,6 +625,7 @@ describe('Tasks', function() {
describe('Archive', async function() { describe('Archive', async function() {
beforeEach(async function() { beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('archives', {user_uid: 'test_user'}); await db_api.removeAllRecords('archives', {user_uid: 'test_user'});
}); });
@ -645,25 +646,50 @@ describe('Archive', async function() {
const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'}); const archive_items = await db_api.getRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
console.log(archive_items); console.log(archive_items);
assert(archive_items.length === 4); assert(archive_items.length === 4);
assert(archive_items.filter(archive_item => archive_item.key.extractor === 'testextractor2').length === 1); assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor2').length === 1);
assert(archive_items.filter(archive_item => archive_item.key.extractor === 'testextractor1').length === 3); assert(archive_items.filter(archive_item => archive_item.extractor === 'testextractor1').length === 3);
const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'}); const success = await db_api.removeAllRecords('archives', {user_uid: 'test_user', sub_id: 'test_sub'});
assert(success); assert(success);
}); });
it('Get archive', async function() { it('Get archive', async function() {
await archive_api.addToArchive('testextractor1', 'video', 'testing1', 'test_user'); await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'video', 'testing1', 'test_user'); await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'video', 'testing1', 'test_user');
const archive_item1 = await db_api.getRecord('archives', {key: {extractor: 'testextractor1', id: 'testing1'}}); const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
const archive_item2 = await db_api.getRecord('archives', {key: {extractor: 'testextractor2', id: 'testing1'}}); const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
assert(archive_item1 && archive_item2); assert(archive_item1 && archive_item2);
});
it('Archive duplicates', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor1', 'testing1', 'audio', 'test_user');
const count = await db_api.getRecords('archives', {id: 'testing1'}, true);
assert(count === 3);
});
it('Remove from archive', async function() {
await archive_api.addToArchive('testextractor1', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing1', 'video', 'test_user');
await archive_api.addToArchive('testextractor2', 'testing2', 'video', 'test_user');
const success = await archive_api.removeFromArchive('testextractor2', 'testing1', 'video', 'test_user');
assert(success);
const archive_item1 = await db_api.getRecord('archives', {extractor: 'testextractor1', id: 'testing1'});
assert(!!archive_item1);
const archive_item2 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing1'});
assert(!archive_item2);
const count = await db_api.getRecords('archives', {key: {id: 'testing1'}}, true); const archive_item3 = await db_api.getRecord('archives', {extractor: 'testextractor2', id: 'testing2'});
assert(count === 2); assert(!!archive_item3);
}); });
}); });

7
src/api-types/models/DownloadArchiveRequest.ts
Unescape Escape View File

@ -2,8 +2,9 @@
/* tslint:disable */ /* tslint:disable */
/* eslint-disable */ /* eslint-disable */
import type { FileType } from './FileType';
export type DownloadArchiveRequest = { export type DownloadArchiveRequest = {
sub: { type?: FileType;
archive_dir: string; sub_id?: string;
};
}; };

4
src/app/posts.services.ts
Unescape Escape View File

@ -434,8 +434,8 @@ export class PostsService implements CanActivate {
return this.http.post<SuccessObject>(this.path + 'uploadCookies', fileFormData, this.httpOptions); return this.http.post<SuccessObject>(this.path + 'uploadCookies', fileFormData, this.httpOptions);
} }
downloadArchive(sub) { downloadArchive(type: FileType, sub_id: string) {
const body: DownloadArchiveRequest = {sub: sub}; const body: DownloadArchiveRequest = {type: type, sub_id: sub_id};
return this.http.post(this.path + 'downloadArchive', body, {responseType: 'blob', params: this.httpOptions.params}); return this.http.post(this.path + 'downloadArchive', body, {responseType: 'blob', params: this.httpOptions.params});
} }

Write Preview
Loading…
Cancel
Save