aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make utils.recFindByExt and utils.getDownloadedFilesByType async

Browse Source
pull/216/head
Tiger Oakes 5 years ago
parent f535d18cb9
commit 21eafeab22
3 changed files with 55 additions and 75 deletions
Show Stats Download Patch File Download Diff File

111
backend/app.js
Unescape Escape View File

@ -205,54 +205,49 @@ async function wait(ms) {
} }
async function checkMigrations() { async function checkMigrations() {
return new Promise(async resolve => { // 3.5->3.6 migration
// 3.5->3.6 migration const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value();
const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value();
if (!files_to_db_migration_complete) {
logger.info('Beginning migration: 3.5->3.6+')
runFilesToDBMigration().then(success => {
if (success) { logger.info('3.5->3.6+ migration complete!'); }
else { logger.error('Migration failed: 3.5->3.6+'); }
});
}
resolve(true); if (!files_to_db_migration_complete) {
}); logger.info('Beginning migration: 3.5->3.6+')
const success = await runFilesToDBMigration()
if (success) { logger.info('3.5->3.6+ migration complete!'); }
else { logger.error('Migration failed: 3.5->3.6+'); }
}
return true;
} }
async function runFilesToDBMigration() { async function runFilesToDBMigration() {
return new Promise(async resolve => { try {
try { let mp3s = await getMp3s();
let mp3s = getMp3s(); let mp4s = await getMp4s();
let mp4s = getMp4s();
for (let i = 0; i < mp3s.length; i++) {
for (let i = 0; i < mp3s.length; i++) { let file_obj = mp3s[i];
let file_obj = mp3s[i]; const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value();
const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value(); if (!file_already_in_db) {
if (!file_already_in_db) { logger.verbose(`Migrating file ${file_obj.id}`);
logger.verbose(`Migrating file ${file_obj.id}`); await db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
}
} }
}
for (let i = 0; i < mp4s.length; i++) { for (let i = 0; i < mp4s.length; i++) {
let file_obj = mp4s[i]; let file_obj = mp4s[i];
const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value(); const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value();
if (!file_already_in_db) { if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`); logger.verbose(`Migrating file ${file_obj.id}`);
db_api.registerFileDB(file_obj.id + '.mp4', 'video'); await db_api.registerFileDB(file_obj.id + '.mp4', 'video');
}
} }
// sets migration to complete
db.set('files_to_db_migration_complete', true).write();
resolve(true);
} catch(err) {
logger.error(err);
resolve(false);
} }
});
// sets migration to complete
db.set('files_to_db_migration_complete', true).write();
return true;
} catch(err) {
logger.error(err);
return false;
}
} }
async function startServer() { async function startServer() {
@ -701,17 +696,17 @@ function generateEnvVarConfigItem(key) {
return {key: key, value: process['env'][key]}; return {key: key, value: process['env'][key]};
} }
function getMp3s() { async function getMp3s() {
let mp3s = []; let mp3s = [];
var files = utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath); var files = await utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath);
for (let i = 0; i < files.length; i++) { for (let i = 0; i < files.length; i++) {
let file = files[i]; let file = files[i];
var file_path = file.substring(audioFolderPath.length, file.length); var file_path = file.substring(audioFolderPath.length, file.length);
var stats = fs.statSync(file); var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4); var id = file_path.substring(0, file_path.length-4);
var jsonobj = utils.getJSONMp3(id, audioFolderPath); var jsonobj = await utils.getJSONMp3(id, audioFolderPath);
if (!jsonobj) continue; if (!jsonobj) continue;
var title = jsonobj.title; var title = jsonobj.title;
var url = jsonobj.webpage_url; var url = jsonobj.webpage_url;
@ -730,9 +725,9 @@ function getMp3s() {
return mp3s; return mp3s;
} }
function getMp4s(relative_path = true) { async function getMp4s(relative_path = true) {
let mp4s = []; let mp4s = [];
var files = utils.recFindByExt(videoFolderPath, 'mp4'); var files = await utils.recFindByExt(videoFolderPath, 'mp4');
for (let i = 0; i < files.length; i++) { for (let i = 0; i < files.length; i++) {
let file = files[i]; let file = files[i];
var file_path = file.substring(videoFolderPath.length, file.length); var file_path = file.substring(videoFolderPath.length, file.length);
@ -740,7 +735,7 @@ function getMp4s(relative_path = true) {
var stats = fs.statSync(file); var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4); var id = file_path.substring(0, file_path.length-4);
var jsonobj = utils.getJSONMp4(id, videoFolderPath); var jsonobj = await utils.getJSONMp4(id, videoFolderPath);
if (!jsonobj) continue; if (!jsonobj) continue;
var title = jsonobj.title; var title = jsonobj.title;
var url = jsonobj.webpage_url; var url = jsonobj.webpage_url;
@ -934,7 +929,7 @@ async function deleteAudioFile(name, customPath = null, blacklistMode = false) {
// get ID from JSON // get ID from JSON
var jsonobj = utils.getJSONMp3(name, filePath); var jsonobj = await utils.getJSONMp3(name, filePath);
let id = null; let id = null;
if (jsonobj) id = jsonobj.id; if (jsonobj) id = jsonobj.id;
@ -1009,7 +1004,7 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
// get ID from JSON // get ID from JSON
var jsonobj = utils.getJSONMp4(name, filePath); var jsonobj = await utils.getJSONMp4(name, filePath);
let id = null; let id = null;
if (jsonobj) id = jsonobj.id; if (jsonobj) id = jsonobj.id;
@ -1038,22 +1033,6 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
} }
} }
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path;
const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
let unfixed_parts = unfixed_path.split('.');
const old_ext = unfixed_parts[unfixed_parts.length-1];
if (old_ext !== new_ext) {
unfixed_parts[unfixed_parts.length-1] = new_ext;
fixed_path = unfixed_parts.join('.');
}
return fixed_path;
}
/** /**
* @param {'audio' | 'video'} type * @param {'audio' | 'video'} type
* @param {string[]} fileNames * @param {string[]} fileNames
@ -2214,7 +2193,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/'); let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
let files; let files;
try { try {
files = utils.recFindByExt(appended_base_path, 'mp4'); files = await utils.recFindByExt(appended_base_path, 'mp4');
} catch(e) { } catch(e) {
files = null; files = null;
logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path); logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path);

2
backend/db.js
Unescape Escape View File

@ -184,7 +184,7 @@ async function importUnregisteredFiles() {
// run through check list and check each file to see if it's missing from the db // run through check list and check each file to see if it's missing from the db
dirs_to_check.forEach(dir_to_check => { dirs_to_check.forEach(dir_to_check => {
// recursively get all files in dir's path // recursively get all files in dir's path
const files = utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
files.forEach(file => { files.forEach(file => {
// check if file exists in db, if not add it // check if file exists in db, if not add it

17
backend/utils.js
Unescape Escape View File

@ -4,6 +4,7 @@ const config_api = require('./config');
const is_windows = process.platform === 'win32'; const is_windows = process.platform === 'win32';
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) { function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path; let fixed_path = unfixed_path;
@ -19,21 +20,21 @@ function getTrueFileName(unfixed_path, type) {
return fixed_path; return fixed_path;
} }
function getDownloadedFilesByType(basePath, type) { async function getDownloadedFilesByType(basePath, type) {
// return empty array if the path doesn't exist // return empty array if the path doesn't exist
if (!fs.existsSync(basePath)) return []; if (!(await fs.pathExists(basePath))) return [];
let files = []; let files = [];
const ext = type === 'audio' ? 'mp3' : 'mp4'; const ext = type === 'audio' ? 'mp3' : 'mp4';
var located_files = recFindByExt(basePath, ext); var located_files = await recFindByExt(basePath, ext);
for (let i = 0; i < located_files.length; i++) { for (let i = 0; i < located_files.length; i++) {
let file = located_files[i]; let file = located_files[i];
var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length); var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
var stats = fs.statSync(file); var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4); var id = file_path.substring(0, file_path.length-4);
var jsonobj = getJSONByType(type, id, basePath); var jsonobj = await getJSONByType(type, id, basePath);
if (!jsonobj) continue; if (!jsonobj) continue;
var title = jsonobj.title; var title = jsonobj.title;
var url = jsonobj.webpage_url; var url = jsonobj.webpage_url;
@ -129,7 +130,7 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
: config_api.getConfigItem('ytdl_video_folder_path'); : config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4'; const ext = type === 'audio' ? '.mp3' : '.mp4';
const files_to_fix = [ const files_to_fix = [
// JSONs // JSONs
path.join(customPath, name + '.info.json'), path.join(customPath, name + '.info.json'),
@ -158,7 +159,7 @@ function deleteJSONFile(name, type, customPath = null) {
} }
function recFindByExt(base,ext,files,result) async function recFindByExt(base,ext,files,result)
{ {
files = files || fs.readdirSync(base) files = files || fs.readdirSync(base)
result = result || [] result = result || []
@ -168,7 +169,7 @@ function recFindByExt(base,ext,files,result)
var newbase = path.join(base,file) var newbase = path.join(base,file)
if ( fs.statSync(newbase).isDirectory() ) if ( fs.statSync(newbase).isDirectory() )
{ {
result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result) result = await recFindByExt(newbase,ext,fs.readdirSync(newbase),result)
} }
else else
{ {

Write Preview
Loading…
Cancel
Save