aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make utils.recFindByExt and utils.getDownloadedFilesByType async

Browse Source
pull/216/head
Tiger Oakes 5 years ago
parent f535d18cb9
commit 21eafeab22
3 changed files with 55 additions and 75 deletions
Show Stats Download Patch File Download Diff File

111
backend/app.js
Unescape Escape View File

@ -205,54 +205,49 @@ async function wait(ms) {
}
async function checkMigrations() {
return new Promise(async resolve => {
// 3.5->3.6 migration
const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value();
if (!files_to_db_migration_complete) {
logger.info('Beginning migration: 3.5->3.6+')
runFilesToDBMigration().then(success => {
if (success) { logger.info('3.5->3.6+ migration complete!'); }
else { logger.error('Migration failed: 3.5->3.6+'); }
});
}
// 3.5->3.6 migration
const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value();
resolve(true);
});
if (!files_to_db_migration_complete) {
logger.info('Beginning migration: 3.5->3.6+')
const success = await runFilesToDBMigration()
if (success) { logger.info('3.5->3.6+ migration complete!'); }
else { logger.error('Migration failed: 3.5->3.6+'); }
}
return true;
}
async function runFilesToDBMigration() {
return new Promise(async resolve => {
try {
let mp3s = getMp3s();
let mp4s = getMp4s();
for (let i = 0; i < mp3s.length; i++) {
let file_obj = mp3s[i];
const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
}
try {
let mp3s = await getMp3s();
let mp4s = await getMp4s();
for (let i = 0; i < mp3s.length; i++) {
let file_obj = mp3s[i];
const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
await db_api.registerFileDB(file_obj.id + '.mp3', 'audio');
}
}
for (let i = 0; i < mp4s.length; i++) {
let file_obj = mp4s[i];
const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
db_api.registerFileDB(file_obj.id + '.mp4', 'video');
}
for (let i = 0; i < mp4s.length; i++) {
let file_obj = mp4s[i];
const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value();
if (!file_already_in_db) {
logger.verbose(`Migrating file ${file_obj.id}`);
await db_api.registerFileDB(file_obj.id + '.mp4', 'video');
}
// sets migration to complete
db.set('files_to_db_migration_complete', true).write();
resolve(true);
} catch(err) {
logger.error(err);
resolve(false);
}
});
// sets migration to complete
db.set('files_to_db_migration_complete', true).write();
return true;
} catch(err) {
logger.error(err);
return false;
}
}
async function startServer() {
@ -701,17 +696,17 @@ function generateEnvVarConfigItem(key) {
return {key: key, value: process['env'][key]};
}
function getMp3s() {
async function getMp3s() {
let mp3s = [];
var files = utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath);
var files = await utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath);
for (let i = 0; i < files.length; i++) {
let file = files[i];
var file_path = file.substring(audioFolderPath.length, file.length);
var stats = fs.statSync(file);
var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = utils.getJSONMp3(id, audioFolderPath);
var jsonobj = await utils.getJSONMp3(id, audioFolderPath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
@ -730,9 +725,9 @@ function getMp3s() {
return mp3s;
}
function getMp4s(relative_path = true) {
async function getMp4s(relative_path = true) {
let mp4s = [];
var files = utils.recFindByExt(videoFolderPath, 'mp4');
var files = await utils.recFindByExt(videoFolderPath, 'mp4');
for (let i = 0; i < files.length; i++) {
let file = files[i];
var file_path = file.substring(videoFolderPath.length, file.length);
@ -740,7 +735,7 @@ function getMp4s(relative_path = true) {
var stats = fs.statSync(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = utils.getJSONMp4(id, videoFolderPath);
var jsonobj = await utils.getJSONMp4(id, videoFolderPath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
@ -934,7 +929,7 @@ async function deleteAudioFile(name, customPath = null, blacklistMode = false) {
// get ID from JSON
var jsonobj = utils.getJSONMp3(name, filePath);
var jsonobj = await utils.getJSONMp3(name, filePath);
let id = null;
if (jsonobj) id = jsonobj.id;
@ -1009,7 +1004,7 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
// get ID from JSON
var jsonobj = utils.getJSONMp4(name, filePath);
var jsonobj = await utils.getJSONMp4(name, filePath);
let id = null;
if (jsonobj) id = jsonobj.id;
@ -1038,22 +1033,6 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) {
}
}
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path;
const new_ext = (type === 'audio' ? 'mp3' : 'mp4');
let unfixed_parts = unfixed_path.split('.');
const old_ext = unfixed_parts[unfixed_parts.length-1];
if (old_ext !== new_ext) {
unfixed_parts[unfixed_parts.length-1] = new_ext;
fixed_path = unfixed_parts.join('.');
}
return fixed_path;
}
/**
* @param {'audio' | 'video'} type
* @param {string[]} fileNames
@ -2214,7 +2193,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => {
let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/');
let files;
try {
files = utils.recFindByExt(appended_base_path, 'mp4');
files = await utils.recFindByExt(appended_base_path, 'mp4');
} catch(e) {
files = null;
logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path);

2
backend/db.js
Unescape Escape View File

@ -184,7 +184,7 @@ async function importUnregisteredFiles() {
// run through check list and check each file to see if it's missing from the db
dirs_to_check.forEach(dir_to_check => {
// recursively get all files in dir's path
const files = utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
const files = await utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type);
files.forEach(file => {
// check if file exists in db, if not add it

17
backend/utils.js
Unescape Escape View File

@ -4,6 +4,7 @@ const config_api = require('./config');
const is_windows = process.platform === 'win32';
// replaces .webm with appropriate extension
function getTrueFileName(unfixed_path, type) {
let fixed_path = unfixed_path;
@ -19,21 +20,21 @@ function getTrueFileName(unfixed_path, type) {
return fixed_path;
}
function getDownloadedFilesByType(basePath, type) {
async function getDownloadedFilesByType(basePath, type) {
// return empty array if the path doesn't exist
if (!fs.existsSync(basePath)) return [];
if (!(await fs.pathExists(basePath))) return [];
let files = [];
const ext = type === 'audio' ? 'mp3' : 'mp4';
var located_files = recFindByExt(basePath, ext);
var located_files = await recFindByExt(basePath, ext);
for (let i = 0; i < located_files.length; i++) {
let file = located_files[i];
var file_path = file.substring(basePath.includes('\\') ? basePath.length+1 : basePath.length, file.length);
var stats = fs.statSync(file);
var stats = await fs.stat(file);
var id = file_path.substring(0, file_path.length-4);
var jsonobj = getJSONByType(type, id, basePath);
var jsonobj = await getJSONByType(type, id, basePath);
if (!jsonobj) continue;
var title = jsonobj.title;
var url = jsonobj.webpage_url;
@ -129,7 +130,7 @@ function fixVideoMetadataPerms(name, type, customPath = null) {
: config_api.getConfigItem('ytdl_video_folder_path');
const ext = type === 'audio' ? '.mp3' : '.mp4';
const files_to_fix = [
// JSONs
path.join(customPath, name + '.info.json'),
@ -158,7 +159,7 @@ function deleteJSONFile(name, type, customPath = null) {
}
function recFindByExt(base,ext,files,result)
async function recFindByExt(base,ext,files,result)
{
files = files || fs.readdirSync(base)
result = result || []
@ -168,7 +169,7 @@ function recFindByExt(base,ext,files,result)
var newbase = path.join(base,file)
if ( fs.statSync(newbase).isDirectory() )
{
result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result)
result = await recFindByExt(newbase,ext,fs.readdirSync(newbase),result)
}
else
{

Write Preview
Loading…
Cancel
Save