diff --git a/backend/app.js b/backend/app.js index c088ad6..c6cddc5 100644 --- a/backend/app.js +++ b/backend/app.js @@ -585,6 +585,8 @@ async function loadConfig() { }, subscriptionsCheckInterval * 1000); } + db_api.importUnregisteredFiles(); + // check migrations await checkMigrations(); @@ -730,7 +732,7 @@ function generateEnvVarConfigItem(key) { function getMp3s() { let mp3s = []; - var files = recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath); + var files = utils.recFindByExt(audioFolderPath, 'mp3'); // fs.readdirSync(audioFolderPath); for (let i = 0; i < files.length; i++) { let file = files[i]; var file_path = file.substring(audioFolderPath.length, file.length); @@ -759,7 +761,7 @@ function getMp3s() { function getMp4s(relative_path = true) { let mp4s = []; - var files = recFindByExt(videoFolderPath, 'mp4'); + var files = utils.recFindByExt(videoFolderPath, 'mp4'); for (let i = 0; i < files.length; i++) { let file = files[i]; var file_path = file.substring(videoFolderPath.length, file.length); @@ -1078,88 +1080,6 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) { }); } -function recFindByExt(base,ext,files,result) -{ - files = files || fs.readdirSync(base) - result = result || [] - - files.forEach( - function (file) { - var newbase = path.join(base,file) - if ( fs.statSync(newbase).isDirectory() ) - { - result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result) - } - else - { - if ( file.substr(-1*(ext.length+1)) == '.' + ext ) - { - result.push(newbase) - } - } - } - ) - return result -} -/* -function registerFileDB(file_path, type, multiUserMode = null) { - const file_id = file_path.substring(0, file_path.length-4); - const file_object = generateFileObject(file_id, type, multiUserMode && multiUserMode.file_path); - if (!file_object) { - logger.error(`Could not find associated JSON file for ${type} file ${file_id}`); - return false; - } - - // add additional info - file_object['uid'] = uuid(); - file_object['registered'] = Date.now(); - path_object = path.parse(file_object['path']); - file_object['path'] = path.format(path_object); - - if (multiUserMode) { - auth_api.registerUserFile(multiUserMode.user, file_object, type); - } else if (type === 'audio' || type === 'video') { - // remove existing video if overwriting - db.get(`files.${type}`) - .remove({ - path: file_object['path'] - }).write(); - - db.get(`files.${type}`) - .push(file_object) - .write(); - } else if (type == 'subscription') { - - } - - return file_object['uid']; -} - -function generateFileObject(id, type, customPath = null) { - var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true); - if (!jsonobj) { - return null; - } - const ext = (type === 'audio') ? '.mp3' : '.mp4' - const file_path = getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext); - // console. - var stats = fs.statSync(path.join(__dirname, file_path)); - - var title = jsonobj.title; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : 'N/A'; - - var size = stats.size; - - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - var isaudio = type === 'audio'; - var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date); - return file_obj; -} -*/ // replaces .webm with appropriate extension function getTrueFileName(unfixed_path, type) { let fixed_path = unfixed_path; @@ -2292,7 +2212,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/'); let files; try { - files = recFindByExt(appended_base_path, 'mp4'); + files = utils.recFindByExt(appended_base_path, 'mp4'); } catch(e) { files = null; logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path); diff --git a/backend/db.js b/backend/db.js index ca1c2db..aeebee1 100644 --- a/backend/db.js +++ b/backend/db.js @@ -116,8 +116,89 @@ function getAppendedBasePathSub(sub, base_path) { return path.join(base_path, (sub.isPlaylist ? 'playlists/' : 'channels/'), sub.name); } +async function importUnregisteredFiles() { + let dirs_to_check = []; + let subscriptions_to_check = []; + const subscriptions_base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); // only for single-user mode + const multi_user_mode = config_api.getConfigItem('ytdl_multi_user_mode'); + const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); + const subscriptions_enabled = config_api.getConfigItem('ytdl_allow_subscriptions'); + if (multi_user_mode) { + let users = users_db.get('users').value(); + for (let i = 0; i < users.length; i++) { + const user = users[i]; + + if (subscriptions_enabled) subscriptions_to_check = subscriptions_to_check.concat(users[i]['subscriptions']); + + // add user's audio dir to check list + dirs_to_check.push({ + basePath: path.join(usersFileFolder, user.uid, 'audio'), + dbPath: users_db.get('users').find({uid: user.uid}).get('files.audio'), + type: 'audio' + }); + + // add user's video dir to check list + dirs_to_check.push({ + basePath: path.join(usersFileFolder, user.uid, 'video'), + dbPath: users_db.get('users').find({uid: user.uid}).get('files.video'), + type: 'video' + }); + } + } else { + const audioFolderPath = config_api.getConfigItem('ytdl_audio_folder_path'); + const videoFolderPath = config_api.getConfigItem('ytdl_video_folder_path'); + const subscriptions = db.get('subscriptions').value(); + + if (subscriptions_enabled && subscriptions) subscriptions_to_check = subscriptions_to_check.concat(subscriptions); + + // add audio dir to check list + dirs_to_check.push({ + basePath: audioFolderPath, + dbPath: db.get('files.audio'), + type: 'audio' + }); + + // add video dir to check list + dirs_to_check.push({ + basePath: videoFolderPath, + dbPath: db.get('files.video'), + type: 'video' + }); + } + + // add subscriptions to check list + for (let i = 0; i < subscriptions_to_check.length; i++) { + let subscription_to_check = subscriptions_to_check[i]; + console.log(subscription_to_check); + dirs_to_check.push({ + basePath: multi_user_mode ? path.join(usersFileFolder, subscription_to_check.user_uid, 'subscriptions', subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name) + : path.join(subscriptions_base_path, subscription_to_check.isPlaylist ? 'playlists/' : 'channels/', subscription_to_check.name), + dbPath: multi_user_mode ? users_db.get('users').find({uid: subscription_to_check.user_uid}).get('subscriptions').find({id: subscription_to_check.id}).get('videos') + : db.get('subscriptions').find({id: subscription_to_check.id}).get('videos'), + type: subscription_to_check.type + }); + } + + // run through check list and check each file to see if it's missing from the db + dirs_to_check.forEach(dir_to_check => { + // recursively get all files in dir's path + const files = utils.getDownloadedFilesByType(dir_to_check.basePath, dir_to_check.type); + + files.forEach(file => { + // check if file exists in db, if not add it + const file_is_registered = !!(dir_to_check.dbPath.find({id: file.id}).value()) + if (!file_is_registered) { + dir_to_check.dbPath.push(file).write(); + logger.verbose(`Added discovered file to the database: ${file.id}`); + } + }); + }); + +} + module.exports = { initialize: initialize, registerFileDB: registerFileDB, - updatePlaylist: updatePlaylist + updatePlaylist: updatePlaylist, + importUnregisteredFiles: importUnregisteredFiles } diff --git a/backend/utils.js b/backend/utils.js index 91b01cd..f5dd47b 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -19,6 +19,36 @@ function getTrueFileName(unfixed_path, type) { return fixed_path; } +function getDownloadedFilesByType(basePath, type) { + let files = []; + const ext = type === 'audio' ? 'mp3' : 'mp4'; + var located_files = recFindByExt(basePath, ext); + for (let i = 0; i < located_files.length; i++) { + let file = located_files[i]; + var file_path = path.basename(file); + + var stats = fs.statSync(file); + + var id = file_path.substring(0, file_path.length-4); + var jsonobj = getJSONByType(type, id, basePath); + if (!jsonobj) continue; + var title = jsonobj.title; + var url = jsonobj.webpage_url; + var uploader = jsonobj.uploader; + var upload_date = jsonobj.upload_date; + upload_date = upload_date ? `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}` : null; + var thumbnail = jsonobj.thumbnail; + var duration = jsonobj.duration; + + var size = stats.size; + + var isaudio = type === 'audio'; + var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); + files.push(file_obj); + } + return files; +} + function getJSONMp4(name, customPath, openReadPerms = false) { var obj = null; // output if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path'); @@ -51,6 +81,10 @@ function getJSONMp3(name, customPath, openReadPerms = false) { return obj; } +function getJSONByType(type, name, customPath, openReadPerms = false) { + return type === 'audio' ? getJSONMp3(name, customPath, openReadPerms) : getJSONMp4(name, customPath, openReadPerms) +} + function fixVideoMetadataPerms(name, type, customPath = null) { if (is_windows) return; if (!customPath) customPath = type === 'audio' ? config_api.getConfigItem('ytdl_audio_folder_path') @@ -73,6 +107,30 @@ function fixVideoMetadataPerms(name, type, customPath = null) { } } +function recFindByExt(base,ext,files,result) +{ + files = files || fs.readdirSync(base) + result = result || [] + + files.forEach( + function (file) { + var newbase = path.join(base,file) + if ( fs.statSync(newbase).isDirectory() ) + { + result = recFindByExt(newbase,ext,fs.readdirSync(newbase),result) + } + else + { + if ( file.substr(-1*(ext.length+1)) == '.' + ext ) + { + result.push(newbase) + } + } + } + ) + return result +} + // objects function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) { @@ -93,5 +151,7 @@ module.exports = { getJSONMp4: getJSONMp4, getTrueFileName: getTrueFileName, fixVideoMetadataPerms: fixVideoMetadataPerms, + getDownloadedFilesByType: getDownloadedFilesByType, + recFindByExt: recFindByExt, File: File }