diff --git a/backend/app.js b/backend/app.js index d43f9bc..5578a95 100644 --- a/backend/app.js +++ b/backend/app.js @@ -13,6 +13,8 @@ var express = require("express"); var bodyParser = require("body-parser"); var archiver = require('archiver'); var unzipper = require('unzipper'); +var db_api = require('./db') +var utils = require('./utils') var mergeFiles = require('merge-files'); const low = require('lowdb') var ProgressBar = require('progress'); @@ -73,8 +75,9 @@ const logger = winston.createLogger({ }); config_api.initialize(logger); -subscriptions_api.initialize(db, users_db, logger); auth_api.initialize(users_db, logger); +db_api.initialize(db, users_db, logger); +subscriptions_api.initialize(db, users_db, logger, db_api); // var GithubContent = require('github-content'); @@ -191,27 +194,12 @@ app.use(bodyParser.json()); // use passport app.use(auth_api.passport.initialize()); -// objects - -function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) { - this.id = id; - this.title = title; - this.thumbnailURL = thumbnailURL; - this.isAudio = isAudio; - this.duration = duration; - this.url = url; - this.uploader = uploader; - this.size = size; - this.path = path; - this.upload_date = upload_date; -} - // actual functions async function checkMigrations() { return new Promise(async resolve => { // 3.5->3.6 migration - const files_to_db_migration_complete = db.get('files_to_db_migration_complete').value(); + const files_to_db_migration_complete = true; // migration phased out! previous code: db.get('files_to_db_migration_complete').value(); if (!files_to_db_migration_complete) { logger.info('Beginning migration: 3.5->3.6+') @@ -236,7 +224,7 @@ async function runFilesToDBMigration() { const file_already_in_db = db.get('files.audio').find({id: file_obj.id}).value(); if (!file_already_in_db) { logger.verbose(`Migrating file ${file_obj.id}`); - registerFileDB(file_obj.id + '.mp3', 'audio'); + db_api.registerFileDB(file_obj.id + '.mp3', 'audio'); } } @@ -245,7 +233,7 @@ async function runFilesToDBMigration() { const file_already_in_db = db.get('files.video').find({id: file_obj.id}).value(); if (!file_already_in_db) { logger.verbose(`Migrating file ${file_obj.id}`); - registerFileDB(file_obj.id + '.mp4', 'video'); + db_api.registerFileDB(file_obj.id + '.mp4', 'video'); } } @@ -654,6 +642,11 @@ async function watchSubscriptions() { continue; } + if (!sub.name) { + logger.verbose(`Subscription: skipped check for subscription with uid ${sub.id} as name has not been retrieved yet.`); + continue; + } + logger.verbose('Watching ' + sub.name + ' with delay interval of ' + delay_interval); setTimeout(async () => { await subscriptions_api.getVideosForSub(sub, sub.user_uid); @@ -700,7 +693,7 @@ function getMp3s() { var stats = fs.statSync(file); var id = file_path.substring(0, file_path.length-4); - var jsonobj = getJSONMp3(id); + var jsonobj = utils.getJSONMp3(id, audioFolderPath); if (!jsonobj) continue; var title = jsonobj.title; var url = jsonobj.webpage_url; @@ -713,7 +706,7 @@ function getMp3s() { var thumbnail = jsonobj.thumbnail; var duration = jsonobj.duration; var isaudio = true; - var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); mp3s.push(file_obj); } return mp3s; @@ -729,7 +722,7 @@ function getMp4s(relative_path = true) { var stats = fs.statSync(file); var id = file_path.substring(0, file_path.length-4); - var jsonobj = getJSONMp4(id); + var jsonobj = utils.getJSONMp4(id, videoFolderPath); if (!jsonobj) continue; var title = jsonobj.title; var url = jsonobj.webpage_url; @@ -742,7 +735,7 @@ function getMp4s(relative_path = true) { var size = stats.size; var isaudio = false; - var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); mp4s.push(file_obj); } return mp4s; @@ -750,14 +743,14 @@ function getMp4s(relative_path = true) { function getThumbnailMp3(name) { - var obj = getJSONMp3(name); + var obj = utils.getJSONMp3(name, audioFolderPath); var thumbnailLink = obj.thumbnail; return thumbnailLink; } function getThumbnailMp4(name) { - var obj = getJSONMp4(name); + var obj = utils.getJSONMp4(name, videoFolderPath); var thumbnailLink = obj.thumbnail; return thumbnailLink; } @@ -794,54 +787,6 @@ function getFileSizeMp4(name) return filesize; } -function getJSONMp3(name, customPath = null, openReadPerms = false) -{ - var jsonPath = audioFolderPath+name+".info.json"; - var alternateJsonPath = audioFolderPath+name+".mp3.info.json"; - if (!customPath) { - jsonPath = audioFolderPath + name + ".info.json"; - } else { - jsonPath = customPath + name + ".info.json"; - alternateJsonPath = customPath + name + ".mp3.info.json"; - } - var obj = null; - if (fs.existsSync(jsonPath)) { - obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - if (!is_windows && openReadPerms) fs.chmodSync(jsonPath, 0o755); - } - else if (fs.existsSync(alternateJsonPath)) { - obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8')); - if (!is_windows && openReadPerms) fs.chmodSync(alternateJsonPath, 0o755); - } - else - obj = 0; - - return obj; -} - -function getJSONMp4(name, customPath = null, openReadPerms = false) -{ - var obj = null; // output - let jsonPath = null; - var alternateJsonPath = videoFolderPath + name + ".mp4.info.json"; - if (!customPath) { - jsonPath = videoFolderPath + name + ".info.json"; - } else { - jsonPath = customPath + name + ".info.json"; - alternateJsonPath = customPath + name + ".mp4.info.json"; - } - if (fs.existsSync(jsonPath)) - { - obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); - if (openReadPerms) fs.chmodSync(jsonPath, 0o644); - } else if (fs.existsSync(alternateJsonPath)) { - obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8')); - if (openReadPerms) fs.chmodSync(alternateJsonPath, 0o644); - } - else obj = 0; - return obj; -} - function getAmountDownloadedMp3(name) { var partPath = audioFolderPath+name+".mp3.part"; @@ -965,7 +910,7 @@ async function deleteAudioFile(name, blacklistMode = false) { // get ID from JSON - var jsonobj = getJSONMp3(name); + var jsonobj = utils.getJSONMp3(name, audioFolderPath); let id = null; if (jsonobj) id = jsonobj.id; @@ -1023,7 +968,7 @@ async function deleteVideoFile(name, customPath = null, blacklistMode = false) { // get ID from JSON - var jsonobj = getJSONMp4(name); + var jsonobj = utils.getJSONMp4(name, videoFolderPath); let id = null; if (jsonobj) id = jsonobj.id; @@ -1077,7 +1022,7 @@ function recFindByExt(base,ext,files,result) ) return result } - +/* function registerFileDB(file_path, type, multiUserMode = null) { const file_id = file_path.substring(0, file_path.length-4); const file_object = generateFileObject(file_id, type, multiUserMode && multiUserMode.file_path); @@ -1112,7 +1057,7 @@ function registerFileDB(file_path, type, multiUserMode = null) { } function generateFileObject(id, type, customPath = null) { - var jsonobj = (type === 'audio') ? getJSONMp3(id, customPath, true) : getJSONMp4(id, customPath, true); + var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true); if (!jsonobj) { return null; } @@ -1132,10 +1077,10 @@ function generateFileObject(id, type, customPath = null) { var thumbnail = jsonobj.thumbnail; var duration = jsonobj.duration; var isaudio = type === 'audio'; - var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date); + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date); return file_obj; } - +*/ // replaces .webm with appropriate extension function getTrueFileName(unfixed_path, type) { let fixed_path = unfixed_path; @@ -1292,7 +1237,7 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { } // registers file in DB - file_uid = registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), type, multiUserMode); + file_uid = db_api.registerFileDB(full_file_path.substring(fileFolderPath.length, full_file_path.length), type, multiUserMode); if (file_name) file_names.push(file_name); } @@ -1431,7 +1376,7 @@ async function downloadFileByURL_normal(url, type, options, sessionID = null) { // registers file in DB const base_file_name = video_info._filename.substring(fileFolderPath.length, video_info._filename.length); - file_uid = registerFileDB(base_file_name, type, multiUserMode); + file_uid = db_api.registerFileDB(base_file_name, type, multiUserMode); if (options.merged_string !== null && options.merged_string !== undefined) { let current_merged_archive = fs.readFileSync(path.join(fileFolderPath, `merged_${type}.txt`), 'utf8'); @@ -2234,45 +2179,48 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { // get sub videos if (subscription.name && !subscription.streamingOnly) { - let base_path = null; - if (user_uid) - base_path = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); - else - base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); - - let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/'); - let files; - try { - files = recFindByExt(appended_base_path, 'mp4'); - } catch(e) { - files = null; - logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path); - res.sendStatus(500); - return; - } - var parsed_files = []; - for (let i = 0; i < files.length; i++) { - let file = files[i]; - var file_path = file.substring(appended_base_path.length, file.length); - var stats = fs.statSync(file); - - var id = file_path.substring(0, file_path.length-4); - var jsonobj = getJSONMp4(id, appended_base_path); - if (!jsonobj) continue; - var title = jsonobj.title; - - var thumbnail = jsonobj.thumbnail; - var duration = jsonobj.duration; - var url = jsonobj.webpage_url; - var uploader = jsonobj.uploader; - var upload_date = jsonobj.upload_date; - upload_date = `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}`; - var size = stats.size; - - var isaudio = false; - var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); - parsed_files.push(file_obj); + var parsed_files = subscription.videos; + if (!parsed_files) { + let base_path = null; + if (user_uid) + base_path = path.join(config_api.getConfigItem('ytdl_users_base_path'), user_uid, 'subscriptions'); + else + base_path = config_api.getConfigItem('ytdl_subscriptions_base_path'); + + let appended_base_path = path.join(base_path, (subscription.isPlaylist ? 'playlists' : 'channels'), subscription.name, '/'); + let files; + try { + files = recFindByExt(appended_base_path, 'mp4'); + } catch(e) { + files = null; + logger.info('Failed to get folder for subscription: ' + subscription.name + ' at path ' + appended_base_path); + res.sendStatus(500); + return; + } + for (let i = 0; i < files.length; i++) { + let file = files[i]; + var file_path = file.substring(appended_base_path.length, file.length); + var stats = fs.statSync(file); + + var id = file_path.substring(0, file_path.length-4); + var jsonobj = utils.getJSONMp4(id, appended_base_path); + if (!jsonobj) continue; + var title = jsonobj.title; + + var thumbnail = jsonobj.thumbnail; + var duration = jsonobj.duration; + var url = jsonobj.webpage_url; + var uploader = jsonobj.uploader; + var upload_date = jsonobj.upload_date; + upload_date = `${upload_date.substring(0, 4)}-${upload_date.substring(4, 6)}-${upload_date.substring(6, 8)}`; + var size = stats.size; + + var isaudio = false; + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file, upload_date); + parsed_files.push(file_obj); + } } + res.send({ subscription: subscription, @@ -2284,7 +2232,7 @@ app.post('/api/getSubscription', optionalJwt, async (req, res) => { if (subscription.videos) { for (let i = 0; i < subscription.videos.length; i++) { const video = subscription.videos[i]; - parsed_files.push(new File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date)); + parsed_files.push(new utils.File(video.title, video.title, video.thumbnail, false, video.duration, video.url, video.uploader, video.size, null, null, video.upload_date)); } } res.send({ diff --git a/backend/db.js b/backend/db.js index f612d24..104d343 100644 --- a/backend/db.js +++ b/backend/db.js @@ -1,3 +1,8 @@ +var fs = require('fs-extra') +var path = require('path') +var utils = require('./utils') +const { uuid } = require('uuidv4'); + var logger = null; var db = null; var users_db = null; @@ -23,8 +28,8 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) { path_object = path.parse(file_object['path']); file_object['path'] = path.format(path_object); - if (multiUserMode) { - if (!sub) { + if (!sub) { + if (multiUserMode) { const user_uid = multiUserMode.user; users_db.get('users').find({uid: user_uid}).get(`files.${type}`) .remove({ @@ -35,36 +40,37 @@ function registerFileDB(file_path, type, multiUserMode = null, sub = null) { .push(file_object) .write(); } else { - - } - } else if (!sub) { - // remove existing video if overwriting - db.get(`files.${type}`) - .remove({ - path: file_object['path'] - }).write(); + // remove existing video if overwriting + db.get(`files.${type}`) + .remove({ + path: file_object['path'] + }).write(); - db.get(`files.${type}`) - .push(file_object) - .write(); - } else if (sub) { - if (multi) + db.get(`files.${type}`) + .push(file_object) + .write(); + } } else { - // this should never be used - logger.error('Failed to determine file type during video DB registration.'); - return null; + sub_db = null; + if (multiUserMode) { + const user_uid = multiUserMode.user; + sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); + } else { + sub_db = db.get('subscriptions').find({id: sub.id}); + } + sub_db.get('videos').push(file_object).write(); } return file_object['uid']; } function generateFileObject(id, type, customPath = null) { - var jsonobj = (type === 'audio') ? getJSONMp3(id, customPath, true) : getJSONMp4(id, customPath, true); + var jsonobj = (type === 'audio') ? utils.getJSONMp3(id, customPath, true) : utils.getJSONMp4(id, customPath, true); if (!jsonobj) { return null; } const ext = (type === 'audio') ? '.mp3' : '.mp4' - const file_path = getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext); + const file_path = utils.getTrueFileName(jsonobj['_filename'], type); // path.join(type === 'audio' ? audioFolderPath : videoFolderPath, id + ext); // console. var stats = fs.statSync(path.join(__dirname, file_path)); @@ -79,7 +85,7 @@ function generateFileObject(id, type, customPath = null) { var thumbnail = jsonobj.thumbnail; var duration = jsonobj.duration; var isaudio = type === 'audio'; - var file_obj = new File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date); + var file_obj = new utils.File(id, title, thumbnail, isaudio, duration, url, uploader, size, file_path, upload_date); return file_obj; } diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 01cacb8..6e80c8b 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -6,17 +6,20 @@ var path = require('path'); var youtubedl = require('youtube-dl'); const config_api = require('./config'); +var utils = require('./utils') const debugMode = process.env.YTDL_MODE === 'debug'; var logger = null; var db = null; var users_db = null; -function setDB(input_db, input_users_db) { db = input_db; users_db = input_users_db } +var db_api = null; + +function setDB(input_db, input_users_db, input_db_api) { db = input_db; users_db = input_users_db; db_api = input_db_api } function setLogger(input_logger) { logger = input_logger; } -function initialize(input_db, input_users_db, input_logger) { - setDB(input_db, input_users_db); +function initialize(input_db, input_users_db, input_logger, input_db_api) { + setDB(input_db, input_users_db, input_db_api); setLogger(input_logger); } @@ -28,6 +31,8 @@ async function subscribe(sub, user_uid = null) { return new Promise(async resolve => { // sub should just have url and name. here we will get isPlaylist and path sub.isPlaylist = sub.url.includes('playlist'); + sub.type = 'video'; // TODO: eventually change + sub.videos = []; let url_exists = false; @@ -44,15 +49,25 @@ async function subscribe(sub, user_uid = null) { } // add sub to db - if (user_uid) + let sub_db = null; + if (user_uid) { users_db.get('users').find({uid: user_uid}).get('subscriptions').push(sub).write(); - else + sub_db = users_db.get('users').find({uid: user_uid}).get('subscriptions').find({id: sub.id}); + } else { db.get('subscriptions').push(sub).write(); - + sub_db = db.get('subscriptions').find({id: sub.id}); + } let success = await getSubscriptionInfo(sub, user_uid); + + if (success) { + sub = sub_db.get().value(); + getVideosForSub(sub, user_uid); + } else { + logger.error('Subscribe: Failed to get subscription info. Subscribe failed.') + }; + result_obj.success = success; result_obj.sub = sub; - getVideosForSub(sub, user_uid); resolve(result_obj); }); @@ -237,10 +252,14 @@ async function getVideosForSub(sub, user_uid = null) { const useArchive = config_api.getConfigItem('ytdl_subscriptions_use_youtubedl_archive'); let appendedBasePath = null - if (sub.name) { - appendedBasePath = getAppendedBasePath(sub, basePath); - } else { - appendedBasePath = path.join(basePath, (sub.isPlaylist ? 'playlists/%(playlist_title)s' : 'channels/%(uploader)s')); + appendedBasePath = getAppendedBasePath(sub, basePath); + + let multiUserMode = null; + if (user_uid) { + multiUserMode = { + user: user_uid, + file_path: appendedBasePath + } } let downloadConfig = ['-o', appendedBasePath + '/%(title)s.mp4', '-f', 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4', '-ciw', '--write-info-json', '--print-json']; @@ -286,7 +305,7 @@ async function getVideosForSub(sub, user_uid = null) { const outputs = err.stdout.split(/\r\n|\r|\n/); for (let i = 0; i < outputs.length; i++) { const output = JSON.parse(outputs[i]); - handleOutputJSON(sub, sub_db, output, i === 0) + handleOutputJSON(sub, sub_db, output, i === 0, multiUserMode) if (err.stderr.includes(output['id']) && archive_path) { // we found a video that errored! add it to the archive to prevent future errors fs.appendFileSync(archive_path, output['id']); @@ -315,7 +334,7 @@ async function getVideosForSub(sub, user_uid = null) { } const reset_videos = i === 0; - handleOutputJSON(sub, sub_db, output_json, reset_videos); + handleOutputJSON(sub, sub_db, output_json, multiUserMode, reset_videos); // TODO: Potentially store downloaded files in db? @@ -328,7 +347,7 @@ async function getVideosForSub(sub, user_uid = null) { }); } -function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) { +function handleOutputJSON(sub, sub_db, output_json, multiUserMode = null, reset_videos = false) { if (sub.streamingOnly) { if (reset_videos) { sub_db.assign({videos: []}).write(); @@ -341,7 +360,7 @@ function handleOutputJSON(sub, sub_db, output_json, reset_videos = false) { sub_db.get('videos').push(output_json).write(); } else { // TODO: make multiUserMode obj - db_api.registerFileDB(output_json['_filename'], sub.type, multiUserMode, sub); + db_api.registerFileDB(path.basename(output_json['_filename']), sub.type, multiUserMode, sub); } } diff --git a/backend/utils.js b/backend/utils.js new file mode 100644 index 0000000..4bef941 --- /dev/null +++ b/backend/utils.js @@ -0,0 +1,76 @@ +var fs = require('fs-extra') +var path = require('path') +const config_api = require('./config'); + +function getTrueFileName(unfixed_path, type) { + let fixed_path = unfixed_path; + + const new_ext = (type === 'audio' ? 'mp3' : 'mp4'); + let unfixed_parts = unfixed_path.split('.'); + const old_ext = unfixed_parts[unfixed_parts.length-1]; + + + if (old_ext !== new_ext) { + unfixed_parts[unfixed_parts.length-1] = new_ext; + fixed_path = unfixed_parts.join('.'); + } + return fixed_path; +} + +function getJSONMp4(name, customPath, openReadPerms = false) { + var obj = null; // output + if (!customPath) customPath = config_api.getConfigItem('ytdl_video_folder_path'); + var jsonPath = path.join(customPath, name + ".info.json"); + var alternateJsonPath = path.join(customPath, name + ".mp4.info.json"); + if (fs.existsSync(jsonPath)) + { + obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); + if (openReadPerms) fs.chmodSync(jsonPath, 0o644); + } else if (fs.existsSync(alternateJsonPath)) { + obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8')); + if (openReadPerms) fs.chmodSync(alternateJsonPath, 0o644); + } + else obj = 0; + return obj; +} + +function getJSONMp3(name, customPath, openReadPerms = false) { + var obj = null; + if (!customPath) customPath = config_api.getConfigItem('ytdl_audio_folder_path'); + var jsonPath = customPath + name + ".info.json"; + var alternateJsonPath = customPath + name + ".mp3.info.json"; + if (fs.existsSync(jsonPath)) { + obj = JSON.parse(fs.readFileSync(jsonPath, 'utf8')); + if (!is_windows && openReadPerms) fs.chmodSync(jsonPath, 0o755); + } + else if (fs.existsSync(alternateJsonPath)) { + obj = JSON.parse(fs.readFileSync(alternateJsonPath, 'utf8')); + if (!is_windows && openReadPerms) fs.chmodSync(alternateJsonPath, 0o755); + } + else + obj = 0; + + return obj; +} + +// objects + +function File(id, title, thumbnailURL, isAudio, duration, url, uploader, size, path, upload_date) { + this.id = id; + this.title = title; + this.thumbnailURL = thumbnailURL; + this.isAudio = isAudio; + this.duration = duration; + this.url = url; + this.uploader = uploader; + this.size = size; + this.path = path; + this.upload_date = upload_date; +} + +module.exports = { + getJSONMp3: getJSONMp3, + getJSONMp4: getJSONMp4, + getTrueFileName: getTrueFileName, + File: File +} \ No newline at end of file