From 6eb1e2f898ad5f1a6a9c506aad382ceefab3cab3 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 22 Aug 2021 23:06:16 -0600 Subject: [PATCH 1/2] Fixed issue where different path formatting would lead files to get duplicated in the DB --- backend/db.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/db.js b/backend/db.js index 82a5dd4..415e213 100644 --- a/backend/db.js +++ b/backend/db.js @@ -375,7 +375,8 @@ exports.importUnregisteredFiles = async () => { const file = files[j]; // check if file exists in db, if not add it - const file_is_registered = !!(await exports.getRecord('files', {id: file.id, sub_id: dir_to_check.sub_id})) + const files_with_same_url = await exports.getRecords('files', {url: file.url, sub_id: dir_to_check.sub_id}); + const file_is_registered = !!(files_with_same_url.find(file_with_same_url => path.resolve(file_with_same_url.path) === path.resolve(file.path))); if (!file_is_registered) { // add additional info await exports.registerFileDB2(file['path'], dir_to_check.type, dir_to_check.user_uid, null, dir_to_check.sub_id, null); From f9b1414460ed6cc4b25e524e1bfc7e3d97767c80 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Mon, 23 Aug 2021 20:18:28 -0600 Subject: [PATCH 2/2] Logic to avoid duplicates for subscription files now uses the video URL instead of its path --- backend/subscriptions.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/subscriptions.js b/backend/subscriptions.js index 2f7416e..457636e 100644 --- a/backend/subscriptions.js +++ b/backend/subscriptions.js @@ -463,12 +463,10 @@ async function handleOutputJSON(sub, output_json, multiUserMode = null, reset_vi // add to db sub_db.get('videos').push(output_json).write(); } else { - path_object = path.parse(output_json['_filename']); - const path_string = path.format(path_object); + const file_url = output_json['webpage_url'] - const file_exists = await db_api.getRecord('files', {path: path_string, sub_id: sub.id}); + const file_exists = await db_api.getRecord('files', {url: file_url, sub_id: sub.id}); if (file_exists) { - // TODO: fix issue where files of different paths due to custom path get downloaded multiple times // file already exists in DB, return early to avoid reseting the download date return; }