From c724a8019a8205e7f680e3f725b3881f4f2b9246 Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Mon, 16 Jan 2023 02:42:36 -0500 Subject: [PATCH] Improved DB tests, now both local and remote DB can be tested easily --- .vscode/settings.json | 5 +- backend/db.js | 8 +- backend/package.json | 1 + backend/test/tests.js | 286 +++++++++++++++++++++++------------------- 4 files changed, 169 insertions(+), 131 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 2425b6e..aa1390b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,8 @@ { "mochaExplorer.files": "backend/test/**/*.js", "mochaExplorer.cwd": "backend", - "mochaExplorer.globImplementation": "vscode" + "mochaExplorer.globImplementation": "vscode", + "mochaExplorer.env": { + "YTDL_MODE": "debug" + } } \ No newline at end of file diff --git a/backend/db.js b/backend/db.js index 26f0f7e..39301ca 100644 --- a/backend/db.js +++ b/backend/db.js @@ -2,6 +2,7 @@ var fs = require('fs-extra') var path = require('path') const { MongoClient } = require("mongodb"); const { uuid } = require('uuidv4'); +const _ = require('lodash'); const config_api = require('./config'); var utils = require('./utils') @@ -152,6 +153,7 @@ exports._connectToDB = async (custom_connection_string = null) => { await database.collection(table).createIndex(text_search); } }); + using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false) return true; } catch(err) { logger.error(err); @@ -578,7 +580,6 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => { exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => { // local db override if (using_local_db) { - if (replaceFilter) local_db.get(table).remove(replaceFilter).write(); local_db.get(table).push(doc).write(); return true; } @@ -1141,3 +1142,8 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => { }); return return_val; } + +// should only be used for tests +exports.setLocalDBMode = (mode) => { + using_local_db = mode; +} \ No newline at end of file diff --git a/backend/package.json b/backend/package.json index a1a3527..b53f7be 100644 --- a/backend/package.json +++ b/backend/package.json @@ -32,6 +32,7 @@ "fs-extra": "^9.0.0", "gotify": "^1.1.0", "jsonwebtoken": "^8.5.1", + "lodash": "^4.17.21", "lowdb": "^1.0.0", "md5": "^2.2.1", "merge-files": "^0.1.2", diff --git a/backend/test/tests.js b/backend/test/tests.js index aa23ad3..bb3e2c3 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -104,151 +104,179 @@ describe('Database', async function() { }); describe('Basic functions', async function() { - beforeEach(async function() { - await db_api.connectToDB(); - await db_api.removeAllRecords('test'); - }); - it('Add and read record', async function() { - this.timeout(120000); - await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined}); - const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null}); - assert(added_record['test_add'] === 'test'); - await db_api.removeRecord('test', {test_add: 'test'}); - }); - - it('Find duplicates by key', async function() { - const test_duplicates = [ - { - test: 'testing', - key: '1' - }, - { - test: 'testing', - key: '2' - }, - { - test: 'testing_missing', - key: '3' - }, - { - test: 'testing', - key: '4' - } - ]; - await db_api.insertRecordsIntoTable('test', test_duplicates); - const duplicates = await db_api.findDuplicatesByKey('test', 'test'); - console.log(duplicates); - }); - - it('Update record', async function() { - await db_api.insertRecordIntoTable('test', {test_update: 'test'}); - await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true}); - const updated_record = await db_api.getRecord('test', {test_update: 'test'}); - assert(updated_record['added_field']); - await db_api.removeRecord('test', {test_update: 'test'}); - }); - - it('Remove record', async function() { - await db_api.insertRecordIntoTable('test', {test_remove: 'test'}); - const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'}); - assert(delete_succeeded); - const deleted_record = await db_api.getRecord('test', {test_remove: 'test'}); - assert(!deleted_record); - }); - - it('Push to record array', async function() { - await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []}); - await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); - const record = await db_api.getRecord('test', {test: 'test'}); - assert(record); - assert(record['test_array'].length === 1); - }); - - it('Pull from record array', async function() { - await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']}); - await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); - const record = await db_api.getRecord('test', {test: 'test'}); - assert(record); - assert(record['test_array'].length === 0); - }); + + // test both local_db and remote_db + const local_db_modes = [false, true]; + + for (const local_db_mode of local_db_modes) { + let use_local_db = local_db_mode; + describe(`Use local DB - ${use_local_db}`, async function() { + beforeEach(async function() { + if (!use_local_db) { + this.timeout(120000); + await db_api.connectToDB(0); + } + await db_api.removeAllRecords('test'); + }); + it('Add and read record', async function() { + this.timeout(120000); + await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined}); + const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null}); + assert(added_record['test_add'] === 'test'); + await db_api.removeRecord('test', {test_add: 'test'}); + }); + it('Add and read record - Nested property', async function() { + this.timeout(120000); + await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}); + const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'}); + const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'}); + assert(added_record['test_add'] === 'test'); + assert(!not_added_record); + await db_api.removeRecord('test', {test_add: 'test'}); + }); + it('Replace filter', async function() { + this.timeout(120000); + await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}}); + await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}}); + const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true); + assert(count === 1); + await db_api.removeRecord('test', {test_replace_filter: 'test'}); + }); + it('Find duplicates by key', async function() { + const test_duplicates = [ + { + test: 'testing', + key: '1' + }, + { + test: 'testing', + key: '2' + }, + { + test: 'testing_missing', + key: '3' + }, + { + test: 'testing', + key: '4' + } + ]; + await db_api.insertRecordsIntoTable('test', test_duplicates); + const duplicates = await db_api.findDuplicatesByKey('test', 'test'); + console.log(duplicates); + }); - it('Bulk add', async function() { - this.timeout(120000); - const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000 - const test_records = []; - for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { - test_records.push({ - uid: uuid() + it('Update record', async function() { + await db_api.insertRecordIntoTable('test', {test_update: 'test'}); + await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true}); + const updated_record = await db_api.getRecord('test', {test_update: 'test'}); + assert(updated_record['added_field']); + await db_api.removeRecord('test', {test_update: 'test'}); }); - } - const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records); - const received_records = await db_api.getRecords('test'); - assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD); - }); + it('Remove record', async function() { + await db_api.insertRecordIntoTable('test', {test_remove: 'test'}); + const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'}); + assert(delete_succeeded); + const deleted_record = await db_api.getRecord('test', {test_remove: 'test'}); + assert(!deleted_record); + }); - it('Bulk update', async function() { - // bulk add records - const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000 - const test_records = []; - const update_obj = {}; - for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { - const test_uid = uuid(); - test_records.push({ - uid: test_uid + it('Push to record array', async function() { + await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []}); + await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); + const record = await db_api.getRecord('test', {test: 'test'}); + assert(record); + assert(record['test_array'].length === 1); }); - update_obj[test_uid] = {added_field: true}; - } - let success = await db_api.bulkInsertRecordsIntoTable('test', test_records); - assert(success); - // makes sure they are added - const received_records = await db_api.getRecords('test'); - assert(received_records && received_records.length === NUM_RECORDS_TO_ADD); + it('Pull from record array', async function() { + await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']}); + await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item'); + const record = await db_api.getRecord('test', {test: 'test'}); + assert(record); + assert(record['test_array'].length === 0); + }); - success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj); - assert(success); + it('Bulk add', async function() { + this.timeout(120000); + const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000 + const test_records = []; + for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { + test_records.push({ + uid: uuid() + }); + } + const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records); + + const received_records = await db_api.getRecords('test'); + assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD); + }); - const received_updated_records = await db_api.getRecords('test'); - for (let i = 0; i < received_updated_records.length; i++) { - success &= received_updated_records[i]['added_field']; - } - assert(success); - }); + it('Bulk update', async function() { + // bulk add records + const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000 + const test_records = []; + const update_obj = {}; + for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { + const test_uid = uuid(); + test_records.push({ + uid: test_uid + }); + update_obj[test_uid] = {added_field: true}; + } + let success = await db_api.bulkInsertRecordsIntoTable('test', test_records); + assert(success); + + // makes sure they are added + const received_records = await db_api.getRecords('test'); + assert(received_records && received_records.length === NUM_RECORDS_TO_ADD); + + success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj); + assert(success); + + const received_updated_records = await db_api.getRecords('test'); + for (let i = 0; i < received_updated_records.length; i++) { + success &= received_updated_records[i]['added_field']; + } + assert(success); + }); - it('Stats', async function() { - const stats = await db_api.getDBStats(); - assert(stats); - }); + it('Stats', async function() { + const stats = await db_api.getDBStats(); + assert(stats); + }); - it('Query speed', async function() { - this.timeout(120000); - const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000 - const test_records = []; - let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943'; - for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { - const uid = uuid(); - if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid; - test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632}); - } - const insert_start = Date.now(); - let success = await db_api.bulkInsertRecordsIntoTable('test', test_records); - const insert_end = Date.now(); + it('Query speed', async function() { + this.timeout(120000); + const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000 + const test_records = []; + let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943'; + for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) { + const uid = uuid(); + if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid; + test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632}); + } + const insert_start = Date.now(); + let success = await db_api.bulkInsertRecordsIntoTable('test', test_records); + const insert_end = Date.now(); - console.log(`Insert time: ${(insert_end - insert_start)/1000}s`); + console.log(`Insert time: ${(insert_end - insert_start)/1000}s`); - const query_start = Date.now(); - const random_record = await db_api.getRecord('test', {uid: random_uid}); - const query_end = Date.now(); + const query_start = Date.now(); + const random_record = await db_api.getRecord('test', {uid: random_uid}); + const query_end = Date.now(); - console.log(random_record) + console.log(random_record) - console.log(`Query time: ${(query_end - query_start)/1000}s`); + console.log(`Query time: ${(query_end - query_start)/1000}s`); - success = !!random_record; + success = !!random_record; - assert(success); - }); + assert(success); + }); + }); + } }); describe('Local DB Filters', async function() {