aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improved DB tests, now both local and remote DB can be tested easily

Browse Source
pull/809/head
Tzahi12345 3 years ago
parent f20a31ed0f
commit c724a8019a
4 changed files with 169 additions and 131 deletions
Show Stats Download Patch File Download Diff File

5
.vscode/settings.json vendored
Unescape Escape View File

@ -1,5 +1,8 @@
{
"mochaExplorer.files": "backend/test/**/*.js",
"mochaExplorer.cwd": "backend",
"mochaExplorer.globImplementation": "vscode"
"mochaExplorer.globImplementation": "vscode",
"mochaExplorer.env": {
"YTDL_MODE": "debug"
}
}

8
backend/db.js
Unescape Escape View File

@ -2,6 +2,7 @@ var fs = require('fs-extra')
var path = require('path')
const { MongoClient } = require("mongodb");
const { uuid } = require('uuidv4');
const _ = require('lodash');
const config_api = require('./config');
var utils = require('./utils')
@ -152,6 +153,7 @@ exports._connectToDB = async (custom_connection_string = null) => {
await database.collection(table).createIndex(text_search);
}
});
using_local_db = false; // needs to happen for tests (in normal operation using_local_db is guaranteed false)
return true;
} catch(err) {
logger.error(err);
@ -578,7 +580,6 @@ exports.setVideoProperty = async (file_uid, assignment_obj) => {
exports.insertRecordIntoTable = async (table, doc, replaceFilter = null) => {
// local db override
if (using_local_db) {
if (replaceFilter) local_db.get(table).remove(replaceFilter).write();
local_db.get(table).push(doc).write();
return true;
}
@ -1141,3 +1142,8 @@ exports.applyFilterLocalDB = (db_path, filter_obj, operation) => {
});
return return_val;
}
// should only be used for tests
exports.setLocalDBMode = (mode) => {
using_local_db = mode;
}

1
backend/package.json
Unescape Escape View File

@ -32,6 +32,7 @@
"fs-extra": "^9.0.0",
"gotify": "^1.1.0",
"jsonwebtoken": "^8.5.1",
"lodash": "^4.17.21",
"lowdb": "^1.0.0",
"md5": "^2.2.1",
"merge-files": "^0.1.2",

286
backend/test/tests.js
Unescape Escape View File

@ -104,151 +104,179 @@ describe('Database', async function() {
});
describe('Basic functions', async function() {
beforeEach(async function() {
await db_api.connectToDB();
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
});
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
// test both local_db and remote_db
const local_db_modes = [false, true];
for (const local_db_mode of local_db_modes) {
let use_local_db = local_db_mode;
describe(`Use local DB - ${use_local_db}`, async function() {
beforeEach(async function() {
if (!use_local_db) {
this.timeout(120000);
await db_api.connectToDB(0);
}
await db_api.removeAllRecords('test');
});
it('Add and read record', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_undefined: undefined, test_null: undefined});
const added_record = await db_api.getRecord('test', {test_add: 'test', test_undefined: undefined, test_null: null});
assert(added_record['test_add'] === 'test');
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Add and read record - Nested property', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_add: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}});
const added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test2'});
const not_added_record = await db_api.getRecord('test', {test_add: 'test', 'test_nested.test_key1': 'test1', 'test_nested.test_key2': 'test3'});
assert(added_record['test_add'] === 'test');
assert(!not_added_record);
await db_api.removeRecord('test', {test_add: 'test'});
});
it('Replace filter', async function() {
this.timeout(120000);
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
await db_api.insertRecordIntoTable('test', {test_replace_filter: 'test', test_nested: {test_key1: 'test1', test_key2: 'test2'}}, {test_nested: {test_key1: 'test1', test_key2: 'test2'}});
const count = await db_api.getRecords('test', {test_replace_filter: 'test'}, true);
assert(count === 1);
await db_api.removeRecord('test', {test_replace_filter: 'test'});
});
it('Find duplicates by key', async function() {
const test_duplicates = [
{
test: 'testing',
key: '1'
},
{
test: 'testing',
key: '2'
},
{
test: 'testing_missing',
key: '3'
},
{
test: 'testing',
key: '4'
}
];
await db_api.insertRecordsIntoTable('test', test_duplicates);
const duplicates = await db_api.findDuplicatesByKey('test', 'test');
console.log(duplicates);
});
it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
it('Update record', async function() {
await db_api.insertRecordIntoTable('test', {test_update: 'test'});
await db_api.updateRecord('test', {test_update: 'test'}, {added_field: true});
const updated_record = await db_api.getRecord('test', {test_update: 'test'});
assert(updated_record['added_field']);
await db_api.removeRecord('test', {test_update: 'test'});
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
it('Remove record', async function() {
await db_api.insertRecordIntoTable('test', {test_remove: 'test'});
const delete_succeeded = await db_api.removeRecord('test', {test_remove: 'test'});
assert(delete_succeeded);
const deleted_record = await db_api.getRecord('test', {test_remove: 'test'});
assert(!deleted_record);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
it('Push to record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: []});
await db_api.pushToRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 1);
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
it('Pull from record array', async function() {
await db_api.insertRecordIntoTable('test', {test: 'test', test_array: ['test_item']});
await db_api.pullFromRecordsArray('test', {test: 'test'}, 'test_array', 'test_item');
const record = await db_api.getRecord('test', {test: 'test'});
assert(record);
assert(record['test_array'].length === 0);
});
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
assert(success);
it('Bulk add', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 2002; // max batch ops is 1000
const test_records = [];
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
test_records.push({
uid: uuid()
});
}
const succcess = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const received_records = await db_api.getRecords('test');
assert(succcess && received_records && received_records.length === NUM_RECORDS_TO_ADD);
});
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Bulk update', async function() {
// bulk add records
const NUM_RECORDS_TO_ADD = 100; // max batch ops is 1000
const test_records = [];
const update_obj = {};
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const test_uid = uuid();
test_records.push({
uid: test_uid
});
update_obj[test_uid] = {added_field: true};
}
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
assert(success);
// makes sure they are added
const received_records = await db_api.getRecords('test');
assert(received_records && received_records.length === NUM_RECORDS_TO_ADD);
success = await db_api.bulkUpdateRecordsByKey('test', 'uid', update_obj);
assert(success);
const received_updated_records = await db_api.getRecords('test');
for (let i = 0; i < received_updated_records.length; i++) {
success &= received_updated_records[i]['added_field'];
}
assert(success);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Stats', async function() {
const stats = await db_api.getDBStats();
assert(stats);
});
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
it('Query speed', async function() {
this.timeout(120000);
const NUM_RECORDS_TO_ADD = 300004; // max batch ops is 1000
const test_records = [];
let random_uid = '06241f83-d1b8-4465-812c-618dfa7f2943';
for (let i = 0; i < NUM_RECORDS_TO_ADD; i++) {
const uid = uuid();
if (i === NUM_RECORDS_TO_ADD/2) random_uid = uid;
test_records.push({"id":"RandomTextRandomText","title":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","thumbnailURL":"https://i.ytimg.com/vi/randomurl/maxresdefault.jpg","isAudio":true,"duration":312,"url":"https://www.youtube.com/watch?v=randomvideo","uploader":"randomUploader","size":5060157,"path":"audio\\RandomTextRandomText.mp3","upload_date":"2016-05-11","description":"RandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomTextRandomText","view_count":118689353,"height":null,"abr":160,"uid": uid,"registered":1626672120632});
}
const insert_start = Date.now();
let success = await db_api.bulkInsertRecordsIntoTable('test', test_records);
const insert_end = Date.now();
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
console.log(`Insert time: ${(insert_end - insert_start)/1000}s`);
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
const query_start = Date.now();
const random_record = await db_api.getRecord('test', {uid: random_uid});
const query_end = Date.now();
console.log(random_record)
console.log(random_record)
console.log(`Query time: ${(query_end - query_start)/1000}s`);
console.log(`Query time: ${(query_end - query_start)/1000}s`);
success = !!random_record;
success = !!random_record;
assert(success);
});
assert(success);
});
});
}
});
describe('Local DB Filters', async function() {

Write Preview
Loading…
Cancel
Save