From 20cedb6c29f46256d53e314c26c1f9e0c74b9211 Mon Sep 17 00:00:00 2001 From: Isaac Abadi Date: Sun, 22 Aug 2021 22:31:01 -0600 Subject: [PATCH] Pagination and filtering of files is now server-side Importing unregistered files does not block server start anymore --- backend/app.js | 26 +++++- backend/db.js | 45 ++++++++-- backend/utils.js | 21 +++++ .../custom-playlists.component.ts | 2 +- .../recent-videos.component.html | 4 +- .../recent-videos/recent-videos.component.ts | 86 ++++++++----------- src/app/posts.services.ts | 4 +- 7 files changed, 125 insertions(+), 63 deletions(-) diff --git a/backend/app.js b/backend/app.js index a9025e9..7ae59ae 100644 --- a/backend/app.js +++ b/backend/app.js @@ -631,7 +631,7 @@ async function loadConfig() { watchSubscriptionsInterval(); } - await db_api.importUnregisteredFiles(); + db_api.importUnregisteredFiles(); // load in previous downloads downloads = await db_api.getRecords('downloads'); @@ -1632,9 +1632,23 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { // these are returned let files = null; let playlists = null; + let sort = req.body.sort; + let range = req.body.range; + let text_search = req.body.text_search; const uuid = req.isAuthenticated() ? req.user.uid : null; - files = await db_api.getRecords('files', {user_uid: uuid}); + const filter_obj = {user_uid: uuid}; + const regex = true; + if (text_search) { + if (regex) { + filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'}; + } else { + filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) }; + } + } + + files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search); + let file_count = await db_api.getRecords('files', filter_obj, true); playlists = await db_api.getRecords('playlists', {user_uid: uuid}); const categories = await categories_api.getCategoriesAsPlaylists(files); @@ -1646,6 +1660,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { res.send({ files: files, + file_count: file_count, playlists: playlists }); }); @@ -2116,8 +2131,15 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => { app.post('/api/getPlaylists', optionalJwt, async (req, res) => { const uuid = req.isAuthenticated() ? req.user.uid : null; + const include_categories = req.body.include_categories; const playlists = await db_api.getRecords('playlists', {user_uid: uuid}); + if (include_categories) { + const categories = await categories_api.getCategoriesAsPlaylists(files); + if (categories) { + playlists = playlists.concat(categories); + } + } res.send({ playlists: playlists diff --git a/backend/db.js b/backend/db.js index 9442150..82a5dd4 100644 --- a/backend/db.js +++ b/backend/db.js @@ -18,7 +18,12 @@ var database = null; const tables = { files: { name: 'files', - primary_key: 'uid' + primary_key: 'uid', + text_search: { + title: 'text', + uploader: 'text', + uid: 'text' + } }, playlists: { name: 'playlists', @@ -131,8 +136,13 @@ exports._connectToDB = async (custom_connection_string = null) => { tables_list.forEach(async table => { const primary_key = tables[table]['primary_key']; - if (!primary_key) return; - await database.collection(table).createIndex({[primary_key]: 1}, { unique: true }); + if (primary_key) { + await database.collection(table).createIndex({[primary_key]: 1}, { unique: true }); + } + const text_search = tables[table]['text_search']; + if (text_search) { + await database.collection(table).createIndex(text_search); + } }); return true; } catch(err) { @@ -695,13 +705,28 @@ exports.getRecord = async (table, filter_obj) => { return await database.collection(table).findOne(filter_obj); } -exports.getRecords = async (table, filter_obj = null) => { +exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => { // local db override if (using_local_db) { - return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value(); + let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value(); + if (sort) { + cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1)); + } + if (range) { + cursor = cursor.slice(range[0], range[1]); + } + return !return_count ? cursor : cursor.length; } - return filter_obj ? await database.collection(table).find(filter_obj).toArray() : await database.collection(table).find().toArray(); + const cursor = filter_obj ? database.collection(table).find(filter_obj) : database.collection(table).find(); + if (sort) { + cursor.sort({[sort['by']]: sort['order']}); + } + if (range) { + cursor.skip(range[0]).limit(range[1] - range[0]); + } + + return !return_count ? await cursor.toArray() : await cursor.count(); } // Update @@ -1027,7 +1052,13 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => { if (filter_prop_value === undefined || filter_prop_value === null) { filtered &= record[filter_prop] === undefined || record[filter_prop] === null } else { - filtered &= record[filter_prop] === filter_prop_value; + if (typeof filter_prop_value === 'object') { + if (filter_prop_value['$regex']) { + filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1); + } + } else { + filtered &= record[filter_prop] === filter_prop_value; + } } } return filtered; diff --git a/backend/utils.js b/backend/utils.js index 9370cf4..d118230 100644 --- a/backend/utils.js +++ b/backend/utils.js @@ -349,6 +349,26 @@ function removeFileExtension(filename) { return filename_parts.join('.'); } +function createEdgeNGrams(str) { + if (str && str.length > 3) { + const minGram = 3 + const maxGram = str.length + + return str.split(" ").reduce((ngrams, token) => { + if (token.length > minGram) { + for (let i = minGram; i <= maxGram && i <= token.length; ++i) { + ngrams = [...ngrams, token.substr(0, i)] + } + } else { + ngrams = [...ngrams, token] + } + return ngrams + }, []).join(" ") + } + + return str +} + /** * setTimeout, but its a promise. * @param {number} ms @@ -399,6 +419,7 @@ module.exports = { getCurrentDownloader: getCurrentDownloader, recFindByExt: recFindByExt, removeFileExtension: removeFileExtension, + createEdgeNGrams: createEdgeNGrams, wait: wait, File: File } diff --git a/src/app/components/custom-playlists/custom-playlists.component.ts b/src/app/components/custom-playlists/custom-playlists.component.ts index 98819e1..7193123 100644 --- a/src/app/components/custom-playlists/custom-playlists.component.ts +++ b/src/app/components/custom-playlists/custom-playlists.component.ts @@ -35,7 +35,7 @@ export class CustomPlaylistsComponent implements OnInit { getAllPlaylists() { this.playlists_received = false; // must call getAllFiles as we need to get category playlists as well - this.postsService.getAllFiles().subscribe(res => { + this.postsService.getPlaylists().subscribe(res => { this.playlists = res['playlists']; this.playlists_received = true; }); diff --git a/src/app/components/recent-videos/recent-videos.component.html b/src/app/components/recent-videos/recent-videos.component.html index 254082c..b56b0bc 100644 --- a/src/app/components/recent-videos/recent-videos.component.html +++ b/src/app/components/recent-videos/recent-videos.component.html @@ -34,7 +34,7 @@
-
+
No videos found.
@@ -46,7 +46,7 @@
- diff --git a/src/app/components/recent-videos/recent-videos.component.ts b/src/app/components/recent-videos/recent-videos.component.ts index d795e29..4f7687f 100644 --- a/src/app/components/recent-videos/recent-videos.component.ts +++ b/src/app/components/recent-videos/recent-videos.component.ts @@ -2,6 +2,8 @@ import { Component, OnInit, ViewChild } from '@angular/core'; import { PostsService } from 'app/posts.services'; import { Router } from '@angular/router'; import { MatPaginator } from '@angular/material/paginator'; +import { Subject } from 'rxjs'; +import { distinctUntilChanged } from 'rxjs/operators'; @Component({ selector: 'app-recent-videos', @@ -15,8 +17,8 @@ export class RecentVideosComponent implements OnInit { normal_files_received = false; subscription_files_received = false; - files: any[] = null; - filtered_files: any[] = null; + file_count = 10; + searchChangedSubject: Subject = new Subject(); downloading_content = {'video': {}, 'audio': {}}; search_mode = false; search_text = ''; @@ -97,6 +99,18 @@ export class RecentVideosComponent implements OnInit { if (cached_filter_property && this.filterProperties[cached_filter_property]) { this.filterProperty = this.filterProperties[cached_filter_property]; } + + this.searchChangedSubject + .debounceTime(500) + .pipe(distinctUntilChanged() + ).subscribe(model => { + if (model.length > 0) { + this.search_mode = true; + } else { + this.search_mode = false; + } + this.getAllFiles(); + }); } getAllPlaylists() { @@ -108,64 +122,40 @@ export class RecentVideosComponent implements OnInit { // search onSearchInputChanged(newvalue) { - if (newvalue.length > 0) { - this.search_mode = true; - this.filterFiles(newvalue); - } else { - this.search_mode = false; - this.filtered_files = this.files; - } - } - - private filterFiles(value: string) { - const filterValue = value.toLowerCase(); - this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue) || option.category?.name?.toLowerCase().includes(filterValue)); - this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex}); - } - - filterByProperty(prop) { - if (this.descendingMode) { - this.filtered_files = this.filtered_files.sort((a, b) => (a[prop] > b[prop] ? -1 : 1)); - } else { - this.filtered_files = this.filtered_files.sort((a, b) => (a[prop] > b[prop] ? 1 : -1)); - } - if (this.paginator) { this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex}) }; + this.normal_files_received = false; + this.searchChangedSubject.next(newvalue); } filterOptionChanged(value) { - this.filterByProperty(value['property']); localStorage.setItem('filter_property', value['key']); + this.getAllFiles(); } toggleModeChange() { this.descendingMode = !this.descendingMode; - this.filterByProperty(this.filterProperty['property']); + this.getAllFiles(); } // get files - getAllFiles() { - this.normal_files_received = false; - this.postsService.getAllFiles().subscribe(res => { - this.files = res['files']; - this.files.sort(this.sortFiles); - for (let i = 0; i < this.files.length; i++) { - const file = this.files[i]; + getAllFiles(cache_mode = false) { + this.normal_files_received = cache_mode; + const current_file_index = (this.paginator?.pageIndex ? this.paginator.pageIndex : 0)*this.pageSize; + const sort = {by: this.filterProperty['property'], order: this.descendingMode ? -1 : 1}; + const range = [current_file_index, current_file_index + this.pageSize]; + this.postsService.getAllFiles(sort, range, this.search_mode ? this.search_text : null).subscribe(res => { + this.file_count = res['file_count']; + this.paged_data = res['files']; + for (let i = 0; i < this.paged_data.length; i++) { + const file = this.paged_data[i]; file.duration = typeof file.duration !== 'string' ? file.duration : this.durationStringToNumber(file.duration); } - if (this.search_mode) { - this.filterFiles(this.search_text); - } else { - this.filtered_files = this.files; - } - this.filterByProperty(this.filterProperty['property']); // set cached file count for future use, note that we convert the amount of files to a string - localStorage.setItem('cached_file_count', '' + this.files.length); + localStorage.setItem('cached_file_count', '' + this.file_count); this.normal_files_received = true; - this.paged_data = this.filtered_files.slice(0, 10); }); } @@ -301,12 +291,9 @@ export class RecentVideosComponent implements OnInit { } removeFileCard(file_to_remove) { - const index = this.files.map(e => e.uid).indexOf(file_to_remove.uid); - this.files.splice(index, 1); - if (this.search_mode) { - this.filterFiles(this.search_text); - } - this.filterByProperty(this.filterProperty['property']); + const index = this.paged_data.map(e => e.uid).indexOf(file_to_remove.uid); + this.paged_data.splice(index, 1); + this.getAllFiles(true); } addFileToPlaylist(info_obj) { @@ -344,7 +331,8 @@ export class RecentVideosComponent implements OnInit { } pageChangeEvent(event) { - const offset = ((event.pageIndex + 1) - 1) * event.pageSize; - this.paged_data = this.filtered_files.slice(offset).slice(0, event.pageSize); + this.pageSize = event.pageSize; + this.loading_files = Array(this.pageSize).fill(0); + this.getAllFiles(); } } diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 0118ac5..a7c35a9 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -239,8 +239,8 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'getFile', {uid: uid, type: type, uuid: uuid}, this.httpOptions); } - getAllFiles() { - return this.http.post(this.path + 'getAllFiles', {}, this.httpOptions); + getAllFiles(sort, range, text_search) { + return this.http.post(this.path + 'getAllFiles', {sort: sort, range: range, text_search: text_search}, this.httpOptions); } getFullTwitchChat(id, type, uuid = null, sub = null) {