Pagination and filtering of files is now server-side

Importing unregistered files does not block server start anymore
pull/443/head
Isaac Abadi 4 years ago
parent 5321624604
commit 20cedb6c29

@ -631,7 +631,7 @@ async function loadConfig() {
watchSubscriptionsInterval();
}
await db_api.importUnregisteredFiles();
db_api.importUnregisteredFiles();
// load in previous downloads
downloads = await db_api.getRecords('downloads');
@ -1632,9 +1632,23 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
// these are returned
let files = null;
let playlists = null;
let sort = req.body.sort;
let range = req.body.range;
let text_search = req.body.text_search;
const uuid = req.isAuthenticated() ? req.user.uid : null;
files = await db_api.getRecords('files', {user_uid: uuid});
const filter_obj = {user_uid: uuid};
const regex = true;
if (text_search) {
if (regex) {
filter_obj['title'] = {$regex: `.*${text_search}.*`, $options: 'i'};
} else {
filter_obj['$text'] = { $search: utils.createEdgeNGrams(text_search) };
}
}
files = await db_api.getRecords('files', filter_obj, false, sort, range, text_search);
let file_count = await db_api.getRecords('files', filter_obj, true);
playlists = await db_api.getRecords('playlists', {user_uid: uuid});
const categories = await categories_api.getCategoriesAsPlaylists(files);
@ -1646,6 +1660,7 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) {
res.send({
files: files,
file_count: file_count,
playlists: playlists
});
});
@ -2116,8 +2131,15 @@ app.post('/api/getPlaylist', optionalJwt, async (req, res) => {
app.post('/api/getPlaylists', optionalJwt, async (req, res) => {
const uuid = req.isAuthenticated() ? req.user.uid : null;
const include_categories = req.body.include_categories;
const playlists = await db_api.getRecords('playlists', {user_uid: uuid});
if (include_categories) {
const categories = await categories_api.getCategoriesAsPlaylists(files);
if (categories) {
playlists = playlists.concat(categories);
}
}
res.send({
playlists: playlists

@ -18,7 +18,12 @@ var database = null;
const tables = {
files: {
name: 'files',
primary_key: 'uid'
primary_key: 'uid',
text_search: {
title: 'text',
uploader: 'text',
uid: 'text'
}
},
playlists: {
name: 'playlists',
@ -131,8 +136,13 @@ exports._connectToDB = async (custom_connection_string = null) => {
tables_list.forEach(async table => {
const primary_key = tables[table]['primary_key'];
if (!primary_key) return;
await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
if (primary_key) {
await database.collection(table).createIndex({[primary_key]: 1}, { unique: true });
}
const text_search = tables[table]['text_search'];
if (text_search) {
await database.collection(table).createIndex(text_search);
}
});
return true;
} catch(err) {
@ -695,13 +705,28 @@ exports.getRecord = async (table, filter_obj) => {
return await database.collection(table).findOne(filter_obj);
}
exports.getRecords = async (table, filter_obj = null) => {
exports.getRecords = async (table, filter_obj = null, return_count = false, sort = null, range = null) => {
// local db override
if (using_local_db) {
return filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
let cursor = filter_obj ? applyFilterLocalDB(local_db.get(table), filter_obj, 'filter').value() : local_db.get(table).value();
if (sort) {
cursor = cursor.sort((a, b) => (a[sort['by']] > b[sort['by']] ? sort['order'] : sort['order']*-1));
}
if (range) {
cursor = cursor.slice(range[0], range[1]);
}
return !return_count ? cursor : cursor.length;
}
return filter_obj ? await database.collection(table).find(filter_obj).toArray() : await database.collection(table).find().toArray();
const cursor = filter_obj ? database.collection(table).find(filter_obj) : database.collection(table).find();
if (sort) {
cursor.sort({[sort['by']]: sort['order']});
}
if (range) {
cursor.skip(range[0]).limit(range[1] - range[0]);
}
return !return_count ? await cursor.toArray() : await cursor.count();
}
// Update
@ -1027,7 +1052,13 @@ const applyFilterLocalDB = (db_path, filter_obj, operation) => {
if (filter_prop_value === undefined || filter_prop_value === null) {
filtered &= record[filter_prop] === undefined || record[filter_prop] === null
} else {
filtered &= record[filter_prop] === filter_prop_value;
if (typeof filter_prop_value === 'object') {
if (filter_prop_value['$regex']) {
filtered &= (record[filter_prop].search(new RegExp(filter_prop_value['$regex'], filter_prop_value['$options'])) !== -1);
}
} else {
filtered &= record[filter_prop] === filter_prop_value;
}
}
}
return filtered;

@ -349,6 +349,26 @@ function removeFileExtension(filename) {
return filename_parts.join('.');
}
function createEdgeNGrams(str) {
if (str && str.length > 3) {
const minGram = 3
const maxGram = str.length
return str.split(" ").reduce((ngrams, token) => {
if (token.length > minGram) {
for (let i = minGram; i <= maxGram && i <= token.length; ++i) {
ngrams = [...ngrams, token.substr(0, i)]
}
} else {
ngrams = [...ngrams, token]
}
return ngrams
}, []).join(" ")
}
return str
}
/**
* setTimeout, but its a promise.
* @param {number} ms
@ -399,6 +419,7 @@ module.exports = {
getCurrentDownloader: getCurrentDownloader,
recFindByExt: recFindByExt,
removeFileExtension: removeFileExtension,
createEdgeNGrams: createEdgeNGrams,
wait: wait,
File: File
}

@ -35,7 +35,7 @@ export class CustomPlaylistsComponent implements OnInit {
getAllPlaylists() {
this.playlists_received = false;
// must call getAllFiles as we need to get category playlists as well
this.postsService.getAllFiles().subscribe(res => {
this.postsService.getPlaylists().subscribe(res => {
this.playlists = res['playlists'];
this.playlists_received = true;
});

@ -34,7 +34,7 @@
<div *ngFor="let file of paged_data; let i = index" class="mb-2 mt-2 d-flex justify-content-center" [ngClass]="[ postsService.card_size === 'small' ? 'col-2 small-col' : '', postsService.card_size === 'medium' ? 'col-6 col-lg-4 medium-col' : '', postsService.card_size === 'large' ? 'col-12 large-col' : '' ]">
<app-unified-file-card [index]="i" [card_size]="postsService.card_size" [locale]="postsService.locale" (goToFile)="goToFile($event)" (goToSubscription)="goToSubscription($event)" [file_obj]="file" [use_youtubedl_archive]="postsService.config['Downloader']['use_youtubedl_archive']" [availablePlaylists]="playlists" (addFileToPlaylist)="addFileToPlaylist($event)" [loading]="false" (deleteFile)="deleteFile($event)" [baseStreamPath]="postsService.path" [jwtString]="postsService.isLoggedIn ? '?jwt=' + this.postsService.token : ''"></app-unified-file-card>
</div>
<div *ngIf="filtered_files.length === 0">
<div *ngIf="paged_data.length === 0">
<ng-container i18n="No videos found">No videos found.</ng-container>
</div>
</ng-container>
@ -46,7 +46,7 @@
</div>
</div>
<mat-paginator class="paginator" #paginator *ngIf="filtered_files && filtered_files.length > 0" (page)="pageChangeEvent($event)" [length]="filtered_files.length"
<mat-paginator class="paginator" #paginator *ngIf="paged_data && paged_data.length > 0" (page)="pageChangeEvent($event)" [length]="file_count"
[pageSize]="pageSize"
[pageSizeOptions]="[5, 10, 25, 100, this.paged_data && this.paged_data.length > 100 ? this.paged_data.length : 250]">
</mat-paginator>

@ -2,6 +2,8 @@ import { Component, OnInit, ViewChild } from '@angular/core';
import { PostsService } from 'app/posts.services';
import { Router } from '@angular/router';
import { MatPaginator } from '@angular/material/paginator';
import { Subject } from 'rxjs';
import { distinctUntilChanged } from 'rxjs/operators';
@Component({
selector: 'app-recent-videos',
@ -15,8 +17,8 @@ export class RecentVideosComponent implements OnInit {
normal_files_received = false;
subscription_files_received = false;
files: any[] = null;
filtered_files: any[] = null;
file_count = 10;
searchChangedSubject: Subject<string> = new Subject<string>();
downloading_content = {'video': {}, 'audio': {}};
search_mode = false;
search_text = '';
@ -97,6 +99,18 @@ export class RecentVideosComponent implements OnInit {
if (cached_filter_property && this.filterProperties[cached_filter_property]) {
this.filterProperty = this.filterProperties[cached_filter_property];
}
this.searchChangedSubject
.debounceTime(500)
.pipe(distinctUntilChanged()
).subscribe(model => {
if (model.length > 0) {
this.search_mode = true;
} else {
this.search_mode = false;
}
this.getAllFiles();
});
}
getAllPlaylists() {
@ -108,64 +122,40 @@ export class RecentVideosComponent implements OnInit {
// search
onSearchInputChanged(newvalue) {
if (newvalue.length > 0) {
this.search_mode = true;
this.filterFiles(newvalue);
} else {
this.search_mode = false;
this.filtered_files = this.files;
}
}
private filterFiles(value: string) {
const filterValue = value.toLowerCase();
this.filtered_files = this.files.filter(option => option.id.toLowerCase().includes(filterValue) || option.category?.name?.toLowerCase().includes(filterValue));
this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex});
}
filterByProperty(prop) {
if (this.descendingMode) {
this.filtered_files = this.filtered_files.sort((a, b) => (a[prop] > b[prop] ? -1 : 1));
} else {
this.filtered_files = this.filtered_files.sort((a, b) => (a[prop] > b[prop] ? 1 : -1));
}
if (this.paginator) { this.pageChangeEvent({pageSize: this.pageSize, pageIndex: this.paginator.pageIndex}) };
this.normal_files_received = false;
this.searchChangedSubject.next(newvalue);
}
filterOptionChanged(value) {
this.filterByProperty(value['property']);
localStorage.setItem('filter_property', value['key']);
this.getAllFiles();
}
toggleModeChange() {
this.descendingMode = !this.descendingMode;
this.filterByProperty(this.filterProperty['property']);
this.getAllFiles();
}
// get files
getAllFiles() {
this.normal_files_received = false;
this.postsService.getAllFiles().subscribe(res => {
this.files = res['files'];
this.files.sort(this.sortFiles);
for (let i = 0; i < this.files.length; i++) {
const file = this.files[i];
getAllFiles(cache_mode = false) {
this.normal_files_received = cache_mode;
const current_file_index = (this.paginator?.pageIndex ? this.paginator.pageIndex : 0)*this.pageSize;
const sort = {by: this.filterProperty['property'], order: this.descendingMode ? -1 : 1};
const range = [current_file_index, current_file_index + this.pageSize];
this.postsService.getAllFiles(sort, range, this.search_mode ? this.search_text : null).subscribe(res => {
this.file_count = res['file_count'];
this.paged_data = res['files'];
for (let i = 0; i < this.paged_data.length; i++) {
const file = this.paged_data[i];
file.duration = typeof file.duration !== 'string' ? file.duration : this.durationStringToNumber(file.duration);
}
if (this.search_mode) {
this.filterFiles(this.search_text);
} else {
this.filtered_files = this.files;
}
this.filterByProperty(this.filterProperty['property']);
// set cached file count for future use, note that we convert the amount of files to a string
localStorage.setItem('cached_file_count', '' + this.files.length);
localStorage.setItem('cached_file_count', '' + this.file_count);
this.normal_files_received = true;
this.paged_data = this.filtered_files.slice(0, 10);
});
}
@ -301,12 +291,9 @@ export class RecentVideosComponent implements OnInit {
}
removeFileCard(file_to_remove) {
const index = this.files.map(e => e.uid).indexOf(file_to_remove.uid);
this.files.splice(index, 1);
if (this.search_mode) {
this.filterFiles(this.search_text);
}
this.filterByProperty(this.filterProperty['property']);
const index = this.paged_data.map(e => e.uid).indexOf(file_to_remove.uid);
this.paged_data.splice(index, 1);
this.getAllFiles(true);
}
addFileToPlaylist(info_obj) {
@ -344,7 +331,8 @@ export class RecentVideosComponent implements OnInit {
}
pageChangeEvent(event) {
const offset = ((event.pageIndex + 1) - 1) * event.pageSize;
this.paged_data = this.filtered_files.slice(offset).slice(0, event.pageSize);
this.pageSize = event.pageSize;
this.loading_files = Array(this.pageSize).fill(0);
this.getAllFiles();
}
}

@ -239,8 +239,8 @@ export class PostsService implements CanActivate {
return this.http.post(this.path + 'getFile', {uid: uid, type: type, uuid: uuid}, this.httpOptions);
}
getAllFiles() {
return this.http.post(this.path + 'getAllFiles', {}, this.httpOptions);
getAllFiles(sort, range, text_search) {
return this.http.post(this.path + 'getAllFiles', {sort: sort, range: range, text_search: text_search}, this.httpOptions);
}
getFullTwitchChat(id, type, uuid = null, sub = null) {

Loading…
Cancel
Save