From 62ad4226d94e408ea85b720bbaa353e6ebeae20e Mon Sep 17 00:00:00 2001 From: Tzahi12345 Date: Sun, 26 Mar 2023 01:01:36 -0400 Subject: [PATCH] Archive improvements Began UI for viewing/modifying archives --- Public API v1.yaml | 141 ++++++++++++++++++ backend/archive.js | 14 +- backend/downloader.js | 2 +- src/api-types/index.ts | 6 + src/api-types/models/Archive.ts | 16 ++ .../models/DeleteArchiveItemRequest.ts | 12 ++ src/api-types/models/GetArchivesRequest.ts | 10 ++ src/api-types/models/GetArchivesResponse.ts | 9 ++ src/api-types/models/ImportArchiveRequest.ts | 11 ++ src/api-types/models/UploadCookiesRequest.ts | 7 + src/app/app.module.ts | 4 +- .../archive-viewer.component.html | 51 +++++++ .../archive-viewer.component.scss | 0 .../archive-viewer.component.spec.ts | 23 +++ .../archive-viewer.component.ts | 41 +++++ .../cookies-uploader-dialog.component.ts | 7 +- src/app/posts.services.ts | 30 +++- 17 files changed, 369 insertions(+), 15 deletions(-) create mode 100644 src/api-types/models/Archive.ts create mode 100644 src/api-types/models/DeleteArchiveItemRequest.ts create mode 100644 src/api-types/models/GetArchivesRequest.ts create mode 100644 src/api-types/models/GetArchivesResponse.ts create mode 100644 src/api-types/models/ImportArchiveRequest.ts create mode 100644 src/api-types/models/UploadCookiesRequest.ts create mode 100644 src/app/components/archive-viewer/archive-viewer.component.html create mode 100644 src/app/components/archive-viewer/archive-viewer.component.scss create mode 100644 src/app/components/archive-viewer/archive-viewer.component.spec.ts create mode 100644 src/app/components/archive-viewer/archive-viewer.component.ts diff --git a/Public API v1.yaml b/Public API v1.yaml index a69551f..912fd3d 100644 --- a/Public API v1.yaml +++ b/Public API v1.yaml @@ -578,6 +578,69 @@ paths: description: If the archive dir is not found, 404 is sent as a response security: - Auth query parameter: [] + /api/deleteArchiveItem: + post: + tags: + - archive + summary: Delete item from archive + description: 'Deletes an item from the archive' + operationId: post-api-deleteArchiveItem + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteArchiveItemRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/importArchive: + post: + tags: + - archive + summary: Imports archive + description: 'Imports an existing archive.txt file' + operationId: post-api-importArchive + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/ImportArchiveRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] + /api/uploadCookies: + post: + tags: + - downloader + summary: Upload cookies + description: 'Uploads cookies file to be used during downloading' + operationId: post-api-uploadCookies + requestBody: + content: + multipart/form-data: + schema: + $ref: '#/components/schemas/UploadCookiesRequest' + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/SuccessObject' + security: + - Auth query parameter: [] /api/updaterStatus: get: tags: @@ -2089,6 +2152,84 @@ components: $ref: '#/components/schemas/FileType' sub_id: type: string + Archive: + required: + - extractor + - id + - type + - title + - timestamp + - uid + type: object + properties: + extractor: + type: string + id: + type: string + type: + $ref: '#/components/schemas/FileType' + title: + type: string + user_uid: + type: string + sub_id: + type: string + timestamp: + type: number + uid: + type: string + DeleteArchiveItemRequest: + type: object + required: + - extractor + - id + - type + properties: + extractor: + type: string + id: + type: string + type: + $ref: '#/components/schemas/FileType' + sub_id: + type: string + ImportArchiveRequest: + type: object + required: + - archive + - type + properties: + archive: + type: string + format: binary + type: + $ref: '#/components/schemas/FileType' + sub_id: + type: string + GetArchivesRequest: + type: object + properties: + type: + $ref: '#/components/schemas/FileType' + sub_id: + type: string + GetArchivesResponse: + type: object + required: + - archives + properties: + archives: + type: array + items: + $ref: '#/components/schemas/Archive' + UploadCookiesRequest: + type: object + required: + - cookies + properties: + cookies: + type: string + format: binary UpdaterStatus: required: - details diff --git a/backend/archive.js b/backend/archive.js index d7e1dde..6df8e0e 100644 --- a/backend/archive.js +++ b/backend/archive.js @@ -1,5 +1,6 @@ const path = require('path'); const fs = require('fs-extra'); +const { uuid } = require('uuidv4'); const db_api = require('./db'); @@ -11,8 +12,8 @@ exports.generateArchive = async (type = null, user_uid = null, sub_id = null) => return archive_item_lines.join('\n'); } -exports.addToArchive = async (extractor, id, type, user_uid = null, sub_id = null) => { - const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id); +exports.addToArchive = async (extractor, id, type, title, user_uid = null, sub_id = null) => { + const archive_item = createArchiveItem(extractor, id, type, title, user_uid, sub_id); const success = await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id, type: type}); return success; } @@ -43,7 +44,7 @@ exports.importArchiveFile = async (archive_text, type, user_uid = null, sub_id = // we can't do a bulk write because we need to avoid duplicate archive items existing in db - const archive_item = createArchiveItem(extractor, id, type, user_uid, sub_id); + const archive_item = createArchiveItem(extractor, id, type, null, user_uid, sub_id); await db_api.insertRecordIntoTable('archives', archive_item, {extractor: extractor, id: id}); archive_import_count++; } @@ -76,12 +77,15 @@ exports.importArchives = async () => { return imported_archives; } -const createArchiveItem = (extractor, id, type, user_uid = null, sub_id = null) => { +const createArchiveItem = (extractor, id, type, title = null, user_uid = null, sub_id = null) => { return { extractor: extractor, id: id, type: type, + title: title, user_uid: user_uid ? user_uid : null, - sub_id: sub_id ? sub_id : null + sub_id: sub_id ? sub_id : null, + timestamp: Date.now() / 1000, + uid: uuid() } } \ No newline at end of file diff --git a/backend/downloader.js b/backend/downloader.js index 58bd0d5..80ecaf4 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -369,7 +369,7 @@ async function downloadQueuedFile(download_uid) { const file_obj = await db_api.registerFileDB(full_file_path, type, download['user_uid'], category, download['sub_id'] ? download['sub_id'] : null, options.cropFileSettings); const useYoutubeDLArchive = config_api.getConfigItem('ytdl_use_youtubedl_archive'); - if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, download['user_uid'], download['sub_id']); + if (useYoutubeDLArchive && !options.ignoreArchive) await archive_api.addToArchive(output_json['extractor'], output_json['id'], type, output_json['title'], download['user_uid'], download['sub_id']); notifications_api.sendDownloadNotification(file_obj, download['user_uid']); diff --git a/src/api-types/index.ts b/src/api-types/index.ts index 0fa0c93..9f153bb 100644 --- a/src/api-types/index.ts +++ b/src/api-types/index.ts @@ -3,6 +3,7 @@ /* eslint-disable */ export type { AddFileToPlaylistRequest } from './models/AddFileToPlaylistRequest'; +export type { Archive } from './models/Archive'; export type { BaseChangePermissionsRequest } from './models/BaseChangePermissionsRequest'; export type { binary } from './models/binary'; export type { body_19 } from './models/body_19'; @@ -26,6 +27,7 @@ export type { DatabaseFile } from './models/DatabaseFile'; export { DBBackup } from './models/DBBackup'; export type { DBInfoResponse } from './models/DBInfoResponse'; export type { DeleteAllFilesResponse } from './models/DeleteAllFilesResponse'; +export type { DeleteArchiveItemRequest } from './models/DeleteArchiveItemRequest'; export type { DeleteCategoryRequest } from './models/DeleteCategoryRequest'; export type { DeleteMp3Mp4Request } from './models/DeleteMp3Mp4Request'; export type { DeleteNotificationRequest } from './models/DeleteNotificationRequest'; @@ -51,6 +53,8 @@ export type { GetAllFilesRequest } from './models/GetAllFilesRequest'; export type { GetAllFilesResponse } from './models/GetAllFilesResponse'; export type { GetAllSubscriptionsResponse } from './models/GetAllSubscriptionsResponse'; export type { GetAllTasksResponse } from './models/GetAllTasksResponse'; +export type { GetArchivesRequest } from './models/GetArchivesRequest'; +export type { GetArchivesResponse } from './models/GetArchivesResponse'; export type { GetDBBackupsResponse } from './models/GetDBBackupsResponse'; export type { GetDownloadRequest } from './models/GetDownloadRequest'; export type { GetDownloadResponse } from './models/GetDownloadResponse'; @@ -75,6 +79,7 @@ export type { GetSubscriptionResponse } from './models/GetSubscriptionResponse'; export type { GetTaskRequest } from './models/GetTaskRequest'; export type { GetTaskResponse } from './models/GetTaskResponse'; export type { GetUsersResponse } from './models/GetUsersResponse'; +export type { ImportArchiveRequest } from './models/ImportArchiveRequest'; export type { IncrementViewCountRequest } from './models/IncrementViewCountRequest'; export type { inline_response_200_15 } from './models/inline_response_200_15'; export type { LoginRequest } from './models/LoginRequest'; @@ -117,6 +122,7 @@ export type { UpdateTaskDataRequest } from './models/UpdateTaskDataRequest'; export type { UpdateTaskOptionsRequest } from './models/UpdateTaskOptionsRequest'; export type { UpdateTaskScheduleRequest } from './models/UpdateTaskScheduleRequest'; export type { UpdateUserRequest } from './models/UpdateUserRequest'; +export type { UploadCookiesRequest } from './models/UploadCookiesRequest'; export type { User } from './models/User'; export { UserPermission } from './models/UserPermission'; export type { Version } from './models/Version'; diff --git a/src/api-types/models/Archive.ts b/src/api-types/models/Archive.ts new file mode 100644 index 0000000..63ce05b --- /dev/null +++ b/src/api-types/models/Archive.ts @@ -0,0 +1,16 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { FileType } from './FileType'; + +export type Archive = { + extractor: string; + id: string; + type: FileType; + title: string; + user_uid?: string; + sub_id?: string; + timestamp: number; + uid: string; +}; diff --git a/src/api-types/models/DeleteArchiveItemRequest.ts b/src/api-types/models/DeleteArchiveItemRequest.ts new file mode 100644 index 0000000..192eb50 --- /dev/null +++ b/src/api-types/models/DeleteArchiveItemRequest.ts @@ -0,0 +1,12 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { FileType } from './FileType'; + +export type DeleteArchiveItemRequest = { + extractor: string; + id: string; + type: FileType; + sub_id?: string; +}; diff --git a/src/api-types/models/GetArchivesRequest.ts b/src/api-types/models/GetArchivesRequest.ts new file mode 100644 index 0000000..0ce245a --- /dev/null +++ b/src/api-types/models/GetArchivesRequest.ts @@ -0,0 +1,10 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { FileType } from './FileType'; + +export type GetArchivesRequest = { + type?: FileType; + sub_id?: string; +}; diff --git a/src/api-types/models/GetArchivesResponse.ts b/src/api-types/models/GetArchivesResponse.ts new file mode 100644 index 0000000..4c0e1ce --- /dev/null +++ b/src/api-types/models/GetArchivesResponse.ts @@ -0,0 +1,9 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { Archive } from './Archive'; + +export type GetArchivesResponse = { + archives: Array; +}; diff --git a/src/api-types/models/ImportArchiveRequest.ts b/src/api-types/models/ImportArchiveRequest.ts new file mode 100644 index 0000000..e5a5093 --- /dev/null +++ b/src/api-types/models/ImportArchiveRequest.ts @@ -0,0 +1,11 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +import type { FileType } from './FileType'; + +export type ImportArchiveRequest = { + archive: Blob; + type: FileType; + sub_id?: string; +}; diff --git a/src/api-types/models/UploadCookiesRequest.ts b/src/api-types/models/UploadCookiesRequest.ts new file mode 100644 index 0000000..e240063 --- /dev/null +++ b/src/api-types/models/UploadCookiesRequest.ts @@ -0,0 +1,7 @@ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ + +export type UploadCookiesRequest = { + cookies: Blob; +}; diff --git a/src/app/app.module.ts b/src/app/app.module.ts index e60e0fb..8768910 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -94,6 +94,7 @@ import { TaskSettingsComponent } from './components/task-settings/task-settings. import { GenerateRssUrlComponent } from './dialogs/generate-rss-url/generate-rss-url.component'; import { SortPropertyComponent } from './components/sort-property/sort-property.component'; import { OnlyNumberDirective } from './directives/only-number.directive'; +import { ArchiveViewerComponent } from './components/archive-viewer/archive-viewer.component'; registerLocaleData(es, 'es'); @@ -145,7 +146,8 @@ registerLocaleData(es, 'es'); TaskSettingsComponent, GenerateRssUrlComponent, SortPropertyComponent, - OnlyNumberDirective + OnlyNumberDirective, + ArchiveViewerComponent ], imports: [ CommonModule, diff --git a/src/app/components/archive-viewer/archive-viewer.component.html b/src/app/components/archive-viewer/archive-viewer.component.html new file mode 100644 index 0000000..935cdd0 --- /dev/null +++ b/src/app/components/archive-viewer/archive-viewer.component.html @@ -0,0 +1,51 @@ + + +
+
+ + + + + Date + {{element.timestamp | date: 'short'}} + + + + + Title + + + {{element.title}} + + + + + + + ID + + + {{element.id}} + + + + + + + Extractor + + + {{element.extractor}} + + + + + + + +
+
+ +
+

Archives empty

+
\ No newline at end of file diff --git a/src/app/components/archive-viewer/archive-viewer.component.scss b/src/app/components/archive-viewer/archive-viewer.component.scss new file mode 100644 index 0000000..e69de29 diff --git a/src/app/components/archive-viewer/archive-viewer.component.spec.ts b/src/app/components/archive-viewer/archive-viewer.component.spec.ts new file mode 100644 index 0000000..d86a711 --- /dev/null +++ b/src/app/components/archive-viewer/archive-viewer.component.spec.ts @@ -0,0 +1,23 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ArchiveViewerComponent } from './archive-viewer.component'; + +describe('ArchiveViewerComponent', () => { + let component: ArchiveViewerComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ ArchiveViewerComponent ] + }) + .compileComponents(); + + fixture = TestBed.createComponent(ArchiveViewerComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/components/archive-viewer/archive-viewer.component.ts b/src/app/components/archive-viewer/archive-viewer.component.ts new file mode 100644 index 0000000..2684af8 --- /dev/null +++ b/src/app/components/archive-viewer/archive-viewer.component.ts @@ -0,0 +1,41 @@ +import { Component, ViewChild } from '@angular/core'; +import { MatSort } from '@angular/material/sort'; +import { MatTableDataSource } from '@angular/material/table'; +import { Archive } from 'api-types/models/Archive'; +import { PostsService } from 'app/posts.services'; + +@Component({ + selector: 'app-archive-viewer', + templateUrl: './archive-viewer.component.html', + styleUrls: ['./archive-viewer.component.scss'] +}) +export class ArchiveViewerComponent { + archives = null; + displayedColumns: string[] = ['timestamp', 'title', 'id', 'extractor']; + dataSource = null; + archives_retrieved = false; + + @ViewChild(MatSort) sort: MatSort; + + constructor(private postsService: PostsService) { + + } + + filterSelectionChanged(value: string): void { + this.getArchives(value); + } + + getArchives(sub_id: string = null): void { + this.postsService.getArchives(sub_id).subscribe(res => { + if (res['archives'] !== null + && res['archives'] !== undefined + && JSON.stringify(this.archives) !== JSON.stringify(res['archives'])) { + this.archives = res['archives'] + this.dataSource = new MatTableDataSource(this.archives); + this.dataSource.sort = this.sort; + } else { + // failed to get downloads + } + }); + } +} diff --git a/src/app/dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component.ts b/src/app/dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component.ts index fc17a64..bc3b69e 100644 --- a/src/app/dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component.ts +++ b/src/app/dialogs/cookies-uploader-dialog/cookies-uploader-dialog.component.ts @@ -31,11 +31,8 @@ export class CookiesUploaderDialogComponent implements OnInit { // Is it a file? if (droppedFile.fileEntry.isFile) { const fileEntry = droppedFile.fileEntry as FileSystemFileEntry; - fileEntry.file((file: File) => { - // You could upload it like this: - const formData = new FormData() - formData.append('cookies', file, droppedFile.relativePath); - this.postsService.uploadCookiesFile(formData).subscribe(res => { + fileEntry.file((file: File) => { + this.postsService.uploadCookiesFile(file, droppedFile.relativePath).subscribe(res => { this.uploading = false; if (res['success']) { this.uploaded = true; diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index eb392b4..11575cd 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -106,7 +106,10 @@ import { SetNotificationsToReadRequest, GetNotificationsResponse, UpdateTaskOptionsRequest, - User + User, + DeleteArchiveItemRequest, + GetArchivesRequest, + GetArchivesResponse } from '../api-types'; import { isoLangs } from './settings/locales_list'; import { Title } from '@angular/platform-browser'; @@ -431,8 +434,11 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'updateConcurrentStream', body, this.httpOptions); } - uploadCookiesFile(fileFormData) { - return this.http.post(this.path + 'uploadCookies', fileFormData, this.httpOptions); + uploadCookiesFile(cookiesFile: File, cookiesFilePath: string) { + const formData = new FormData() + formData.append('cookies', cookiesFile, cookiesFilePath); + // const body: UploadCookiesRequest = formData; + return this.http.post(this.path + 'uploadCookies', formData, this.httpOptions); } downloadArchive(type: FileType, sub_id: string) { @@ -440,6 +446,24 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'downloadArchive', body, {responseType: 'blob', params: this.httpOptions.params}); } + getArchives(sub_id: string) { + const body: GetArchivesRequest = {sub_id: sub_id}; + return this.http.post(this.path + 'getArchives', body, this.httpOptions); + } + + importArchive(archiveFile: File, type: FileType, sub_id: string = null) { + const formData = new FormData() + formData.append('archive', archiveFile, 'archive.txt'); + formData.append('type', type); + formData.append('sub_id', sub_id); + return this.http.post(this.path + 'importArchive', formData, this.httpOptions); + } + + deleteArchiveItem(extractor: string, id: string, type: FileType, sub_id: string = null) { + const body: DeleteArchiveItemRequest = {extractor: extractor, id: id, type: type, sub_id: sub_id}; + return this.http.post(this.path + 'deleteArchiveItem', body, this.httpOptions); + } + getFileFormats(url) { const body: GetFileFormatsRequest = {url: url}; return this.http.post(this.path + 'getFileFormats', body, this.httpOptions);