diff --git a/backend/app.js b/backend/app.js index 3c1af08..e50dc8d 100644 --- a/backend/app.js +++ b/backend/app.js @@ -27,6 +27,7 @@ const url_api = require('url'); var config_api = require('./config.js'); var subscriptions_api = require('./subscriptions') var categories_api = require('./categories'); +var twitch_api = require('./twitch'); const CONSTS = require('./consts') const { spawn } = require('child_process') const read_last_lines = require('read-last-lines'); @@ -38,6 +39,7 @@ var app = express(); // database setup const FileSync = require('lowdb/adapters/FileSync'); +const config = require('./config.js'); const adapter = new FileSync('./appdata/db.json'); const db = low(adapter) @@ -1186,6 +1188,13 @@ async function downloadFileByURL_exec(url, type, options, sessionID = null) { var full_file_path = filepath_no_extension + ext; var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length); + if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 + && config.getConfigItem('ytdl_use_twitch_api') && config.getConfigItem('ytdl_twitch_auto_download_chat')) { + let vodId = url.split('twitch.tv/videos/')[1]; + vodId = vodId.split('?')[0]; + downloadTwitchChatByVODID(vodId, file_name, type, options.user); + } + // renames file if necessary due to bug if (!fs.existsSync(output_json['_filename'] && fs.existsSync(output_json['_filename'] + '.webm'))) { try { @@ -1759,6 +1768,42 @@ function removeFileExtension(filename) { return filename_parts.join('.'); } +async function getTwitchChatByFileID(id, type, user_uid, uuid) { + let file_path = null; + + if (user_uid) { + file_path = path.join('users', user_uid, type, id + '.twitch_chat.json'); + } else { + file_path = path.join(type, id + '.twitch_chat.json'); + } + + var chat_file = null; + if (fs.existsSync(file_path)) { + chat_file = fs.readJSONSync(file_path); + } + + return chat_file; +} + +async function downloadTwitchChatByVODID(vodId, id, type, user_uid) { + const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key'); + const chat = await twitch_api.getCommentsForVOD(twitch_api_key, vodId); + + // save file if needec params are included + if (id && type) { + let file_path = null; + if (user_uid) { + file_path = path.join('users', user_uid, type, id + '.twitch_chat.json'); + } else { + file_path = path.join(type, id + '.twitch_chat.json'); + } + + if (chat) fs.writeJSONSync(file_path, chat); + } + + return chat; +} + app.use(function(req, res, next) { res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, Authorization"); res.header("Access-Control-Allow-Origin", getOrigin()); @@ -2058,6 +2103,54 @@ app.post('/api/getAllFiles', optionalJwt, async function (req, res) { }); }); +app.post('/api/getFullTwitchChat', optionalJwt, async (req, res) => { + var id = req.body.id; + var type = req.body.type; + var uuid = req.body.uuid; + var user_uid = null; + + if (req.isAuthenticated()) user_uid = req.user.uid; + + const chat_file = await getTwitchChatByFileID(id, type, user_uid, uuid); + + res.send({ + chat: chat_file + }); +}); + +app.post('/api/downloadTwitchChatByVODID', optionalJwt, async (req, res) => { + var id = req.body.id; + var type = req.body.type; + var vodId = req.body.vodId; + var uuid = req.body.uuid; + var user_uid = null; + + if (req.isAuthenticated()) user_uid = req.user.uid; + + // check if file already exists. if so, send that instead + const file_exists_check = await getTwitchChatByFileID(id, type, user_uid, uuid); + if (file_exists_check) { + res.send({chat: file_exists_check}); + return; + } + + const full_chat = await downloadTwitchChatByVODID(vodId); + + let file_path = null; + + if (user_uid) { + file_path = path.join('users', req.user.uid, type, id + '.twitch_chat.json'); + } else { + file_path = path.join(type, id + '.twitch_chat.json'); + } + + if (full_chat) fs.writeJSONSync(file_path, full_chat); + + res.send({ + chat: full_chat + }); +}); + // video sharing app.post('/api/enableSharing', optionalJwt, function(req, res) { var type = req.body.type; diff --git a/backend/appdata/default.json b/backend/appdata/default.json index 46dca1b..001f515 100644 --- a/backend/appdata/default.json +++ b/backend/appdata/default.json @@ -26,7 +26,10 @@ "use_API_key": false, "API_key": "", "use_youtube_API": false, - "youtube_API_key": "" + "youtube_API_key": "", + "use_twitch_API": false, + "twitch_API_key": "", + "twitch_auto_download_chat": false }, "Themes": { "default_theme": "default", diff --git a/backend/config.js b/backend/config.js index c0c9402..b2aff63 100644 --- a/backend/config.js +++ b/backend/config.js @@ -203,7 +203,10 @@ DEFAULT_CONFIG = { "use_API_key": false, "API_key": "", "use_youtube_API": false, - "youtube_API_key": "" + "youtube_API_key": "", + "use_twitch_API": false, + "twitch_API_key": "", + "twitch_auto_download_chat": false }, "Themes": { "default_theme": "default", diff --git a/backend/consts.js b/backend/consts.js index b0b8b22..bffe486 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -86,6 +86,18 @@ let CONFIG_ITEMS = { 'key': 'ytdl_youtube_api_key', 'path': 'YoutubeDLMaterial.API.youtube_API_key' }, + 'ytdl_use_twitch_api': { + 'key': 'ytdl_use_twitch_api', + 'path': 'YoutubeDLMaterial.API.use_twitch_API' + }, + 'ytdl_twitch_api_key': { + 'key': 'ytdl_twitch_api_key', + 'path': 'YoutubeDLMaterial.API.twitch_API_key' + }, + 'ytdl_twitch_auto_download_chat': { + 'key': 'ytdl_twitch_auto_download_chat', + 'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat' + }, // Themes 'ytdl_default_theme': { diff --git a/backend/twitch.js b/backend/twitch.js new file mode 100644 index 0000000..62c390d --- /dev/null +++ b/backend/twitch.js @@ -0,0 +1,71 @@ +var moment = require('moment'); +var Axios = require('axios'); + +async function getCommentsForVOD(clientID, vodId) { + let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`, + batch, + cursor; + + let comments = null; + + try { + do { + batch = (await Axios.get(url, { + headers: { + 'Client-ID': clientID, + Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8', + 'Content-Type': 'application/json; charset=UTF-8', + } + })).data; + + const str = batch.comments.map(c => { + let { + created_at: msgCreated, + content_offset_seconds: timestamp, + commenter: { + name, + _id, + created_at: acctCreated + }, + message: { + body: msg + } + } = c; + + const timestamp_str = moment.duration(timestamp, 'seconds') + .toISOString() + .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/, + (_, ...ms) => { + const seg = v => v ? v.padStart(2, '0') : '00'; + return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`; + }); + + acctCreated = moment(acctCreated).utc(); + msgCreated = moment(msgCreated).utc(); + + if (!comments) comments = []; + + comments.push({ + timestamp: timestamp, + timestamp_str: timestamp_str, + name: name, + message: msg + }); + // let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`; + // return line; + }).join('\n'); + + cursor = batch._next; + url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`; + await new Promise(res => setTimeout(res, 300)); + } while (cursor); + } catch (err) { + console.error(err); + } + + return comments; +} + +module.exports = { + getCommentsForVOD: getCommentsForVOD +} \ No newline at end of file diff --git a/src/app/app.module.ts b/src/app/app.module.ts index 423f714..72c2658 100644 --- a/src/app/app.module.ts +++ b/src/app/app.module.ts @@ -80,6 +80,7 @@ import { RecentVideosComponent } from './components/recent-videos/recent-videos. import { EditSubscriptionDialogComponent } from './dialogs/edit-subscription-dialog/edit-subscription-dialog.component'; import { CustomPlaylistsComponent } from './components/custom-playlists/custom-playlists.component'; import { EditCategoryDialogComponent } from './dialogs/edit-category-dialog/edit-category-dialog.component'; +import { TwitchChatComponent } from './components/twitch-chat/twitch-chat.component'; registerLocaleData(es, 'es'); @@ -125,7 +126,8 @@ export function isVisible({ event, element, scrollContainer, offset }: IsVisible RecentVideosComponent, EditSubscriptionDialogComponent, CustomPlaylistsComponent, - EditCategoryDialogComponent + EditCategoryDialogComponent, + TwitchChatComponent ], imports: [ CommonModule, diff --git a/src/app/components/twitch-chat/twitch-chat.component.html b/src/app/components/twitch-chat/twitch-chat.component.html new file mode 100644 index 0000000..7ad40fc --- /dev/null +++ b/src/app/components/twitch-chat/twitch-chat.component.html @@ -0,0 +1,11 @@ +
+
Twitch Chat
+
+ {{chat.timestamp_str}} - {{chat.name}}: {{chat.message}} +
+
+ + + + + \ No newline at end of file diff --git a/src/app/components/twitch-chat/twitch-chat.component.scss b/src/app/components/twitch-chat/twitch-chat.component.scss new file mode 100644 index 0000000..3c8f4ab --- /dev/null +++ b/src/app/components/twitch-chat/twitch-chat.component.scss @@ -0,0 +1,13 @@ +.chat-container { + height: 100%; + overflow-y: scroll; +} + +.download-button { + margin: 10px; +} + +.downloading-spinner { + top: 50%; + left: 80px; +} \ No newline at end of file diff --git a/src/app/components/twitch-chat/twitch-chat.component.spec.ts b/src/app/components/twitch-chat/twitch-chat.component.spec.ts new file mode 100644 index 0000000..eafdece --- /dev/null +++ b/src/app/components/twitch-chat/twitch-chat.component.spec.ts @@ -0,0 +1,25 @@ +import { async, ComponentFixture, TestBed } from '@angular/core/testing'; + +import { TwitchChatComponent } from './twitch-chat.component'; + +describe('TwitchChatComponent', () => { + let component: TwitchChatComponent; + let fixture: ComponentFixture; + + beforeEach(async(() => { + TestBed.configureTestingModule({ + declarations: [ TwitchChatComponent ] + }) + .compileComponents(); + })); + + beforeEach(() => { + fixture = TestBed.createComponent(TwitchChatComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/app/components/twitch-chat/twitch-chat.component.ts b/src/app/components/twitch-chat/twitch-chat.component.ts new file mode 100644 index 0000000..eace952 --- /dev/null +++ b/src/app/components/twitch-chat/twitch-chat.component.ts @@ -0,0 +1,135 @@ +import { AfterViewInit, Component, ElementRef, Input, OnInit, QueryList, ViewChild, ViewChildren } from '@angular/core'; +import { PostsService } from 'app/posts.services'; + +@Component({ + selector: 'app-twitch-chat', + templateUrl: './twitch-chat.component.html', + styleUrls: ['./twitch-chat.component.scss'] +}) +export class TwitchChatComponent implements OnInit, AfterViewInit { + + full_chat = null; + visible_chat = null; + chat_response_received = false; + downloading_chat = false; + + current_chat_index = null; + + CHAT_CHECK_INTERVAL_MS = 200; + chat_check_interval_obj = null; + + scrollContainer = null; + + @Input() db_file = null; + @Input() current_timestamp = null; + + @ViewChild('scrollContainer') scrollRef: ElementRef; + + constructor(private postsService: PostsService) { } + + ngOnInit(): void { + this.getFullChat(); + } + + ngAfterViewInit() { + } + + private isUserNearBottom(): boolean { + const threshold = 300; + const position = this.scrollContainer.scrollTop + this.scrollContainer.offsetHeight; + const height = this.scrollContainer.scrollHeight; + return position > height - threshold; + } + + scrollToBottom = () => { + this.scrollContainer.scrollTop = this.scrollContainer.scrollHeight; + } + + addNewChatMessages() { + if (!this.scrollContainer) { + this.scrollContainer = this.scrollRef.nativeElement; + } + if (this.current_chat_index === null) { + this.current_chat_index = this.getIndexOfNextChat(); + } + + const latest_chat_timestamp = this.visible_chat.length ? this.visible_chat[this.visible_chat.length - 1]['timestamp'] : 0; + + for (let i = this.current_chat_index + 1; i < this.full_chat.length; i++) { + if (this.full_chat[i]['timestamp'] >= latest_chat_timestamp && this.full_chat[i]['timestamp'] <= this.current_timestamp) { + this.visible_chat.push(this.full_chat[i]); + this.current_chat_index = i; + if (this.isUserNearBottom()) { + this.scrollToBottom(); + } + } else if (this.full_chat[i]['timestamp'] > this.current_timestamp) { + break; + } + } + } + + getIndexOfNextChat() { + const index = binarySearch(this.full_chat, 'timestamp', this.current_timestamp); + return index; + } + + getFullChat() { + this.postsService.getFullTwitchChat(this.db_file.id, this.db_file.isAudio ? 'audio' : 'video', null).subscribe(res => { + this.chat_response_received = true; + if (res['chat']) { + this.initializeChatCheck(res['chat']); + } + }); + } + + renewChat() { + this.visible_chat = []; + this.current_chat_index = this.getIndexOfNextChat(); + } + + downloadTwitchChat() { + this.downloading_chat = true; + let vodId = this.db_file.url.split('videos/').length > 1 && this.db_file.url.split('videos/')[1]; + vodId = vodId.split('?')[0]; + if (!vodId) { + this.postsService.openSnackBar('VOD url for this video is not supported. VOD ID must be after "twitch.tv/videos/"'); + } + this.postsService.downloadTwitchChat(this.db_file.id, this.db_file.isAudio ? 'audio' : 'video', vodId, null).subscribe(res => { + if (res['chat']) { + this.initializeChatCheck(res['chat']); + } else { + this.downloading_chat = false; + this.postsService.openSnackBar('Download failed.') + } + }, err => { + this.downloading_chat = false; + this.postsService.openSnackBar('Chat could not be downloaded.') + }); + } + + initializeChatCheck(full_chat) { + this.full_chat = full_chat; + this.visible_chat = []; + this.chat_check_interval_obj = setInterval(() => this.addNewChatMessages(), this.CHAT_CHECK_INTERVAL_MS); + } + +} + +function binarySearch(arr, key, n) { + let min = 0; + let max = arr.length - 1; + let mid; + while (min <= max) { + // tslint:disable-next-line: no-bitwise + mid = (min + max) >>> 1; + if (arr[mid][key] === n) { + return mid; + } else if (arr[mid][key] < n) { + min = mid + 1; + } else { + max = mid - 1; + } + } + + return min; +} diff --git a/src/app/player/player.component.html b/src/app/player/player.component.html index ad65493..c70800d 100644 --- a/src/app/player/player.component.html +++ b/src/app/player/player.component.html @@ -1,35 +1,46 @@ -
-
-
-
- - - -
-
- - {{playlist_item.label}} - -
-
-
+
+
+
+ +
+ + + +
+
+ + {{playlist_item.label}} + +
+ + + + + + +
+ +
-
- - - -
- -
- - - -
-
- - +
+ + + +
+ +
+ + + +
+
+ + +
+ +
\ No newline at end of file diff --git a/src/app/player/player.component.ts b/src/app/player/player.component.ts index 477c112..0ce60ab 100644 --- a/src/app/player/player.component.ts +++ b/src/app/player/player.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit, HostListener, EventEmitter, OnDestroy, AfterViewInit } from '@angular/core'; +import { Component, OnInit, HostListener, EventEmitter, OnDestroy, AfterViewInit, ViewChild } from '@angular/core'; import { VgAPI } from 'ngx-videogular'; import { PostsService } from 'app/posts.services'; import { ActivatedRoute, Router } from '@angular/router'; @@ -7,6 +7,7 @@ import { MatSnackBar } from '@angular/material/snack-bar'; import { InputDialogComponent } from 'app/input-dialog/input-dialog.component'; import { CdkDragDrop, moveItemInArray } from '@angular/cdk/drag-drop'; import { ShareMediaDialogComponent } from '../dialogs/share-media-dialog/share-media-dialog.component'; +import { TwitchChatComponent } from 'app/components/twitch-chat/twitch-chat.component'; export interface IMedia { title: string; @@ -31,6 +32,7 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { currentIndex = 0; currentItem: IMedia = null; api: VgAPI; + api_ready = false; // params fileNames: string[]; @@ -65,6 +67,8 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { save_volume_timer = null; original_volume = null; + @ViewChild('twitchchat') twitchChat: TwitchChatComponent; + @HostListener('window:resize', ['$event']) onResize(event) { this.innerWidth = window.innerWidth; @@ -270,6 +274,13 @@ export class PlayerComponent implements OnInit, AfterViewInit, OnDestroy { onPlayerReady(api: VgAPI) { this.api = api; + this.api_ready = true; + + this.api.subscriptions.seeked.subscribe(data => { + if (this.twitchChat) { + this.twitchChat.renewChat(); + } + }); // checks if volume has been previously set. if so, use that as default if (localStorage.getItem('player_volume')) { diff --git a/src/app/posts.services.ts b/src/app/posts.services.ts index 307c3c5..da479fb 100644 --- a/src/app/posts.services.ts +++ b/src/app/posts.services.ts @@ -234,6 +234,14 @@ export class PostsService implements CanActivate { return this.http.post(this.path + 'getAllFiles', {}, this.httpOptions); } + getFullTwitchChat(id, type, uuid = null) { + return this.http.post(this.path + 'getFullTwitchChat', {id: id, type: type, uuid: uuid}, this.httpOptions); + } + + downloadTwitchChat(id, type, vodId, uuid = null) { + return this.http.post(this.path + 'downloadTwitchChatByVODID', {id: id, type: type, vodId: vodId, uuid: uuid}, this.httpOptions); + } + downloadFileFromServer(fileName, type, outputName = null, fullPathProvided = null, subscriptionName = null, subPlaylist = null, uid = null, uuid = null, id = null) { return this.http.post(this.path + 'downloadFile', {fileNames: fileName, diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index f551a9d..08d4970 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -228,12 +228,24 @@
Use YouTube API
-
+
Generating a key is easy!
+
+ Use Twitch API +
+
+ Auto-download Twitch Chat +
+
+ + + Also known as a Client ID. Generating a key is easy! + +