diff --git a/backend/appdata/default.json b/backend/appdata/default.json
index 44f2f02..639e269 100644
--- a/backend/appdata/default.json
+++ b/backend/appdata/default.json
@@ -31,7 +31,8 @@
"use_youtube_API": false,
"youtube_API_key": "",
"use_twitch_API": false,
- "twitch_API_key": "",
+ "twitch_client_ID": "",
+ "twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
diff --git a/backend/config.js b/backend/config.js
index 4e208fb..27faab8 100644
--- a/backend/config.js
+++ b/backend/config.js
@@ -206,7 +206,8 @@ const DEFAULT_CONFIG = {
"use_youtube_API": false,
"youtube_API_key": "",
"use_twitch_API": false,
- "twitch_API_key": "",
+ "twitch_client_ID": "",
+ "twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false
diff --git a/backend/consts.js b/backend/consts.js
index 222d6bc..f473e07 100644
--- a/backend/consts.js
+++ b/backend/consts.js
@@ -102,9 +102,13 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_use_twitch_api',
'path': 'YoutubeDLMaterial.API.use_twitch_API'
},
- 'ytdl_twitch_api_key': {
- 'key': 'ytdl_twitch_api_key',
- 'path': 'YoutubeDLMaterial.API.twitch_API_key'
+ 'ytdl_twitch_client_id': {
+ 'key': 'ytdl_twitch_client_id',
+ 'path': 'YoutubeDLMaterial.API.twitch_client_ID'
+ },
+ 'ytdl_twitch_client_secret': {
+ 'key': 'ytdl_twitch_client_secret',
+ 'path': 'YoutubeDLMaterial.API.twitch_client_secret'
},
'ytdl_twitch_auto_download_chat': {
'key': 'ytdl_twitch_auto_download_chat',
diff --git a/backend/test/tests.js b/backend/test/tests.js
index c87045a..9236a42 100644
--- a/backend/test/tests.js
+++ b/backend/test/tests.js
@@ -1,6 +1,7 @@
-var assert = require('assert');
+const assert = require('assert');
const low = require('lowdb')
-var winston = require('winston');
+const winston = require('winston');
+const path = require('path');
process.chdir('./backend')
@@ -465,6 +466,20 @@ describe('Downloader', function() {
console.log(updated_args2);
assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
});
+ describe('Twitch', async function () {
+ const twitch_api = require('../twitch');
+ const example_vod = '1493770675';
+ it('Download VOD', async function() {
+ const sample_path = path.join('test', 'sample.twitch_chat.json');
+ if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+ this.timeout(300000);
+ await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
+ assert(fs.existsSync(sample_path));
+
+ // cleanup
+ if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
+ });
+ });
});
describe('Tasks', function() {
diff --git a/backend/twitch.js b/backend/twitch.js
index 2a231e9..151e177 100644
--- a/backend/twitch.js
+++ b/backend/twitch.js
@@ -1,90 +1,53 @@
-var moment = require('moment');
-var Axios = require('axios');
-var fs = require('fs-extra')
-var path = require('path');
const config_api = require('./config');
-
-async function getCommentsForVOD(clientID, vodId) {
- let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
- batch,
- cursor;
-
- let comments = null;
-
- try {
- do {
- batch = (await Axios.get(url, {
- headers: {
- 'Client-ID': clientID,
- Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
- 'Content-Type': 'application/json; charset=UTF-8',
- }
- })).data;
-
- const str = batch.comments.map(c => {
- let {
- created_at: msgCreated,
- content_offset_seconds: timestamp,
- commenter: {
- name,
- _id,
- created_at: acctCreated
- },
- message: {
- body: msg,
- user_color: user_color
- }
- } = c;
-
- const timestamp_str = moment.duration(timestamp, 'seconds')
- .toISOString()
- .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
- (_, ...ms) => {
- const seg = v => v ? v.padStart(2, '0') : '00';
- return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
- });
-
- acctCreated = moment(acctCreated).utc();
- msgCreated = moment(msgCreated).utc();
-
- if (!comments) comments = [];
-
- comments.push({
- timestamp: timestamp,
- timestamp_str: timestamp_str,
- name: name,
- message: msg,
- user_color: user_color
- });
- // let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
- // return line;
- }).join('\n');
-
- cursor = batch._next;
- url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
- await new Promise(res => setTimeout(res, 300));
- } while (cursor);
- } catch (err) {
- console.error(err);
+const logger = require('./logger');
+
+const moment = require('moment');
+const fs = require('fs-extra')
+const path = require('path');
+
+async function getCommentsForVOD(clientID, clientSecret, vodId) {
+ const { promisify } = require('util');
+ const child_process = require('child_process');
+ const exec = promisify(child_process.exec);
+ const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
+
+ if (result['stderr']) {
+ logger.error(`Failed to download twitch comments for ${vodId}`);
+ logger.error(result['stderr']);
+ return null;
}
- return comments;
+ const raw_json = fs.readJSONSync(path.join('appdata', `${vodId}.json`));
+ const new_json = raw_json.comments.map(comment_obj => {
+ return {
+ timestamp: comment_obj.content_offset_seconds,
+ timestamp_str: convertTimestamp(comment_obj.content_offset_seconds),
+ name: comment_obj.commenter.name,
+ message: comment_obj.message.body,
+ user_color: comment_obj.message.user_color
+ }
+ });
+
+ return new_json;
}
async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
+ const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+ const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
let file_path = null;
if (user_uid) {
if (sub) {
- file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+ file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
} else {
- file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+ file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
}
} else {
if (sub) {
- file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+ file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
} else {
- file_path = path.join(type, id + '.twitch_chat.json');
+ const typeFolder = config_api.getConfigItem(`ytdl_${type}_folder_path`);
+ file_path = path.join(typeFolder, `${id}.twitch_chat.json`);
}
}
@@ -96,23 +59,28 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
return chat_file;
}
-async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
- const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
- const chat = await getCommentsForVOD(twitch_api_key, vodId);
+async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
+ const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
+ const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
+ const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
+ const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
+ const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
// save file if needed params are included
let file_path = null;
- if (user_uid) {
+ if (customFileFolderPath) {
+ file_path = path.join(customFileFolderPath, `${id}.twitch_chat.json`)
+ } else if (user_uid) {
if (sub) {
- file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+ file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
} else {
- file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
+ file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
}
} else {
if (sub) {
- file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
+ file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
} else {
- file_path = path.join(type, id + '.twitch_chat.json');
+ file_path = path.join(type, `${id}.twitch_chat.json`);
}
}
@@ -121,6 +89,14 @@ async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
return chat;
}
+const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
+ .toISOString()
+ .replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
+ (_, ...ms) => {
+ const seg = v => v ? v.padStart(2, '0') : '00';
+ return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
+});
+
module.exports = {
getCommentsForVOD: getCommentsForVOD,
getTwitchChatByFileID: getTwitchChatByFileID,
diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html
index aa453a4..24c1c0c 100644
--- a/src/app/settings/settings.component.html
+++ b/src/app/settings/settings.component.html
@@ -263,11 +263,16 @@
-
+
+
+
+
+
+
Use SponsorBlock API