aiden
/
YoutubeDL-Material
mirror of https://github.com/Tzahi12345/YoutubeDL-Material
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed twitch chat downloads, tcd is now required and client secret must be supplied

Browse Source
pull/657/head
Isaac Abadi 3 years ago
parent cbdd1a6253
commit 7bfb2976fe
6 changed files with 92 additions and 90 deletions
Show Stats Download Patch File Download Diff File

3
backend/appdata/default.json
Unescape Escape View File

@ -31,7 +31,8 @@
"use_youtube_API": false,
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_API_key": "",
"twitch_client_ID": "",
"twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false

3
backend/config.js
Unescape Escape View File

@ -206,7 +206,8 @@ const DEFAULT_CONFIG = {
"use_youtube_API": false,
"youtube_API_key": "",
"use_twitch_API": false,
"twitch_API_key": "",
"twitch_client_ID": "",
"twitch_client_secret": "",
"twitch_auto_download_chat": false,
"use_sponsorblock_API": false,
"generate_NFO_files": false

10
backend/consts.js
Unescape Escape View File

@ -102,9 +102,13 @@ exports.CONFIG_ITEMS = {
'key': 'ytdl_use_twitch_api',
'path': 'YoutubeDLMaterial.API.use_twitch_API'
},
'ytdl_twitch_api_key': {
'key': 'ytdl_twitch_api_key',
'path': 'YoutubeDLMaterial.API.twitch_API_key'
'ytdl_twitch_client_id': {
'key': 'ytdl_twitch_client_id',
'path': 'YoutubeDLMaterial.API.twitch_client_ID'
},
'ytdl_twitch_client_secret': {
'key': 'ytdl_twitch_client_secret',
'path': 'YoutubeDLMaterial.API.twitch_client_secret'
},
'ytdl_twitch_auto_download_chat': {
'key': 'ytdl_twitch_auto_download_chat',

19
backend/test/tests.js
Unescape Escape View File

@ -1,6 +1,7 @@
var assert = require('assert');
const assert = require('assert');
const low = require('lowdb')
var winston = require('winston');
const winston = require('winston');
const path = require('path');
process.chdir('./backend')
@ -465,6 +466,20 @@ describe('Downloader', function() {
console.log(updated_args2);
assert(JSON.stringify(updated_args2), JSON.stringify(expected_args2));
});
describe('Twitch', async function () {
const twitch_api = require('../twitch');
const example_vod = '1493770675';
it('Download VOD', async function() {
const sample_path = path.join('test', 'sample.twitch_chat.json');
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
this.timeout(300000);
await twitch_api.downloadTwitchChatByVODID(example_vod, 'sample', null, null, null, './test');
assert(fs.existsSync(sample_path));
// cleanup
if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path);
});
});
});
describe('Tasks', function() {

136
backend/twitch.js
Unescape Escape View File

@ -1,90 +1,53 @@
var moment = require('moment');
var Axios = require('axios');
var fs = require('fs-extra')
var path = require('path');
const config_api = require('./config');
async function getCommentsForVOD(clientID, vodId) {
let url = `https://api.twitch.tv/v5/videos/${vodId}/comments?content_offset_seconds=0`,
batch,
cursor;
let comments = null;
try {
do {
batch = (await Axios.get(url, {
headers: {
'Client-ID': clientID,
Accept: 'application/vnd.twitchtv.v5+json; charset=UTF-8',
'Content-Type': 'application/json; charset=UTF-8',
}
})).data;
const str = batch.comments.map(c => {
let {
created_at: msgCreated,
content_offset_seconds: timestamp,
commenter: {
name,
_id,
created_at: acctCreated
},
message: {
body: msg,
user_color: user_color
}
} = c;
const timestamp_str = moment.duration(timestamp, 'seconds')
.toISOString()
.replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
(_, ...ms) => {
const seg = v => v ? v.padStart(2, '0') : '00';
return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
});
acctCreated = moment(acctCreated).utc();
msgCreated = moment(msgCreated).utc();
if (!comments) comments = [];
comments.push({
timestamp: timestamp,
timestamp_str: timestamp_str,
name: name,
message: msg,
user_color: user_color
});
// let line = `${timestamp},${msgCreated.format(tsFormat)},${name},${_id},"${msg.replace(/"/g, '""')}",${acctCreated.format(tsFormat)}`;
// return line;
}).join('\n');
cursor = batch._next;
url = `https://api.twitch.tv/v5/videos/${vodId}/comments?cursor=${cursor}`;
await new Promise(res => setTimeout(res, 300));
} while (cursor);
} catch (err) {
console.error(err);
const logger = require('./logger');
const moment = require('moment');
const fs = require('fs-extra')
const path = require('path');
async function getCommentsForVOD(clientID, clientSecret, vodId) {
const { promisify } = require('util');
const child_process = require('child_process');
const exec = promisify(child_process.exec);
const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]});
if (result['stderr']) {
logger.error(`Failed to download twitch comments for ${vodId}`);
logger.error(result['stderr']);
return null;
}
return comments;
const raw_json = fs.readJSONSync(path.join('appdata', `${vodId}.json`));
const new_json = raw_json.comments.map(comment_obj => {
return {
timestamp: comment_obj.content_offset_seconds,
timestamp_str: convertTimestamp(comment_obj.content_offset_seconds),
name: comment_obj.commenter.name,
message: comment_obj.message.body,
user_color: comment_obj.message.user_color
}
});
return new_json;
}
async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
let file_path = null;
if (user_uid) {
if (sub) {
file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
} else {
file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
}
} else {
if (sub) {
file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
} else {
file_path = path.join(type, id + '.twitch_chat.json');
const typeFolder = config_api.getConfigItem(`ytdl_${type}_folder_path`);
file_path = path.join(typeFolder, `${id}.twitch_chat.json`);
}
}
@ -96,23 +59,28 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) {
return chat_file;
}
async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
const twitch_api_key = config_api.getConfigItem('ytdl_twitch_api_key');
const chat = await getCommentsForVOD(twitch_api_key, vodId);
async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) {
const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path');
const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path');
const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id');
const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret');
const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId);
// save file if needed params are included
let file_path = null;
if (user_uid) {
if (customFileFolderPath) {
file_path = path.join(customFileFolderPath, `${id}.twitch_chat.json`)
} else if (user_uid) {
if (sub) {
file_path = path.join('users', user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
file_path = path.join(usersFileFolder, user_uid, 'subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
} else {
file_path = path.join('users', user_uid, type, id + '.twitch_chat.json');
file_path = path.join(usersFileFolder, user_uid, type, `${id}.twitch_chat.json`);
}
} else {
if (sub) {
file_path = path.join('subscriptions', sub.isPlaylist ? 'playlists' : 'channels', sub.name, id + '.twitch_chat.json');
file_path = path.join(subscriptionsFileFolder, sub.isPlaylist ? 'playlists' : 'channels', sub.name, `${id}.twitch_chat.json`);
} else {
file_path = path.join(type, id + '.twitch_chat.json');
file_path = path.join(type, `${id}.twitch_chat.json`);
}
}
@ -121,6 +89,14 @@ async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub) {
return chat;
}
const convertTimestamp = (timestamp) => moment.duration(timestamp, 'seconds')
.toISOString()
.replace(/P.*?T(?:(\d+?)H)?(?:(\d+?)M)?(?:(\d+).*?S)?/,
(_, ...ms) => {
const seg = v => v ? v.padStart(2, '0') : '00';
return `${seg(ms[0])}:${seg(ms[1])}:${seg(ms[2])}`;
});
module.exports = {
getCommentsForVOD: getCommentsForVOD,
getTwitchChatByFileID: getTwitchChatByFileID,

11
src/app/settings/settings.component.html
Unescape Escape View File

@ -263,11 +263,16 @@
</div>
<div class="col-12">
<mat-form-field class="text-field" color="accent">
<input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_API_key']" matInput placeholder="Twitch API Key" i18n-placeholder="Twitch API Key setting placeholder" required>
<mat-hint><ng-container i18n="Twitch API Key setting hint AKA preamble">Also known as a Client ID.</ng-container>&nbsp;<a target="_blank" href="https://dev.twitch.tv/docs/api/"><ng-container i18n="Twitch API Key setting hint">Generating a key is easy!</ng-container></a></mat-hint>
<input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_client_ID']" matInput placeholder="Twitch Client ID" i18n-placeholder="Twitch Client ID setting placeholder" required>
<mat-hint><a target="_blank" href="https://dev.twitch.tv/docs/api/"><ng-container i18n="Twitch Client ID setting hint">Generating an ID/secret is easy!</ng-container></a></mat-hint>
</mat-form-field>
</div>
<div class="col-12 mt-4">
<div class="col-12 mt-2">
<mat-form-field class="text-field" color="accent">
<input [disabled]="!new_config['API']['use_twitch_API']" [(ngModel)]="new_config['API']['twitch_client_secret']" matInput placeholder="Twitch Client Secret" i18n-placeholder="Twitch Client Secret setting placeholder" required>
</mat-form-field>
</div>
<div class="col-12 mt-2">
<mat-checkbox color="accent" [(ngModel)]="new_config['API']['use_sponsorblock_API']" matTooltip="Enables a button to skip ads when viewing supported videos." i18n-matTooltip="SponsorBlock API tooltip"><ng-container i18n="Use SponsorBlock API setting">Use SponsorBlock API</ng-container></mat-checkbox>
</div>
<div class="col-12 mt-2 mb-3">

Write Preview
Loading…
Cancel
Save