diff --git a/Dockerfile b/Dockerfile index f5a76c3..d44fb63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM ubuntu:22.04 AS ffmpeg ENV DEBIAN_FRONTEND=noninteractive # Use script due local build compability -COPY ffmpeg-fetch.sh . +COPY docker-utils/ffmpeg-fetch.sh . RUN chmod +x ffmpeg-fetch.sh RUN sh ./ffmpeg-fetch.sh @@ -55,8 +55,10 @@ RUN npm install -g pm2 && \ apt install -y --no-install-recommends gosu python3-minimal python-is-python3 python3-pip atomicparsley build-essential && \ apt clean && \ rm -rf /var/lib/apt/lists/* -RUN pip install tdh-tcd pycryptodomex +RUN pip install pycryptodomex WORKDIR /app +COPY docker-utils/GetTwitchDownloader.py . +RUN python GetTwitchDownloader.py # User 1000 already exist from base image COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffmpeg", "/usr/local/bin/ffmpeg" ] COPY --chown=$UID:$GID --from=ffmpeg [ "/usr/local/bin/ffprobe", "/usr/local/bin/ffprobe" ] diff --git a/backend/config.js b/backend/config.js index d8de131..28b58c2 100644 --- a/backend/config.js +++ b/backend/config.js @@ -208,9 +208,6 @@ const DEFAULT_CONFIG = { "API_key": "", "use_youtube_API": false, "youtube_API_key": "", - "use_twitch_API": false, - "twitch_client_ID": "", - "twitch_client_secret": "", "twitch_auto_download_chat": false, "use_sponsorblock_API": false, "generate_NFO_files": false, diff --git a/backend/consts.js b/backend/consts.js index 714afcc..506d545 100644 --- a/backend/consts.js +++ b/backend/consts.js @@ -110,18 +110,6 @@ exports.CONFIG_ITEMS = { 'key': 'ytdl_youtube_api_key', 'path': 'YoutubeDLMaterial.API.youtube_API_key' }, - 'ytdl_use_twitch_api': { - 'key': 'ytdl_use_twitch_api', - 'path': 'YoutubeDLMaterial.API.use_twitch_API' - }, - 'ytdl_twitch_client_id': { - 'key': 'ytdl_twitch_client_id', - 'path': 'YoutubeDLMaterial.API.twitch_client_ID' - }, - 'ytdl_twitch_client_secret': { - 'key': 'ytdl_twitch_client_secret', - 'path': 'YoutubeDLMaterial.API.twitch_client_secret' - }, 'ytdl_twitch_auto_download_chat': { 'key': 'ytdl_twitch_auto_download_chat', 'path': 'YoutubeDLMaterial.API.twitch_auto_download_chat' diff --git a/backend/downloader.js b/backend/downloader.js index 050c088..11caa36 100644 --- a/backend/downloader.js +++ b/backend/downloader.js @@ -350,7 +350,7 @@ async function downloadQueuedFile(download_uid) { var file_name = filepath_no_extension.substring(fileFolderPath.length, filepath_no_extension.length); if (type === 'video' && url.includes('twitch.tv/videos/') && url.split('twitch.tv/videos/').length > 1 - && config_api.getConfigItem('ytdl_use_twitch_api') && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) { + && config_api.getConfigItem('ytdl_twitch_auto_download_chat')) { let vodId = url.split('twitch.tv/videos/')[1]; vodId = vodId.split('?')[0]; twitch_api.downloadTwitchChatByVODID(vodId, file_name, type, download['user_uid']); diff --git a/backend/test/tests.js b/backend/test/tests.js index 7bee912..980f4e7 100644 --- a/backend/test/tests.js +++ b/backend/test/tests.js @@ -508,7 +508,7 @@ describe('Downloader', function() { }); describe('Twitch', async function () { const twitch_api = require('../twitch'); - const example_vod = '1493770675'; + const example_vod = '1710641401'; it('Download VOD', async function() { const sample_path = path.join('test', 'sample.twitch_chat.json'); if (fs.existsSync(sample_path)) fs.unlinkSync(sample_path); diff --git a/backend/twitch.js b/backend/twitch.js index c713676..de2b1de 100644 --- a/backend/twitch.js +++ b/backend/twitch.js @@ -4,19 +4,28 @@ const logger = require('./logger'); const moment = require('moment'); const fs = require('fs-extra') const path = require('path'); +const { promisify } = require('util'); +const child_process = require('child_process'); -async function getCommentsForVOD(clientID, clientSecret, vodId) { - const { promisify } = require('util'); - const child_process = require('child_process'); +async function getCommentsForVOD(vodId) { const exec = promisify(child_process.exec); // Reject invalid params to prevent command injection attack - if (!clientID.match(/^[0-9a-z]+$/) || !clientSecret.match(/^[0-9a-z]+$/) || !vodId.match(/^[0-9a-z]+$/)) { - logger.error('Client ID, client secret, and VOD ID must be purely alphanumeric. Twitch chat download failed!'); + if (!vodId.match(/^[0-9a-z]+$/)) { + logger.error('VOD ID must be purely alphanumeric. Twitch chat download failed!'); return null; } - const result = await exec(`tcd --video ${vodId} --client-id ${clientID} --client-secret ${clientSecret} --format json -o appdata`, {stdio:[0,1,2]}); + const is_windows = process.platform === 'win32'; + const cliExt = is_windows ? '.exe' : '' + const cliPath = `TwitchDownloaderCLI${cliExt}` + + if (!fs.existsSync(cliPath)) { + logger.error(`${cliPath} does not exist. Twitch chat download failed!`); + return null; + } + + const result = await exec(`TwitchDownloaderCLI chatdownload -u ${vodId} -o appdata/${vodId}.json`, {stdio:[0,1,2]}); if (result['stderr']) { logger.error(`Failed to download twitch comments for ${vodId}`); @@ -73,9 +82,7 @@ async function getTwitchChatByFileID(id, type, user_uid, uuid, sub) { async function downloadTwitchChatByVODID(vodId, id, type, user_uid, sub, customFileFolderPath = null) { const usersFileFolder = config_api.getConfigItem('ytdl_users_base_path'); const subscriptionsFileFolder = config_api.getConfigItem('ytdl_subscriptions_base_path'); - const twitch_client_id = config_api.getConfigItem('ytdl_twitch_client_id'); - const twitch_client_secret = config_api.getConfigItem('ytdl_twitch_client_secret'); - const chat = await getCommentsForVOD(twitch_client_id, twitch_client_secret, vodId); + const chat = await getCommentsForVOD(vodId); // save file if needed params are included let file_path = null; diff --git a/docker-utils/GetTwitchDownloader.py b/docker-utils/GetTwitchDownloader.py new file mode 100644 index 0000000..3229f3a --- /dev/null +++ b/docker-utils/GetTwitchDownloader.py @@ -0,0 +1,53 @@ +import platform +import requests +import shutil +import os +import re + +from github import Github + +machine = platform.machine() + +def isARM(): + return True if machine.startswith('arm') else False + +def getLatestFileInRepo(repo, search_string): + # Create an unauthenticated instance of the Github object + g = Github(os.environ.get('GH_TOKEN')) + + # Replace with the repository owner and name + repo = g.get_repo(repo) + + # Get all releases of the repository + releases = repo.get_releases() + + # Loop through the releases in reverse order (from latest to oldest) + for release in list(releases): + # Get the release assets (files attached to the release) + assets = release.get_assets() + + # Loop through the assets + for asset in assets: + if re.search(search_string, asset.name): + print(f'Downloading: {asset.name}') + response = requests.get(asset.browser_download_url) + with open(asset.name, 'wb') as f: + f.write(response.content) + print(f'Download complete: {asset.name}. Unzipping...') + shutil.unpack_archive(asset.name, './') + print(f'Unzipping complete!') + os.remove(asset.name) + break + else: + continue + break + else: + # If no matching release is found, print a message + print(f'No release found with {search_string}') + +def getLatestCLIRelease(): + isArm = isARM() + searchString = r'.*CLI.*' + "LinuxArm.zip" if isArm else "Linux-x64.zip" + getLatestFileInRepo("lay295/TwitchDownloader", searchString) + +getLatestCLIRelease() \ No newline at end of file diff --git a/ffmpeg-fetch.sh b/docker-utils/ffmpeg-fetch.sh similarity index 100% rename from ffmpeg-fetch.sh rename to docker-utils/ffmpeg-fetch.sh diff --git a/src/app/player/player.component.html b/src/app/player/player.component.html index 4d0769f..3b12c98 100644 --- a/src/app/player/player.component.html +++ b/src/app/player/player.component.html @@ -38,7 +38,7 @@ - + @@ -51,7 +51,7 @@ - + diff --git a/src/app/settings/settings.component.html b/src/app/settings/settings.component.html index ee8f5eb..f95bbc6 100644 --- a/src/app/settings/settings.component.html +++ b/src/app/settings/settings.component.html @@ -269,25 +269,9 @@ Generating a key is easy! -
- Use Twitch API -
-
+
Auto-download Twitch Chat
-
- - Twitch Client ID - - Generating an ID/secret is easy! - -
-
- - Twitch Client Secret - - -
Use SponsorBlock API