From ef5bd3a7b202dd2698271684669db51dfff2922f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 03:12:57 +0300 Subject: [PATCH 01/23] Delete clean.js --- scripts/clean.js | 74 ------------------------------------------------ 1 file changed, 74 deletions(-) delete mode 100644 scripts/clean.js diff --git a/scripts/clean.js b/scripts/clean.js deleted file mode 100644 index d1ab7c2e7e..0000000000 --- a/scripts/clean.js +++ /dev/null @@ -1,74 +0,0 @@ -const IPTVChecker = require('iptv-checker') -const { program } = require('commander') -const ProgressBar = require('progress') -const parser = require('./helpers/parser') -const utils = require('./helpers/utils') -const log = require('./helpers/log') - -program - .usage('[OPTIONS]...') - .option('-d, --debug', 'Enable debug mode') - .option('-c, --country ', 'Comma-separated list of country codes', '') - .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .option('--timeout ', 'Set timeout for each request', 5000) - .parse(process.argv) - -let bar -const config = program.opts() -const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline'] -const checker = new IPTVChecker({ - timeout: config.timeout -}) - -async function main() { - log.start() - - if (config.debug) log.print(`Debug mode enabled\n`) - - let playlists = parser.parseIndex() - playlists = utils.filterPlaylists(playlists, config.country, config.exclude) - for (const playlist of playlists) { - await parser - .parsePlaylist(playlist.url) - .then(checkPlaylist) - .then(p => p.save()) - } - - log.finish() -} - -async function checkPlaylist(playlist) { - if (!config.debug) { - bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, { - total: playlist.channels.length - }) - } - const channels = [] - const total = playlist.channels.length - for (const [index, channel] of playlist.channels.entries()) { - const skipChannel = - channel.status && - ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) - if (skipChannel) { - channels.push(channel) - } else { - const result = await checker.checkStream(channel.data) - if (result.status.ok || result.status.reason.includes('timed out')) { - channels.push(channel) - } else { - if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`) - } - } - if (!config.debug) bar.tick() - } - - if (playlist.channels.length !== channels.length) { - log.print(`File '${playlist.url}' has been updated\n`) - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -main() From e28bb1792ad8d755438a58aa5ed7ce8a2607263e Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 03:13:00 +0300 Subject: [PATCH 02/23] Delete detect-resolution.js --- scripts/detect-resolution.js | 114 ----------------------------------- 1 file changed, 114 deletions(-) delete mode 100644 scripts/detect-resolution.js diff --git a/scripts/detect-resolution.js b/scripts/detect-resolution.js deleted file mode 100644 index 5da6a282f5..0000000000 --- a/scripts/detect-resolution.js +++ /dev/null @@ -1,114 +0,0 @@ -const { program } = require('commander') -const ProgressBar = require('progress') -const axios = require('axios') -const https = require('https') -const parser = require('./helpers/parser') -const utils = require('./helpers/utils') -const log = require('./helpers/log') - -program - .usage('[OPTIONS]...') - .option('-c, --country ', 'Comma-separated list of country codes', '') - .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .option('--delay ', 'Delay between parser requests', 1000) - .option('--timeout ', 'Set timeout for each request', 5000) - .parse(process.argv) - -const config = program.opts() -const ignoreStatus = ['Offline'] -const instance = axios.create({ - timeout: config.timeout, - maxContentLength: 200000, - httpsAgent: new https.Agent({ - rejectUnauthorized: false - }) -}) - -async function main() { - log.start() - - log.print(`Parsing 'index.m3u'...\n`) - let playlists = parser.parseIndex() - playlists = utils - .filterPlaylists(playlists, config.country, config.exclude) - .filter(i => i.url !== 'channels/unsorted.m3u') - - for (const playlist of playlists) { - await parser - .parsePlaylist(playlist.url) - .then(detectResolution) - .then(p => p.save()) - } - - log.finish() -} - -async function detectResolution(playlist) { - const channels = [] - const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { - total: playlist.channels.length - }) - let updated = false - for (const channel of playlist.channels) { - bar.tick() - const skipChannel = - channel.status && - ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) - if (!channel.resolution.height && !skipChannel) { - const CancelToken = axios.CancelToken - const source = CancelToken.source() - const timeout = setTimeout(() => { - source.cancel() - }, config.timeout) - - const response = await instance - .get(channel.url, { cancelToken: source.token }) - .then(res => { - clearTimeout(timeout) - - return res - }) - .then(utils.sleep(config.delay)) - .catch(err => { - clearTimeout(timeout) - }) - - if (response && response.status === 200) { - if (/^#EXTM3U/.test(response.data)) { - const resolution = parseResolution(response.data) - if (resolution) { - channel.resolution = resolution - updated = true - } - } - } - } - - channels.push(channel) - } - - if (updated) { - log.print(`File '${playlist.url}' has been updated\n`) - playlist.channels = channels - playlist.updated = true - } - - return playlist -} - -function parseResolution(string) { - const regex = /RESOLUTION=(\d+)x(\d+)/gm - const match = string.matchAll(regex) - const arr = Array.from(match).map(m => ({ - width: parseInt(m[1]), - height: parseInt(m[2]) - })) - - return arr.length - ? arr.reduce(function (prev, current) { - return prev.height > current.height ? prev : current - }) - : undefined -} - -main() From 895284ea924264bcc3cd49508d8be78fc6ae03bb Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 03:13:05 +0300 Subject: [PATCH 03/23] Update format.js --- scripts/format.js | 112 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 89 insertions(+), 23 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index fb77cab20f..63ec199fd3 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -1,21 +1,44 @@ +const IPTVChecker = require('iptv-checker') +const { program } = require('commander') +const ProgressBar = require('progress') const parser = require('./helpers/parser') const utils = require('./helpers/utils') const file = require('./helpers/file') const log = require('./helpers/log') +program + .usage('[OPTIONS]...') + .option('-d, --debug', 'Enable debug mode') + .option('-s, --status', 'Update stream status') + .option('-r, --resolution', 'Detect stream resolution') + .option('-c, --country ', 'Comma-separated list of country codes', '') + .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') + .option('--timeout ', 'Set timeout for each request', 5000) + .parse(process.argv) + +let bar +const ignoreStatus = ['Geo-blocked', 'Not 24/7'] +const config = program.opts() +const checker = new IPTVChecker({ + timeout: config.timeout +}) + async function main() { log.start() - log.print(`Parsing 'index.m3u'...`) + if (config.debug) log.print(`Debug mode enabled\n`) + if (config.status) log.print(`Updating channel status...\n`) + if (config.resolution) log.print(`Detecting channel resolution...\n`) + let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') + playlists = utils.filterPlaylists(playlists, config.country, config.exclude) for (const playlist of playlists) { - log.print(`\nProcessing '${playlist.url}'...`) await parser .parsePlaylist(playlist.url) - .then(formatPlaylist) + .then(updatePlaylist) .then(playlist => { if (file.read(playlist.url) !== playlist.toString()) { - log.print('updated') + log.print(`File '${playlist.url}' has been updated\n`) playlist.updated = true } @@ -23,33 +46,76 @@ async function main() { }) } - log.print('\n') log.finish() } -async function formatPlaylist(playlist) { +async function updatePlaylist(playlist) { + if (!config.debug) { + bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + } + for (const channel of playlist.channels) { - const code = file.getBasename(playlist.url) - // add missing tvg-name - if (!channel.tvg.name && code !== 'unsorted' && channel.name) { - channel.tvg.name = channel.name.replace(/\"/gi, '') + addMissingData(channel) + const checkOnline = config.status || config.resolution + const skip = + channel.status && + ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) + if (checkOnline && !skip) { + await checker + .checkStream(channel.data) + .then(result => { + if (result.status.ok || result.status.reason.includes('timed out')) { + if (config.status) updateStatus(channel, null) + if (config.resolution) updateResolution(channel, result.status.metadata) + } else { + if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`) + if (config.status) updateStatus(channel, 'Offline') + } + }) + .catch(err => { + if (config.debug) log.print(`ERR: ${channel.url} (${err.message})\n`) + }) } - // add missing tvg-id - if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) { - const id = utils.name2id(channel.tvg.name) - channel.tvg.id = id ? `${id}.${code}` : '' - } - // add missing country - if (!channel.countries.length) { - const name = utils.code2name(code) - channel.countries = name ? [{ code, name }] : [] - channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') - } - // update group-title - channel.group.title = channel.category + if (!config.debug) bar.tick() } return playlist } +function addMissingData(channel) { + // add tvg-name + if (!channel.tvg.name && channel.name) { + channel.tvg.name = channel.name.replace(/\"/gi, '') + } + // add tvg-id + if (!channel.tvg.id && channel.tvg.name) { + const id = utils.name2id(channel.tvg.name) + channel.tvg.id = id ? `${id}.${code}` : '' + } + // add country + if (!channel.countries.length) { + const name = utils.code2name(code) + channel.countries = name ? [{ code, name }] : [] + channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') + } + // update group-title + channel.group.title = channel.category +} + +function updateStatus(channel, status) { + channel.status = status +} + +function updateResolution(channel, metadata) { + const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : [] + if (!channel.resolution.height && streams.length) { + channel.resolution = streams.reduce((acc, curr) => { + if (curr.height > acc.height) return { width: curr.width, height: curr.height } + return acc + }) + } +} + main() From 6a08bceddd3ac840efc90e47fd74c95124edeb35 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 03:34:13 +0300 Subject: [PATCH 04/23] Update auto-update.yml --- .github/workflows/auto-update.yml | 159 +++++++++++++----------------- 1 file changed, 69 insertions(+), 90 deletions(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 213deb39b0..b50763ec26 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -20,90 +20,6 @@ jobs: format: runs-on: ubuntu-latest needs: create-branch - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Format Playlists - run: node scripts/format.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Formate playlists' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - sort: - runs-on: ubuntu-latest - needs: format - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Sort Channels - run: node scripts/sort.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Sort channels' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - remove-duplicates: - runs-on: ubuntu-latest - needs: sort - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Remove Duplicates - run: node scripts/remove-duplicates.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Remove duplicates' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - filter: - runs-on: ubuntu-latest - needs: remove-duplicates - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Filter Playlists - run: node scripts/filter.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Filter channels' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - detect-resolution: - runs-on: ubuntu-latest - needs: filter continue-on-error: true strategy: fail-fast: false @@ -279,8 +195,8 @@ jobs: ref: bot/auto-update - name: Install Dependencies run: npm install - - name: Detect Resolution - run: node scripts/detect-resolution.js --country=${{ matrix.country }} + - name: Format Playlists + run: node scripts/format.js --country=${{ matrix.country }} --status --resolution --debug - name: Upload Artifact uses: actions/upload-artifact@v2 with: @@ -288,7 +204,7 @@ jobs: path: channels/${{ matrix.country }}.m3u commit-changes: runs-on: ubuntu-latest - needs: detect-resolution + needs: format steps: - name: Checkout uses: actions/checkout@v2 @@ -302,15 +218,78 @@ jobs: - name: Commit Changes uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: '[Bot] Detect resolution' + commit_message: '[Bot] Format playlists' commit_user_name: iptv-bot commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' branch: bot/auto-update file_pattern: channels/* - generate: + remove-duplicates: runs-on: ubuntu-latest needs: commit-changes + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Remove Duplicates + run: node scripts/remove-duplicates.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove duplicates' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + sort: + runs-on: ubuntu-latest + needs: remove-duplicates + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Sort Channels + run: node scripts/sort.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Sort channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + filter: + runs-on: ubuntu-latest + needs: sort + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Filter Playlists + run: node scripts/filter.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Filter channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + generate: + runs-on: ubuntu-latest + needs: filter steps: - name: Checkout uses: actions/checkout@v2 @@ -376,6 +355,7 @@ jobs: branch: bot/auto-update file_pattern: README.md pull-request: + if: ${{ github.ref == 'refs/heads/master' }} needs: update-readme runs-on: ubuntu-latest steps: @@ -408,7 +388,6 @@ jobs: pull-request-number: ${{ steps.pr.outputs.pr_number }} merge-method: squash - name: Approve Pull Request - if: github.ref == 'refs/heads/master' uses: juliangruber/approve-pull-request-action@v1 with: github-token: ${{ secrets.PAT }} From 355be19ebe238cb6f044d7872998435fb9deb7c7 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 03:39:05 +0300 Subject: [PATCH 05/23] Update format.js --- scripts/format.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index 63ec199fd3..b7cce4d0e3 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -54,6 +54,8 @@ async function updatePlaylist(playlist) { bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { total: playlist.channels.length }) + } else { + log.print(`Processing '${playlist.url}'...\n`) } for (const channel of playlist.channels) { @@ -70,12 +72,12 @@ async function updatePlaylist(playlist) { if (config.status) updateStatus(channel, null) if (config.resolution) updateResolution(channel, result.status.metadata) } else { - if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`) + if (config.debug) log.print(` ${channel.url} (${result.status.reason})\n`) if (config.status) updateStatus(channel, 'Offline') } }) .catch(err => { - if (config.debug) log.print(`ERR: ${channel.url} (${err.message})\n`) + if (config.debug) log.print(` ${channel.url} (${err.message})\n`) }) } if (!config.debug) bar.tick() From b480397ddb72ad5b708be21e8eb7d7f511ff356b Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 03:47:11 +0300 Subject: [PATCH 06/23] Update format.js --- scripts/format.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index b7cce4d0e3..cb3b407d03 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -72,12 +72,12 @@ async function updatePlaylist(playlist) { if (config.status) updateStatus(channel, null) if (config.resolution) updateResolution(channel, result.status.metadata) } else { - if (config.debug) log.print(` ${channel.url} (${result.status.reason})\n`) + if (config.debug) log.print(` ERR: ${channel.url} (${result.status.reason})\n`) if (config.status) updateStatus(channel, 'Offline') } }) .catch(err => { - if (config.debug) log.print(` ${channel.url} (${err.message})\n`) + if (config.debug) log.print(` ERR: ${channel.url} (${err.message})\n`) }) } if (!config.debug) bar.tick() From 4584f31c8f290662927c493f687dcfd80c082ed5 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 03:58:51 +0300 Subject: [PATCH 07/23] Update auto-update.yml --- .github/workflows/auto-update.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index b50763ec26..51cf6df50f 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -193,6 +193,8 @@ jobs: uses: actions/checkout@v2 with: ref: bot/auto-update + - name: Setup FFmpeg + uses: FedericoCarboni/setup-ffmpeg@v1 - name: Install Dependencies run: npm install - name: Format Playlists From e365166b76b1da63b5795536a6bf7cc774f5a8d4 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 04:04:44 +0300 Subject: [PATCH 08/23] Delete clean.yml --- .github/workflows/clean.yml | 257 ------------------------------------ 1 file changed, 257 deletions(-) delete mode 100644 .github/workflows/clean.yml diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml deleted file mode 100644 index 3c1f9a4520..0000000000 --- a/.github/workflows/clean.yml +++ /dev/null @@ -1,257 +0,0 @@ -name: clean -on: - workflow_dispatch: - schedule: - - cron: '0 6 * * 0' -jobs: - create-branch: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: ${{ github.ref }} - - name: Create Branch - uses: peterjgrainger/action-create-branch@v2.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - branch: 'bot/remove-broken-links' - check: - runs-on: ubuntu-latest - needs: create-branch - continue-on-error: true - strategy: - fail-fast: false - matrix: - country: - [ - ad, - ae, - af, - ag, - al, - am, - an, - ao, - ar, - at, - au, - aw, - az, - ba, - bb, - bd, - be, - bf, - bg, - bh, - bn, - bo, - br, - bs, - by, - ca, - cd, - cg, - ch, - ci, - cl, - cm, - cn, - co, - cr, - cu, - cw, - cy, - cz, - de, - dk, - do, - dz, - ec, - ee, - eg, - es, - et, - fi, - fj, - fo, - fr, - pf, - ge, - gh, - gm, - gn, - gp, - gq, - gr, - gt, - hk, - hn, - hr, - ht, - hu, - id, - ie, - il, - in, - iq, - ir, - is, - it, - jm, - jo, - jp, - ke, - kg, - kh, - kp, - kr, - kw, - kz, - la, - lb, - li, - lk, - lt, - lu, - lv, - ly, - ma, - mc, - md, - me, - mk, - ml, - mm, - mn, - mo, - mt, - mv, - mx, - my, - mz, - ne, - ng, - ni, - nl, - no, - np, - nz, - om, - pa, - pe, - ph, - pk, - pl, - pr, - ps, - pt, - py, - qa, - ro, - rs, - ru, - rw, - sa, - sd, - se, - sg, - si, - sk, - sl, - sm, - sn, - so, - sv, - sy, - th, - tj, - tm, - tn, - tr, - tt, - tw, - tz, - ua, - ug, - uk, - us, - uy, - uz, - va, - ve, - vi, - vn, - xk, - ye, - zm, - unsorted - ] - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/remove-broken-links - - name: Setup FFmpeg - uses: FedericoCarboni/setup-ffmpeg@v1 - - name: Install Dependencies - run: npm install - - name: Remove Broken Links - run: node scripts/clean.js --country=${{ matrix.country }} --debug - - name: Upload Artifact - uses: actions/upload-artifact@v2 - with: - name: channels - path: channels/${{ matrix.country }}.m3u - commit-changes: - runs-on: ubuntu-latest - needs: check - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/remove-broken-links - - name: Download Artifacts - uses: actions/download-artifact@v2 - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Remove broken links' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/remove-broken-links - file_pattern: channels/* - pull-request: - if: ${{ github.ref == 'refs/heads/master' }} - runs-on: ubuntu-latest - needs: commit-changes - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/remove-broken-links - - name: Generate Token - uses: tibdex/github-app-token@v1 - id: generate-token - with: - app_id: ${{ secrets.APP_ID }} - private_key: ${{ secrets.APP_PRIVATE_KEY }} - - name: Create Pull Request - uses: repo-sync/pull-request@v2 - with: - source_branch: 'bot/remove-broken-links' - destination_branch: 'master' - pr_title: '[Bot] Remove broken links' - pr_body: | - This pull request is created by [clean][1] workflow. - - The script checks all links except those with labels `[Geo-blocked]`, `[Offline]` or `[Not 24/7]` in the title. - - **IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. If the link works for you but occasionally return an HTTP code 403 (Forbidden) then it should be marked as `[Geo-blocked]`. If the link does not work but has no alternative, you can mark it as `[Offline]` to save it in the playlist along with a description. Working links should be marked as `[Not 24/7]` so that the script will skip them next time. - - [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} - pr_draft: true - github_token: ${{ steps.generate-token.outputs.token }} From d9633c8bf53fe9673ef9c34ed9268d5a38a0e3e3 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 04:08:48 +0300 Subject: [PATCH 09/23] Update CONTRIBUTING.md --- CONTRIBUTING.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a111b67135..ef08f6c077 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,7 +125,7 @@ STREAM_URL | `LANGUAGE` | Channel language. The name of the language must conform to the standard [ISO 639-3](https://iso639-3.sil.org/code_tables/639/data?title=&field_iso639_cd_st_mmbrshp_639_1_tid=94671&name_3=&field_iso639_element_scope_tid=All&field_iso639_language_type_tid=51&items_per_page=500). If the channel is broadcast in several languages you can list them separated by a semicolon. (optional) | | `LOGO_URL` | The logo of the channel that will be displayed if the player supports it. Supports files in png, jpeg and gif format. (optional) | | `CATEGORY` | The category to which the channel belongs. The list of currently supported categories can be found [here](https://github.com/iptv-org/iptv#playlists-by-category). (optional) | -| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except plus sign, minus sign, round and square brackets. | +| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except round and square brackets. | | `STREAM_TIME_SHIFT` | Must be specified if the channel is broadcast with a shift in time relative to the main stream. Should only contain a number and a sign. (optional) | | `ALTERNATIVE_NAME` | Can be used to specify a short name or name in another language. May contain any characters except round and square brackets. (optional) | | `STREAM_RESOLUTION` | The maximum height of the frame with a "p" at the end. In case of VLC Player this information can be found in `Window > Media Information... > Codec Details`. (optional) | @@ -153,7 +153,8 @@ http://example.com/stream.m3u8 - `.github/` - `ISSUE_TEMPLATE/`: issue templates for this repository. - `workflows/` - - `auto-update.yml`: contain actions that automatically updates all playlists every day. + - `auto-update.yml`: GitHub Action that automatically updates all playlists every day. + - `check.yml`: GitHub Action that automatically checks every pull request for syntax errors. - `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned. - `.readme/` - `_categories.md`: automatically generated list of all categories and their corresponding playlists. @@ -168,8 +169,6 @@ http://example.com/stream.m3u8 - `unsorted.m3u`: playlist with channels not yet sorted. - `scripts/` - `helpers/`: helper scripts used in GitHub Actions. - - `clean.js`: used in GitHub Action to check all links and remove broken ones. - - `detect-resolution.js`: used in GitHub Action to detect resolution of the streams. - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists. - `format.js`: used within GitHub Action to format channel descriptions. - `generate.js`: used within GitHub Action to generate all additional playlists. From 25ad4013c73d580c46c98ea5da80d0ee117808b9 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 04:15:11 +0300 Subject: [PATCH 10/23] Update utils.js --- scripts/helpers/utils.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/scripts/helpers/utils.js b/scripts/helpers/utils.js index 7a2bae0633..f0d9b3e50d 100644 --- a/scripts/helpers/utils.js +++ b/scripts/helpers/utils.js @@ -1,4 +1,3 @@ -const escapeStringRegexp = require('escape-string-regexp') const transliteration = require('transliteration') const iso6393 = require('@freearhey/iso-639-3') const categories = require('./categories') @@ -71,16 +70,6 @@ utils.sortBy = function (arr, fields) { }) } -utils.escapeStringRegexp = function (scring) { - return escapeStringRegexp(string) -} - -utils.sleep = function (ms) { - return function (x) { - return new Promise(resolve => setTimeout(() => resolve(x), ms)) - } -} - utils.removeProtocol = function (string) { return string.replace(/(^\w+:|^)\/\//, '') } From 33b276a8769c707796c6c20f9a9d070ec1641129 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 04:21:12 +0300 Subject: [PATCH 11/23] Remove unused dependencies --- package-lock.json | 3 --- package.json | 3 --- 2 files changed, 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index 06cebc1d6c..0eb569d0c2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,10 +7,7 @@ "license": "MIT", "dependencies": { "@freearhey/iso-639-3": "^1.0.0", - "axios": "^0.21.1", - "chalk": "^4.1.1", "commander": "^7.0.0", - "escape-string-regexp": "^2.0.0", "iptv-checker": "^0.20.2", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", diff --git a/package.json b/package.json index ea37000635..280313fec9 100644 --- a/package.json +++ b/package.json @@ -11,10 +11,7 @@ "license": "MIT", "dependencies": { "@freearhey/iso-639-3": "^1.0.0", - "axios": "^0.21.1", - "chalk": "^4.1.1", "commander": "^7.0.0", - "escape-string-regexp": "^2.0.0", "iptv-checker": "^0.20.2", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", From ab7c3662211f322645bf0e1083712233c4663d1d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 05:47:12 +0300 Subject: [PATCH 12/23] Update format.js --- scripts/format.js | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index cb3b407d03..3757a1868b 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -68,13 +68,9 @@ async function updatePlaylist(playlist) { await checker .checkStream(channel.data) .then(result => { - if (result.status.ok || result.status.reason.includes('timed out')) { - if (config.status) updateStatus(channel, null) - if (config.resolution) updateResolution(channel, result.status.metadata) - } else { - if (config.debug) log.print(` ERR: ${channel.url} (${result.status.reason})\n`) - if (config.status) updateStatus(channel, 'Offline') - } + if (config.status) updateStatus(channel, result.status) + if (config.resolution && result.status.ok) + updateResolution(channel, result.status.metadata) }) .catch(err => { if (config.debug) log.print(` ERR: ${channel.url} (${err.message})\n`) @@ -107,7 +103,17 @@ function addMissingData(channel) { } function updateStatus(channel, status) { - channel.status = status + if (status.ok) { + channel.status = null + } else if (status.reason.includes('timed out')) { + // nothing + } else if (status.reason.includes('403')) { + channel.status = 'Geo-blocked' + if (config.debug) log.print(` ERR: ${channel.url} (${status.reason})\n`) + } else { + channel.status = 'Offline' + if (config.debug) log.print(` ERR: ${channel.url} (${status.reason})\n`) + } } function updateResolution(channel, metadata) { From 9930219e55f505f1ce8c8e436ce179e878b384c1 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 06:49:40 +0300 Subject: [PATCH 13/23] Update format.js --- scripts/format.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index 3757a1868b..4de4171858 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -107,9 +107,6 @@ function updateStatus(channel, status) { channel.status = null } else if (status.reason.includes('timed out')) { // nothing - } else if (status.reason.includes('403')) { - channel.status = 'Geo-blocked' - if (config.debug) log.print(` ERR: ${channel.url} (${status.reason})\n`) } else { channel.status = 'Offline' if (config.debug) log.print(` ERR: ${channel.url} (${status.reason})\n`) From 999154d4fa0eb947c4d5b31b8372e40195a92f8c Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 07:12:25 +0300 Subject: [PATCH 14/23] Update format.js --- scripts/format.js | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/format.js b/scripts/format.js index 4de4171858..d6383df060 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -32,6 +32,7 @@ async function main() { let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') playlists = utils.filterPlaylists(playlists, config.country, config.exclude) + if (!playlists.length) log.print(`No playlist is selected\n`) for (const playlist of playlists) { await parser .parsePlaylist(playlist.url) From da89dcd1b85cbcafe4276e6b3b2d0e8c4f9e3e80 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 07:13:39 +0300 Subject: [PATCH 15/23] Update auto-update.yml For the test only --- .github/workflows/auto-update.yml | 325 +++++++++++++++--------------- 1 file changed, 162 insertions(+), 163 deletions(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 51cf6df50f..74a9b35752 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -24,169 +24,168 @@ jobs: strategy: fail-fast: false matrix: - country: - [ - ad, - ae, - af, - ag, - al, - am, - an, - ao, - ar, - at, - au, - aw, - az, - ba, - bb, - bd, - be, - bf, - bg, - bh, - bn, - bo, - br, - bs, - by, - ca, - cd, - cg, - ch, - ci, - cl, - cm, - cn, - co, - cr, - cu, - cw, - cy, - cz, - de, - dk, - do, - dz, - ec, - ee, - eg, - es, - et, - fi, - fj, - fo, - fr, - pf, - ge, - gh, - gm, - gn, - gp, - gq, - gr, - gt, - hk, - hn, - hr, - ht, - hu, - id, - ie, - il, - in, - iq, - ir, - is, - it, - jm, - jo, - jp, - ke, - kg, - kh, - kp, - kr, - kw, - kz, - la, - lb, - li, - lk, - lt, - lu, - lv, - ly, - ma, - mc, - md, - me, - mk, - ml, - mm, - mn, - mo, - mt, - mv, - mx, - my, - mz, - ne, - ng, - ni, - nl, - no, - np, - nz, - om, - pa, - pe, - ph, - pk, - pl, - pr, - ps, - pt, - py, - qa, - ro, - rs, - ru, - rw, - sa, - sd, - se, - sg, - si, - sk, - sl, - sm, - sn, - so, - sv, - sy, - th, - tj, - tm, - tn, - tr, - tt, - tw, - tz, - ua, - ug, - uk, - us, - uy, - uz, - va, - ve, - vi, - vn, - xk, - ye, - zm + country: [ + # ad, + # ae, + # af, + # ag, + # al, + # am, + # an, + # ao, + # ar, + # at, + # au, + # aw, + # az, + # ba, + # bb, + # bd, + # be, + # bf, + # bg, + # bh, + # bn, + # bo, + # br, + # bs, + # by, + # ca, + # cd, + # cg, + # ch, + # ci, + # cl, + # cm, + # cn, + # co, + # cr, + # cu, + # cw, + # cy, + # cz, + # de, + # dk, + # do, + # dz, + # ec, + # ee, + # eg, + # es, + # et, + # fi, + # fj, + # fo, + # fr, + # pf, + # ge, + # gh, + # gm, + # gn, + # gp, + # gq, + # gr, + # gt, + # hk, + # hn, + # hr, + # ht, + # hu, + # id, + # ie, + # il, + # in, + # iq, + # ir, + # is, + # it, + # jm, + # jo, + # jp, + # ke, + # kg, + # kh, + # kp, + # kr, + # kw, + # kz, + # la, + # lb, + # li, + # lk, + # lt, + # lu, + # lv, + # ly, + # ma, + # mc, + # md, + # me, + # mk, + # ml, + # mm, + # mn, + # mo, + # mt, + # mv, + # mx, + # my, + # mz, + # ne, + # ng, + # ni, + # nl, + # no, + # np, + # nz, + # om, + # pa, + # pe, + # ph, + # pk, + # pl, + # pr, + # ps, + # pt, + # py, + # qa, + # ro, + # rs, + ru + # rw, + # sa, + # sd, + # se, + # sg, + # si, + # sk, + # sl, + # sm, + # sn, + # so, + # sv, + # sy, + # th, + # tj, + # tm, + # tn, + # tr, + # tt, + # tw, + # tz, + # ua, + # ug, + # uk, + # us, + # uy, + # uz, + # va, + # ve, + # vi, + # vn, + # xk, + # ye, + # zm ] steps: - name: Checkout From c43ca9b66ff14b2b24d27d3bdc7a0780f914cdbc Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 07:45:52 +0300 Subject: [PATCH 16/23] Update format.js --- scripts/format.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/format.js b/scripts/format.js index d6383df060..496e8c8e68 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -106,7 +106,10 @@ function addMissingData(channel) { function updateStatus(channel, status) { if (status.ok) { channel.status = null - } else if (status.reason.includes('timed out')) { + } else if ( + status.reason.includes('timed out') || + status.reason.includes('not one of 40{0,1,3,4}') + ) { // nothing } else { channel.status = 'Offline' From b2cfb2be5e97a287860d02f8025a82869d821fe0 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 08:27:31 +0300 Subject: [PATCH 17/23] Update format.js --- scripts/format.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/format.js b/scripts/format.js index 496e8c8e68..bde2dbcebe 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -108,7 +108,8 @@ function updateStatus(channel, status) { channel.status = null } else if ( status.reason.includes('timed out') || - status.reason.includes('not one of 40{0,1,3,4}') + status.reason.includes('not one of 40{0,1,3,4}') || + status.reason.includes('403') ) { // nothing } else { From ff44649caa3ef6676e3a6b06f38338bed593f65a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 10:13:56 +0300 Subject: [PATCH 18/23] Update auto-update.yml --- .github/workflows/auto-update.yml | 325 +++++++++++++++--------------- 1 file changed, 163 insertions(+), 162 deletions(-) diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 74a9b35752..51cf6df50f 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -24,168 +24,169 @@ jobs: strategy: fail-fast: false matrix: - country: [ - # ad, - # ae, - # af, - # ag, - # al, - # am, - # an, - # ao, - # ar, - # at, - # au, - # aw, - # az, - # ba, - # bb, - # bd, - # be, - # bf, - # bg, - # bh, - # bn, - # bo, - # br, - # bs, - # by, - # ca, - # cd, - # cg, - # ch, - # ci, - # cl, - # cm, - # cn, - # co, - # cr, - # cu, - # cw, - # cy, - # cz, - # de, - # dk, - # do, - # dz, - # ec, - # ee, - # eg, - # es, - # et, - # fi, - # fj, - # fo, - # fr, - # pf, - # ge, - # gh, - # gm, - # gn, - # gp, - # gq, - # gr, - # gt, - # hk, - # hn, - # hr, - # ht, - # hu, - # id, - # ie, - # il, - # in, - # iq, - # ir, - # is, - # it, - # jm, - # jo, - # jp, - # ke, - # kg, - # kh, - # kp, - # kr, - # kw, - # kz, - # la, - # lb, - # li, - # lk, - # lt, - # lu, - # lv, - # ly, - # ma, - # mc, - # md, - # me, - # mk, - # ml, - # mm, - # mn, - # mo, - # mt, - # mv, - # mx, - # my, - # mz, - # ne, - # ng, - # ni, - # nl, - # no, - # np, - # nz, - # om, - # pa, - # pe, - # ph, - # pk, - # pl, - # pr, - # ps, - # pt, - # py, - # qa, - # ro, - # rs, - ru - # rw, - # sa, - # sd, - # se, - # sg, - # si, - # sk, - # sl, - # sm, - # sn, - # so, - # sv, - # sy, - # th, - # tj, - # tm, - # tn, - # tr, - # tt, - # tw, - # tz, - # ua, - # ug, - # uk, - # us, - # uy, - # uz, - # va, - # ve, - # vi, - # vn, - # xk, - # ye, - # zm + country: + [ + ad, + ae, + af, + ag, + al, + am, + an, + ao, + ar, + at, + au, + aw, + az, + ba, + bb, + bd, + be, + bf, + bg, + bh, + bn, + bo, + br, + bs, + by, + ca, + cd, + cg, + ch, + ci, + cl, + cm, + cn, + co, + cr, + cu, + cw, + cy, + cz, + de, + dk, + do, + dz, + ec, + ee, + eg, + es, + et, + fi, + fj, + fo, + fr, + pf, + ge, + gh, + gm, + gn, + gp, + gq, + gr, + gt, + hk, + hn, + hr, + ht, + hu, + id, + ie, + il, + in, + iq, + ir, + is, + it, + jm, + jo, + jp, + ke, + kg, + kh, + kp, + kr, + kw, + kz, + la, + lb, + li, + lk, + lt, + lu, + lv, + ly, + ma, + mc, + md, + me, + mk, + ml, + mm, + mn, + mo, + mt, + mv, + mx, + my, + mz, + ne, + ng, + ni, + nl, + no, + np, + nz, + om, + pa, + pe, + ph, + pk, + pl, + pr, + ps, + pt, + py, + qa, + ro, + rs, + ru, + rw, + sa, + sd, + se, + sg, + si, + sk, + sl, + sm, + sn, + so, + sv, + sy, + th, + tj, + tm, + tn, + tr, + tt, + tw, + tz, + ua, + ug, + uk, + us, + uy, + uz, + va, + ve, + vi, + vn, + xk, + ye, + zm ] steps: - name: Checkout From 70b6fd59d91541efa96530f868429f138b9f664e Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 12:50:23 +0300 Subject: [PATCH 19/23] Update format.js --- scripts/format.js | 58 +++++++++++++++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index bde2dbcebe..fff28268c7 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -27,8 +27,8 @@ async function main() { log.start() if (config.debug) log.print(`Debug mode enabled\n`) - if (config.status) log.print(`Updating channel status...\n`) - if (config.resolution) log.print(`Detecting channel resolution...\n`) + if (config.status) log.print(`Status check enabled\n`) + if (config.resolution) log.print(`Resolution detection enabled\n`) let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') playlists = utils.filterPlaylists(playlists, config.country, config.exclude) @@ -69,9 +69,19 @@ async function updatePlaylist(playlist) { await checker .checkStream(channel.data) .then(result => { - if (config.status) updateStatus(channel, result.status) - if (config.resolution && result.status.ok) + const status = parseStatus(result.status) + + if (config.status) { + updateStatus(channel, status) + } + + if (config.resolution && status === 'online') { updateResolution(channel, result.status.metadata) + } + + if (config.debug && status === 'offline') { + log.print(` ERR: ${channel.url} (${result.status.reason})\n`) + } }) .catch(err => { if (config.debug) log.print(` ERR: ${channel.url} (${err.message})\n`) @@ -83,6 +93,31 @@ async function updatePlaylist(playlist) { return playlist } +function parseStatus(status) { + if (status.ok) { + return 'online' + } else if (status.reason.includes('timed out')) { + return 'timeout' + } else if (status.reason.includes('403')) { + return 'error_403' + } else if (status.reason.includes('not one of 40{0,1,3,4}')) { + return 'error_40x' // 402, 451 + } else { + return 'offline' + } +} + +function updateStatus(channel, status) { + switch (status) { + case 'online': + channel.status = null + break + case 'offline': + channel.status = 'Offline' + break + } +} + function addMissingData(channel) { // add tvg-name if (!channel.tvg.name && channel.name) { @@ -103,21 +138,6 @@ function addMissingData(channel) { channel.group.title = channel.category } -function updateStatus(channel, status) { - if (status.ok) { - channel.status = null - } else if ( - status.reason.includes('timed out') || - status.reason.includes('not one of 40{0,1,3,4}') || - status.reason.includes('403') - ) { - // nothing - } else { - channel.status = 'Offline' - if (config.debug) log.print(` ERR: ${channel.url} (${status.reason})\n`) - } -} - function updateResolution(channel, metadata) { const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : [] if (!channel.resolution.height && streams.length) { From 02a2d41cfdcafbe080f655667e1a960b8c7e733a Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 12:52:11 +0300 Subject: [PATCH 20/23] Update format.js --- scripts/format.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index fff28268c7..af382fb4fe 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -62,10 +62,10 @@ async function updatePlaylist(playlist) { for (const channel of playlist.channels) { addMissingData(channel) const checkOnline = config.status || config.resolution - const skip = + const skipChannel = channel.status && ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) - if (checkOnline && !skip) { + if (checkOnline && !skipChannel) { await checker .checkStream(channel.data) .then(result => { From b75d790e0291751ff04ffdc22c275178f1a94464 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 16:12:43 +0300 Subject: [PATCH 21/23] Install normalize-url package --- package-lock.json | 17 +++++++++++++++++ package.json | 1 + 2 files changed, 18 insertions(+) diff --git a/package-lock.json b/package-lock.json index 0eb569d0c2..ffc2acf870 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", "markdown-include": "^0.4.3", + "normalize-url": "^6.1.0", "pre-push": "^0.1.1", "progress": "^2.0.3", "transliteration": "^2.2.0" @@ -2960,6 +2961,17 @@ "node": ">=0.10.0" } }, + "node_modules/normalize-url": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", + "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -6042,6 +6054,11 @@ "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, + "normalize-url": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", + "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==" + }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", diff --git a/package.json b/package.json index 280313fec9..64d55e8f7b 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", "markdown-include": "^0.4.3", + "normalize-url": "^6.1.0", "pre-push": "^0.1.1", "progress": "^2.0.3", "transliteration": "^2.2.0" From e4b697b229671f375b7103b4a1c30f8d466cf59f Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 16:52:18 +0300 Subject: [PATCH 22/23] Update Channel.js --- scripts/helpers/Channel.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/helpers/Channel.js b/scripts/helpers/Channel.js index 043f7886f3..8c4b0c397a 100644 --- a/scripts/helpers/Channel.js +++ b/scripts/helpers/Channel.js @@ -22,6 +22,11 @@ module.exports = class Channel { this.languages = this.parseLanguages(data.tvg.language) } + updateUrl(url) { + this.url = url + this.data.url = url + } + parseName(title) { return title .trim() From 24912fe182bbc6ea4c16c5f0e3adadd2c6aa89f8 Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Tue, 10 Aug 2021 17:06:22 +0300 Subject: [PATCH 23/23] Update format.js --- scripts/format.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/format.js b/scripts/format.js index af382fb4fe..acc2fca3c8 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -1,4 +1,5 @@ const IPTVChecker = require('iptv-checker') +const normalize = require('normalize-url') const { program } = require('commander') const ProgressBar = require('progress') const parser = require('./helpers/parser') @@ -61,6 +62,9 @@ async function updatePlaylist(playlist) { for (const channel of playlist.channels) { addMissingData(channel) + updateGroupTitle(channel) + normalizeUrl(channel) + const checkOnline = config.status || config.resolution const skipChannel = channel.status && @@ -119,25 +123,32 @@ function updateStatus(channel, status) { } function addMissingData(channel) { - // add tvg-name + // tvg-name if (!channel.tvg.name && channel.name) { channel.tvg.name = channel.name.replace(/\"/gi, '') } - // add tvg-id + // tvg-id if (!channel.tvg.id && channel.tvg.name) { const id = utils.name2id(channel.tvg.name) channel.tvg.id = id ? `${id}.${code}` : '' } - // add country + // country if (!channel.countries.length) { const name = utils.code2name(code) channel.countries = name ? [{ code, name }] : [] channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') } - // update group-title +} + +function updateGroupTitle(channel) { channel.group.title = channel.category } +function normalizeUrl(channel) { + const normalized = normalize(channel.url, { stripWWW: false }) + channel.updateUrl(normalized) +} + function updateResolution(channel, metadata) { const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : [] if (!channel.resolution.height && streams.length) {