From 82a3f824f22416b7cbbb2704572a4ce3b9aeda82 Mon Sep 17 00:00:00 2001 From: Dum4G Date: Tue, 10 Aug 2021 23:45:32 +0300 Subject: [PATCH] Revert "Update "format" script" --- .github/workflows/auto-update.yml | 161 ++++++++++--------- .github/workflows/clean.yml | 257 ++++++++++++++++++++++++++++++ CONTRIBUTING.md | 7 +- package-lock.json | 20 +-- package.json | 4 +- scripts/clean.js | 74 +++++++++ scripts/detect-resolution.js | 114 +++++++++++++ scripts/format.js | 153 +++--------------- scripts/helpers/Channel.js | 5 - scripts/helpers/utils.js | 11 ++ 10 files changed, 579 insertions(+), 227 deletions(-) create mode 100644 .github/workflows/clean.yml create mode 100644 scripts/clean.js create mode 100644 scripts/detect-resolution.js diff --git a/.github/workflows/auto-update.yml b/.github/workflows/auto-update.yml index 51cf6df50..213deb39b 100644 --- a/.github/workflows/auto-update.yml +++ b/.github/workflows/auto-update.yml @@ -20,6 +20,90 @@ jobs: format: runs-on: ubuntu-latest needs: create-branch + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Format Playlists + run: node scripts/format.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Formate playlists' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + sort: + runs-on: ubuntu-latest + needs: format + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Sort Channels + run: node scripts/sort.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Sort channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + remove-duplicates: + runs-on: ubuntu-latest + needs: sort + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Remove Duplicates + run: node scripts/remove-duplicates.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove duplicates' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + filter: + runs-on: ubuntu-latest + needs: remove-duplicates + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/auto-update + - name: Install Dependencies + run: npm install + - name: Filter Playlists + run: node scripts/filter.js + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Filter channels' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/auto-update + file_pattern: channels/* + detect-resolution: + runs-on: ubuntu-latest + needs: filter continue-on-error: true strategy: fail-fast: false @@ -193,12 +277,10 @@ jobs: uses: actions/checkout@v2 with: ref: bot/auto-update - - name: Setup FFmpeg - uses: FedericoCarboni/setup-ffmpeg@v1 - name: Install Dependencies run: npm install - - name: Format Playlists - run: node scripts/format.js --country=${{ matrix.country }} --status --resolution --debug + - name: Detect Resolution + run: node scripts/detect-resolution.js --country=${{ matrix.country }} - name: Upload Artifact uses: actions/upload-artifact@v2 with: @@ -206,7 +288,7 @@ jobs: path: channels/${{ matrix.country }}.m3u commit-changes: runs-on: ubuntu-latest - needs: format + needs: detect-resolution steps: - name: Checkout uses: actions/checkout@v2 @@ -220,70 +302,7 @@ jobs: - name: Commit Changes uses: stefanzweifel/git-auto-commit-action@v4 with: - commit_message: '[Bot] Format playlists' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - remove-duplicates: - runs-on: ubuntu-latest - needs: commit-changes - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Remove Duplicates - run: node scripts/remove-duplicates.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Remove duplicates' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - sort: - runs-on: ubuntu-latest - needs: remove-duplicates - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Sort Channels - run: node scripts/sort.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Sort channels' - commit_user_name: iptv-bot - commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com - commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' - branch: bot/auto-update - file_pattern: channels/* - filter: - runs-on: ubuntu-latest - needs: sort - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - ref: bot/auto-update - - name: Install Dependencies - run: npm install - - name: Filter Playlists - run: node scripts/filter.js - - name: Commit Changes - uses: stefanzweifel/git-auto-commit-action@v4 - with: - commit_message: '[Bot] Filter channels' + commit_message: '[Bot] Detect resolution' commit_user_name: iptv-bot commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' @@ -291,7 +310,7 @@ jobs: file_pattern: channels/* generate: runs-on: ubuntu-latest - needs: filter + needs: commit-changes steps: - name: Checkout uses: actions/checkout@v2 @@ -357,7 +376,6 @@ jobs: branch: bot/auto-update file_pattern: README.md pull-request: - if: ${{ github.ref == 'refs/heads/master' }} needs: update-readme runs-on: ubuntu-latest steps: @@ -390,6 +408,7 @@ jobs: pull-request-number: ${{ steps.pr.outputs.pr_number }} merge-method: squash - name: Approve Pull Request + if: github.ref == 'refs/heads/master' uses: juliangruber/approve-pull-request-action@v1 with: github-token: ${{ secrets.PAT }} diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml new file mode 100644 index 000000000..3c1f9a452 --- /dev/null +++ b/.github/workflows/clean.yml @@ -0,0 +1,257 @@ +name: clean +on: + workflow_dispatch: + schedule: + - cron: '0 6 * * 0' +jobs: + create-branch: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: ${{ github.ref }} + - name: Create Branch + uses: peterjgrainger/action-create-branch@v2.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + branch: 'bot/remove-broken-links' + check: + runs-on: ubuntu-latest + needs: create-branch + continue-on-error: true + strategy: + fail-fast: false + matrix: + country: + [ + ad, + ae, + af, + ag, + al, + am, + an, + ao, + ar, + at, + au, + aw, + az, + ba, + bb, + bd, + be, + bf, + bg, + bh, + bn, + bo, + br, + bs, + by, + ca, + cd, + cg, + ch, + ci, + cl, + cm, + cn, + co, + cr, + cu, + cw, + cy, + cz, + de, + dk, + do, + dz, + ec, + ee, + eg, + es, + et, + fi, + fj, + fo, + fr, + pf, + ge, + gh, + gm, + gn, + gp, + gq, + gr, + gt, + hk, + hn, + hr, + ht, + hu, + id, + ie, + il, + in, + iq, + ir, + is, + it, + jm, + jo, + jp, + ke, + kg, + kh, + kp, + kr, + kw, + kz, + la, + lb, + li, + lk, + lt, + lu, + lv, + ly, + ma, + mc, + md, + me, + mk, + ml, + mm, + mn, + mo, + mt, + mv, + mx, + my, + mz, + ne, + ng, + ni, + nl, + no, + np, + nz, + om, + pa, + pe, + ph, + pk, + pl, + pr, + ps, + pt, + py, + qa, + ro, + rs, + ru, + rw, + sa, + sd, + se, + sg, + si, + sk, + sl, + sm, + sn, + so, + sv, + sy, + th, + tj, + tm, + tn, + tr, + tt, + tw, + tz, + ua, + ug, + uk, + us, + uy, + uz, + va, + ve, + vi, + vn, + xk, + ye, + zm, + unsorted + ] + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/remove-broken-links + - name: Setup FFmpeg + uses: FedericoCarboni/setup-ffmpeg@v1 + - name: Install Dependencies + run: npm install + - name: Remove Broken Links + run: node scripts/clean.js --country=${{ matrix.country }} --debug + - name: Upload Artifact + uses: actions/upload-artifact@v2 + with: + name: channels + path: channels/${{ matrix.country }}.m3u + commit-changes: + runs-on: ubuntu-latest + needs: check + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/remove-broken-links + - name: Download Artifacts + uses: actions/download-artifact@v2 + - name: Commit Changes + uses: stefanzweifel/git-auto-commit-action@v4 + with: + commit_message: '[Bot] Remove broken links' + commit_user_name: iptv-bot + commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com + commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>' + branch: bot/remove-broken-links + file_pattern: channels/* + pull-request: + if: ${{ github.ref == 'refs/heads/master' }} + runs-on: ubuntu-latest + needs: commit-changes + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: bot/remove-broken-links + - name: Generate Token + uses: tibdex/github-app-token@v1 + id: generate-token + with: + app_id: ${{ secrets.APP_ID }} + private_key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Create Pull Request + uses: repo-sync/pull-request@v2 + with: + source_branch: 'bot/remove-broken-links' + destination_branch: 'master' + pr_title: '[Bot] Remove broken links' + pr_body: | + This pull request is created by [clean][1] workflow. + + The script checks all links except those with labels `[Geo-blocked]`, `[Offline]` or `[Not 24/7]` in the title. + + **IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. If the link works for you but occasionally return an HTTP code 403 (Forbidden) then it should be marked as `[Geo-blocked]`. If the link does not work but has no alternative, you can mark it as `[Offline]` to save it in the playlist along with a description. Working links should be marked as `[Not 24/7]` so that the script will skip them next time. + + [1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }} + pr_draft: true + github_token: ${{ steps.generate-token.outputs.token }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef08f6c07..a111b6713 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -125,7 +125,7 @@ STREAM_URL | `LANGUAGE` | Channel language. The name of the language must conform to the standard [ISO 639-3](https://iso639-3.sil.org/code_tables/639/data?title=&field_iso639_cd_st_mmbrshp_639_1_tid=94671&name_3=&field_iso639_element_scope_tid=All&field_iso639_language_type_tid=51&items_per_page=500). If the channel is broadcast in several languages you can list them separated by a semicolon. (optional) | | `LOGO_URL` | The logo of the channel that will be displayed if the player supports it. Supports files in png, jpeg and gif format. (optional) | | `CATEGORY` | The category to which the channel belongs. The list of currently supported categories can be found [here](https://github.com/iptv-org/iptv#playlists-by-category). (optional) | -| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except round and square brackets. | +| `FULL_NAME` | Full name of the channel. It is recommended to use the name listed on [lyngsat](https://www.lyngsat.com/search.html) or [wikipedia](https://www.wikipedia.org/) if possible. May contain any characters except plus sign, minus sign, round and square brackets. | | `STREAM_TIME_SHIFT` | Must be specified if the channel is broadcast with a shift in time relative to the main stream. Should only contain a number and a sign. (optional) | | `ALTERNATIVE_NAME` | Can be used to specify a short name or name in another language. May contain any characters except round and square brackets. (optional) | | `STREAM_RESOLUTION` | The maximum height of the frame with a "p" at the end. In case of VLC Player this information can be found in `Window > Media Information... > Codec Details`. (optional) | @@ -153,8 +153,7 @@ http://example.com/stream.m3u8 - `.github/` - `ISSUE_TEMPLATE/`: issue templates for this repository. - `workflows/` - - `auto-update.yml`: GitHub Action that automatically updates all playlists every day. - - `check.yml`: GitHub Action that automatically checks every pull request for syntax errors. + - `auto-update.yml`: contain actions that automatically updates all playlists every day. - `CODE_OF_CONDUCT.md`: rules you shouldn't break if you don't want to get banned. - `.readme/` - `_categories.md`: automatically generated list of all categories and their corresponding playlists. @@ -169,6 +168,8 @@ http://example.com/stream.m3u8 - `unsorted.m3u`: playlist with channels not yet sorted. - `scripts/` - `helpers/`: helper scripts used in GitHub Actions. + - `clean.js`: used in GitHub Action to check all links and remove broken ones. + - `detect-resolution.js`: used in GitHub Action to detect resolution of the streams. - `filter.js`: used within GitHub Action to remove blacklisted channels from playlists. - `format.js`: used within GitHub Action to format channel descriptions. - `generate.js`: used within GitHub Action to generate all additional playlists. diff --git a/package-lock.json b/package-lock.json index ffc2acf87..06cebc1d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,12 +7,14 @@ "license": "MIT", "dependencies": { "@freearhey/iso-639-3": "^1.0.0", + "axios": "^0.21.1", + "chalk": "^4.1.1", "commander": "^7.0.0", + "escape-string-regexp": "^2.0.0", "iptv-checker": "^0.20.2", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", "markdown-include": "^0.4.3", - "normalize-url": "^6.1.0", "pre-push": "^0.1.1", "progress": "^2.0.3", "transliteration": "^2.2.0" @@ -2961,17 +2963,6 @@ "node": ">=0.10.0" } }, - "node_modules/normalize-url": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", - "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", @@ -6054,11 +6045,6 @@ "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==" }, - "normalize-url": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-6.1.0.tgz", - "integrity": "sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==" - }, "npm-run-path": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", diff --git a/package.json b/package.json index 64d55e8f7..ea3700063 100644 --- a/package.json +++ b/package.json @@ -11,12 +11,14 @@ "license": "MIT", "dependencies": { "@freearhey/iso-639-3": "^1.0.0", + "axios": "^0.21.1", + "chalk": "^4.1.1", "commander": "^7.0.0", + "escape-string-regexp": "^2.0.0", "iptv-checker": "^0.20.2", "iptv-playlist-parser": "^0.5.4", "m3u-linter": "^0.1.3", "markdown-include": "^0.4.3", - "normalize-url": "^6.1.0", "pre-push": "^0.1.1", "progress": "^2.0.3", "transliteration": "^2.2.0" diff --git a/scripts/clean.js b/scripts/clean.js new file mode 100644 index 000000000..d1ab7c2e7 --- /dev/null +++ b/scripts/clean.js @@ -0,0 +1,74 @@ +const IPTVChecker = require('iptv-checker') +const { program } = require('commander') +const ProgressBar = require('progress') +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') + +program + .usage('[OPTIONS]...') + .option('-d, --debug', 'Enable debug mode') + .option('-c, --country ', 'Comma-separated list of country codes', '') + .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') + .option('--timeout ', 'Set timeout for each request', 5000) + .parse(process.argv) + +let bar +const config = program.opts() +const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline'] +const checker = new IPTVChecker({ + timeout: config.timeout +}) + +async function main() { + log.start() + + if (config.debug) log.print(`Debug mode enabled\n`) + + let playlists = parser.parseIndex() + playlists = utils.filterPlaylists(playlists, config.country, config.exclude) + for (const playlist of playlists) { + await parser + .parsePlaylist(playlist.url) + .then(checkPlaylist) + .then(p => p.save()) + } + + log.finish() +} + +async function checkPlaylist(playlist) { + if (!config.debug) { + bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + } + const channels = [] + const total = playlist.channels.length + for (const [index, channel] of playlist.channels.entries()) { + const skipChannel = + channel.status && + ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) + if (skipChannel) { + channels.push(channel) + } else { + const result = await checker.checkStream(channel.data) + if (result.status.ok || result.status.reason.includes('timed out')) { + channels.push(channel) + } else { + if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`) + } + } + if (!config.debug) bar.tick() + } + + if (playlist.channels.length !== channels.length) { + log.print(`File '${playlist.url}' has been updated\n`) + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +main() diff --git a/scripts/detect-resolution.js b/scripts/detect-resolution.js new file mode 100644 index 000000000..5da6a282f --- /dev/null +++ b/scripts/detect-resolution.js @@ -0,0 +1,114 @@ +const { program } = require('commander') +const ProgressBar = require('progress') +const axios = require('axios') +const https = require('https') +const parser = require('./helpers/parser') +const utils = require('./helpers/utils') +const log = require('./helpers/log') + +program + .usage('[OPTIONS]...') + .option('-c, --country ', 'Comma-separated list of country codes', '') + .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') + .option('--delay ', 'Delay between parser requests', 1000) + .option('--timeout ', 'Set timeout for each request', 5000) + .parse(process.argv) + +const config = program.opts() +const ignoreStatus = ['Offline'] +const instance = axios.create({ + timeout: config.timeout, + maxContentLength: 200000, + httpsAgent: new https.Agent({ + rejectUnauthorized: false + }) +}) + +async function main() { + log.start() + + log.print(`Parsing 'index.m3u'...\n`) + let playlists = parser.parseIndex() + playlists = utils + .filterPlaylists(playlists, config.country, config.exclude) + .filter(i => i.url !== 'channels/unsorted.m3u') + + for (const playlist of playlists) { + await parser + .parsePlaylist(playlist.url) + .then(detectResolution) + .then(p => p.save()) + } + + log.finish() +} + +async function detectResolution(playlist) { + const channels = [] + const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { + total: playlist.channels.length + }) + let updated = false + for (const channel of playlist.channels) { + bar.tick() + const skipChannel = + channel.status && + ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) + if (!channel.resolution.height && !skipChannel) { + const CancelToken = axios.CancelToken + const source = CancelToken.source() + const timeout = setTimeout(() => { + source.cancel() + }, config.timeout) + + const response = await instance + .get(channel.url, { cancelToken: source.token }) + .then(res => { + clearTimeout(timeout) + + return res + }) + .then(utils.sleep(config.delay)) + .catch(err => { + clearTimeout(timeout) + }) + + if (response && response.status === 200) { + if (/^#EXTM3U/.test(response.data)) { + const resolution = parseResolution(response.data) + if (resolution) { + channel.resolution = resolution + updated = true + } + } + } + } + + channels.push(channel) + } + + if (updated) { + log.print(`File '${playlist.url}' has been updated\n`) + playlist.channels = channels + playlist.updated = true + } + + return playlist +} + +function parseResolution(string) { + const regex = /RESOLUTION=(\d+)x(\d+)/gm + const match = string.matchAll(regex) + const arr = Array.from(match).map(m => ({ + width: parseInt(m[1]), + height: parseInt(m[2]) + })) + + return arr.length + ? arr.reduce(function (prev, current) { + return prev.height > current.height ? prev : current + }) + : undefined +} + +main() diff --git a/scripts/format.js b/scripts/format.js index acc2fca3c..fb77cab20 100644 --- a/scripts/format.js +++ b/scripts/format.js @@ -1,46 +1,21 @@ -const IPTVChecker = require('iptv-checker') -const normalize = require('normalize-url') -const { program } = require('commander') -const ProgressBar = require('progress') const parser = require('./helpers/parser') const utils = require('./helpers/utils') const file = require('./helpers/file') const log = require('./helpers/log') -program - .usage('[OPTIONS]...') - .option('-d, --debug', 'Enable debug mode') - .option('-s, --status', 'Update stream status') - .option('-r, --resolution', 'Detect stream resolution') - .option('-c, --country ', 'Comma-separated list of country codes', '') - .option('-e, --exclude ', 'Comma-separated list of country codes to be excluded', '') - .option('--timeout ', 'Set timeout for each request', 5000) - .parse(process.argv) - -let bar -const ignoreStatus = ['Geo-blocked', 'Not 24/7'] -const config = program.opts() -const checker = new IPTVChecker({ - timeout: config.timeout -}) - async function main() { log.start() - if (config.debug) log.print(`Debug mode enabled\n`) - if (config.status) log.print(`Status check enabled\n`) - if (config.resolution) log.print(`Resolution detection enabled\n`) - + log.print(`Parsing 'index.m3u'...`) let playlists = parser.parseIndex().filter(i => i.url !== 'channels/unsorted.m3u') - playlists = utils.filterPlaylists(playlists, config.country, config.exclude) - if (!playlists.length) log.print(`No playlist is selected\n`) for (const playlist of playlists) { + log.print(`\nProcessing '${playlist.url}'...`) await parser .parsePlaylist(playlist.url) - .then(updatePlaylist) + .then(formatPlaylist) .then(playlist => { if (file.read(playlist.url) !== playlist.toString()) { - log.print(`File '${playlist.url}' has been updated\n`) + log.print('updated') playlist.updated = true } @@ -48,115 +23,33 @@ async function main() { }) } + log.print('\n') log.finish() } -async function updatePlaylist(playlist) { - if (!config.debug) { - bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, { - total: playlist.channels.length - }) - } else { - log.print(`Processing '${playlist.url}'...\n`) - } - +async function formatPlaylist(playlist) { for (const channel of playlist.channels) { - addMissingData(channel) - updateGroupTitle(channel) - normalizeUrl(channel) - - const checkOnline = config.status || config.resolution - const skipChannel = - channel.status && - ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase()) - if (checkOnline && !skipChannel) { - await checker - .checkStream(channel.data) - .then(result => { - const status = parseStatus(result.status) - - if (config.status) { - updateStatus(channel, status) - } - - if (config.resolution && status === 'online') { - updateResolution(channel, result.status.metadata) - } - - if (config.debug && status === 'offline') { - log.print(` ERR: ${channel.url} (${result.status.reason})\n`) - } - }) - .catch(err => { - if (config.debug) log.print(` ERR: ${channel.url} (${err.message})\n`) - }) + const code = file.getBasename(playlist.url) + // add missing tvg-name + if (!channel.tvg.name && code !== 'unsorted' && channel.name) { + channel.tvg.name = channel.name.replace(/\"/gi, '') + } + // add missing tvg-id + if (!channel.tvg.id && code !== 'unsorted' && channel.tvg.name) { + const id = utils.name2id(channel.tvg.name) + channel.tvg.id = id ? `${id}.${code}` : '' } - if (!config.debug) bar.tick() + // add missing country + if (!channel.countries.length) { + const name = utils.code2name(code) + channel.countries = name ? [{ code, name }] : [] + channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') + } + // update group-title + channel.group.title = channel.category } return playlist } -function parseStatus(status) { - if (status.ok) { - return 'online' - } else if (status.reason.includes('timed out')) { - return 'timeout' - } else if (status.reason.includes('403')) { - return 'error_403' - } else if (status.reason.includes('not one of 40{0,1,3,4}')) { - return 'error_40x' // 402, 451 - } else { - return 'offline' - } -} - -function updateStatus(channel, status) { - switch (status) { - case 'online': - channel.status = null - break - case 'offline': - channel.status = 'Offline' - break - } -} - -function addMissingData(channel) { - // tvg-name - if (!channel.tvg.name && channel.name) { - channel.tvg.name = channel.name.replace(/\"/gi, '') - } - // tvg-id - if (!channel.tvg.id && channel.tvg.name) { - const id = utils.name2id(channel.tvg.name) - channel.tvg.id = id ? `${id}.${code}` : '' - } - // country - if (!channel.countries.length) { - const name = utils.code2name(code) - channel.countries = name ? [{ code, name }] : [] - channel.tvg.country = channel.countries.map(c => c.code.toUpperCase()).join(';') - } -} - -function updateGroupTitle(channel) { - channel.group.title = channel.category -} - -function normalizeUrl(channel) { - const normalized = normalize(channel.url, { stripWWW: false }) - channel.updateUrl(normalized) -} - -function updateResolution(channel, metadata) { - const streams = metadata ? metadata.streams.filter(stream => stream.codec_type === 'video') : [] - if (!channel.resolution.height && streams.length) { - channel.resolution = streams.reduce((acc, curr) => { - if (curr.height > acc.height) return { width: curr.width, height: curr.height } - return acc - }) - } -} - main() diff --git a/scripts/helpers/Channel.js b/scripts/helpers/Channel.js index 8c4b0c397..043f7886f 100644 --- a/scripts/helpers/Channel.js +++ b/scripts/helpers/Channel.js @@ -22,11 +22,6 @@ module.exports = class Channel { this.languages = this.parseLanguages(data.tvg.language) } - updateUrl(url) { - this.url = url - this.data.url = url - } - parseName(title) { return title .trim() diff --git a/scripts/helpers/utils.js b/scripts/helpers/utils.js index f0d9b3e50..7a2bae063 100644 --- a/scripts/helpers/utils.js +++ b/scripts/helpers/utils.js @@ -1,3 +1,4 @@ +const escapeStringRegexp = require('escape-string-regexp') const transliteration = require('transliteration') const iso6393 = require('@freearhey/iso-639-3') const categories = require('./categories') @@ -70,6 +71,16 @@ utils.sortBy = function (arr, fields) { }) } +utils.escapeStringRegexp = function (scring) { + return escapeStringRegexp(string) +} + +utils.sleep = function (ms) { + return function (x) { + return new Promise(resolve => setTimeout(() => resolve(x), ms)) + } +} + utils.removeProtocol = function (string) { return string.replace(/(^\w+:|^)\/\//, '') }