mirror of https://github.com/iptv-org/iptv
Merge pull request #4036 from iptv-org/revert-4034-update-format-script
Revert "Update "format" script"pull/4028/head^2
Dum4G
3 years ago
committed by
GitHub
commit
0a3df6c4ea
@ -0,0 +1,257 @@
|
||||
name: clean
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '0 6 * * 0'
|
||||
jobs:
|
||||
create-branch:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
- name: Create Branch
|
||||
uses: peterjgrainger/action-create-branch@v2.0.1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
branch: 'bot/remove-broken-links'
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
needs: create-branch
|
||||
continue-on-error: true
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
country:
|
||||
[
|
||||
ad,
|
||||
ae,
|
||||
af,
|
||||
ag,
|
||||
al,
|
||||
am,
|
||||
an,
|
||||
ao,
|
||||
ar,
|
||||
at,
|
||||
au,
|
||||
aw,
|
||||
az,
|
||||
ba,
|
||||
bb,
|
||||
bd,
|
||||
be,
|
||||
bf,
|
||||
bg,
|
||||
bh,
|
||||
bn,
|
||||
bo,
|
||||
br,
|
||||
bs,
|
||||
by,
|
||||
ca,
|
||||
cd,
|
||||
cg,
|
||||
ch,
|
||||
ci,
|
||||
cl,
|
||||
cm,
|
||||
cn,
|
||||
co,
|
||||
cr,
|
||||
cu,
|
||||
cw,
|
||||
cy,
|
||||
cz,
|
||||
de,
|
||||
dk,
|
||||
do,
|
||||
dz,
|
||||
ec,
|
||||
ee,
|
||||
eg,
|
||||
es,
|
||||
et,
|
||||
fi,
|
||||
fj,
|
||||
fo,
|
||||
fr,
|
||||
pf,
|
||||
ge,
|
||||
gh,
|
||||
gm,
|
||||
gn,
|
||||
gp,
|
||||
gq,
|
||||
gr,
|
||||
gt,
|
||||
hk,
|
||||
hn,
|
||||
hr,
|
||||
ht,
|
||||
hu,
|
||||
id,
|
||||
ie,
|
||||
il,
|
||||
in,
|
||||
iq,
|
||||
ir,
|
||||
is,
|
||||
it,
|
||||
jm,
|
||||
jo,
|
||||
jp,
|
||||
ke,
|
||||
kg,
|
||||
kh,
|
||||
kp,
|
||||
kr,
|
||||
kw,
|
||||
kz,
|
||||
la,
|
||||
lb,
|
||||
li,
|
||||
lk,
|
||||
lt,
|
||||
lu,
|
||||
lv,
|
||||
ly,
|
||||
ma,
|
||||
mc,
|
||||
md,
|
||||
me,
|
||||
mk,
|
||||
ml,
|
||||
mm,
|
||||
mn,
|
||||
mo,
|
||||
mt,
|
||||
mv,
|
||||
mx,
|
||||
my,
|
||||
mz,
|
||||
ne,
|
||||
ng,
|
||||
ni,
|
||||
nl,
|
||||
no,
|
||||
np,
|
||||
nz,
|
||||
om,
|
||||
pa,
|
||||
pe,
|
||||
ph,
|
||||
pk,
|
||||
pl,
|
||||
pr,
|
||||
ps,
|
||||
pt,
|
||||
py,
|
||||
qa,
|
||||
ro,
|
||||
rs,
|
||||
ru,
|
||||
rw,
|
||||
sa,
|
||||
sd,
|
||||
se,
|
||||
sg,
|
||||
si,
|
||||
sk,
|
||||
sl,
|
||||
sm,
|
||||
sn,
|
||||
so,
|
||||
sv,
|
||||
sy,
|
||||
th,
|
||||
tj,
|
||||
tm,
|
||||
tn,
|
||||
tr,
|
||||
tt,
|
||||
tw,
|
||||
tz,
|
||||
ua,
|
||||
ug,
|
||||
uk,
|
||||
us,
|
||||
uy,
|
||||
uz,
|
||||
va,
|
||||
ve,
|
||||
vi,
|
||||
vn,
|
||||
xk,
|
||||
ye,
|
||||
zm,
|
||||
unsorted
|
||||
]
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/remove-broken-links
|
||||
- name: Setup FFmpeg
|
||||
uses: FedericoCarboni/setup-ffmpeg@v1
|
||||
- name: Install Dependencies
|
||||
run: npm install
|
||||
- name: Remove Broken Links
|
||||
run: node scripts/clean.js --country=${{ matrix.country }} --debug
|
||||
- name: Upload Artifact
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: channels
|
||||
path: channels/${{ matrix.country }}.m3u
|
||||
commit-changes:
|
||||
runs-on: ubuntu-latest
|
||||
needs: check
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/remove-broken-links
|
||||
- name: Download Artifacts
|
||||
uses: actions/download-artifact@v2
|
||||
- name: Commit Changes
|
||||
uses: stefanzweifel/git-auto-commit-action@v4
|
||||
with:
|
||||
commit_message: '[Bot] Remove broken links'
|
||||
commit_user_name: iptv-bot
|
||||
commit_user_email: 84861620+iptv-bot[bot]@users.noreply.github.com
|
||||
commit_author: 'iptv-bot[bot] <84861620+iptv-bot[bot]@users.noreply.github.com>'
|
||||
branch: bot/remove-broken-links
|
||||
file_pattern: channels/*
|
||||
pull-request:
|
||||
if: ${{ github.ref == 'refs/heads/master' }}
|
||||
runs-on: ubuntu-latest
|
||||
needs: commit-changes
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
ref: bot/remove-broken-links
|
||||
- name: Generate Token
|
||||
uses: tibdex/github-app-token@v1
|
||||
id: generate-token
|
||||
with:
|
||||
app_id: ${{ secrets.APP_ID }}
|
||||
private_key: ${{ secrets.APP_PRIVATE_KEY }}
|
||||
- name: Create Pull Request
|
||||
uses: repo-sync/pull-request@v2
|
||||
with:
|
||||
source_branch: 'bot/remove-broken-links'
|
||||
destination_branch: 'master'
|
||||
pr_title: '[Bot] Remove broken links'
|
||||
pr_body: |
|
||||
This pull request is created by [clean][1] workflow.
|
||||
|
||||
The script checks all links except those with labels `[Geo-blocked]`, `[Offline]` or `[Not 24/7]` in the title.
|
||||
|
||||
**IMPORTANT:** Before merging all links should be checked manually to make sure that the response from the server has not changed. If the link works for you but occasionally return an HTTP code 403 (Forbidden) then it should be marked as `[Geo-blocked]`. If the link does not work but has no alternative, you can mark it as `[Offline]` to save it in the playlist along with a description. Working links should be marked as `[Not 24/7]` so that the script will skip them next time.
|
||||
|
||||
[1]: https://github.com/iptv-org/iptv/actions/runs/${{ github.run_id }}
|
||||
pr_draft: true
|
||||
github_token: ${{ steps.generate-token.outputs.token }}
|
||||
@ -0,0 +1,74 @@
|
||||
const IPTVChecker = require('iptv-checker')
|
||||
const { program } = require('commander')
|
||||
const ProgressBar = require('progress')
|
||||
const parser = require('./helpers/parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-d, --debug', 'Enable debug mode')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
|
||||
.parse(process.argv)
|
||||
|
||||
let bar
|
||||
const config = program.opts()
|
||||
const ignoreStatus = ['Geo-blocked', 'Not 24/7', 'Offline']
|
||||
const checker = new IPTVChecker({
|
||||
timeout: config.timeout
|
||||
})
|
||||
|
||||
async function main() {
|
||||
log.start()
|
||||
|
||||
if (config.debug) log.print(`Debug mode enabled\n`)
|
||||
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = utils.filterPlaylists(playlists, config.country, config.exclude)
|
||||
for (const playlist of playlists) {
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(checkPlaylist)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function checkPlaylist(playlist) {
|
||||
if (!config.debug) {
|
||||
bar = new ProgressBar(`Checking '${playlist.url}': [:bar] :current/:total (:percent) `, {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
}
|
||||
const channels = []
|
||||
const total = playlist.channels.length
|
||||
for (const [index, channel] of playlist.channels.entries()) {
|
||||
const skipChannel =
|
||||
channel.status &&
|
||||
ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
|
||||
if (skipChannel) {
|
||||
channels.push(channel)
|
||||
} else {
|
||||
const result = await checker.checkStream(channel.data)
|
||||
if (result.status.ok || result.status.reason.includes('timed out')) {
|
||||
channels.push(channel)
|
||||
} else {
|
||||
if (config.debug) log.print(`ERR: ${channel.url} (${result.status.reason})\n`)
|
||||
}
|
||||
}
|
||||
if (!config.debug) bar.tick()
|
||||
}
|
||||
|
||||
if (playlist.channels.length !== channels.length) {
|
||||
log.print(`File '${playlist.url}' has been updated\n`)
|
||||
playlist.channels = channels
|
||||
playlist.updated = true
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
main()
|
||||
@ -0,0 +1,114 @@
|
||||
const { program } = require('commander')
|
||||
const ProgressBar = require('progress')
|
||||
const axios = require('axios')
|
||||
const https = require('https')
|
||||
const parser = require('./helpers/parser')
|
||||
const utils = require('./helpers/utils')
|
||||
const log = require('./helpers/log')
|
||||
|
||||
program
|
||||
.usage('[OPTIONS]...')
|
||||
.option('-c, --country <country>', 'Comma-separated list of country codes', '')
|
||||
.option('-e, --exclude <exclude>', 'Comma-separated list of country codes to be excluded', '')
|
||||
.option('--delay <delay>', 'Delay between parser requests', 1000)
|
||||
.option('--timeout <timeout>', 'Set timeout for each request', 5000)
|
||||
.parse(process.argv)
|
||||
|
||||
const config = program.opts()
|
||||
const ignoreStatus = ['Offline']
|
||||
const instance = axios.create({
|
||||
timeout: config.timeout,
|
||||
maxContentLength: 200000,
|
||||
httpsAgent: new https.Agent({
|
||||
rejectUnauthorized: false
|
||||
})
|
||||
})
|
||||
|
||||
async function main() {
|
||||
log.start()
|
||||
|
||||
log.print(`Parsing 'index.m3u'...\n`)
|
||||
let playlists = parser.parseIndex()
|
||||
playlists = utils
|
||||
.filterPlaylists(playlists, config.country, config.exclude)
|
||||
.filter(i => i.url !== 'channels/unsorted.m3u')
|
||||
|
||||
for (const playlist of playlists) {
|
||||
await parser
|
||||
.parsePlaylist(playlist.url)
|
||||
.then(detectResolution)
|
||||
.then(p => p.save())
|
||||
}
|
||||
|
||||
log.finish()
|
||||
}
|
||||
|
||||
async function detectResolution(playlist) {
|
||||
const channels = []
|
||||
const bar = new ProgressBar(`Processing '${playlist.url}': [:bar] :current/:total (:percent) `, {
|
||||
total: playlist.channels.length
|
||||
})
|
||||
let updated = false
|
||||
for (const channel of playlist.channels) {
|
||||
bar.tick()
|
||||
const skipChannel =
|
||||
channel.status &&
|
||||
ignoreStatus.map(i => i.toLowerCase()).includes(channel.status.toLowerCase())
|
||||
if (!channel.resolution.height && !skipChannel) {
|
||||
const CancelToken = axios.CancelToken
|
||||
const source = CancelToken.source()
|
||||
const timeout = setTimeout(() => {
|
||||
source.cancel()
|
||||
}, config.timeout)
|
||||
|
||||
const response = await instance
|
||||
.get(channel.url, { cancelToken: source.token })
|
||||
.then(res => {
|
||||
clearTimeout(timeout)
|
||||
|
||||
return res
|
||||
})
|
||||
.then(utils.sleep(config.delay))
|
||||
.catch(err => {
|
||||
clearTimeout(timeout)
|
||||
})
|
||||
|
||||
if (response && response.status === 200) {
|
||||
if (/^#EXTM3U/.test(response.data)) {
|
||||
const resolution = parseResolution(response.data)
|
||||
if (resolution) {
|
||||
channel.resolution = resolution
|
||||
updated = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
channels.push(channel)
|
||||
}
|
||||
|
||||
if (updated) {
|
||||
log.print(`File '${playlist.url}' has been updated\n`)
|
||||
playlist.channels = channels
|
||||
playlist.updated = true
|
||||
}
|
||||
|
||||
return playlist
|
||||
}
|
||||
|
||||
function parseResolution(string) {
|
||||
const regex = /RESOLUTION=(\d+)x(\d+)/gm
|
||||
const match = string.matchAll(regex)
|
||||
const arr = Array.from(match).map(m => ({
|
||||
width: parseInt(m[1]),
|
||||
height: parseInt(m[2])
|
||||
}))
|
||||
|
||||
return arr.length
|
||||
? arr.reduce(function (prev, current) {
|
||||
return prev.height > current.height ? prev : current
|
||||
})
|
||||
: undefined
|
||||
}
|
||||
|
||||
main()
|
||||
Loading…
Reference in New Issue