diff --git a/scripts/commands/playlist/format.ts b/scripts/commands/playlist/format.ts index 442b1cdf2f..20425d1b09 100644 --- a/scripts/commands/playlist/format.ts +++ b/scripts/commands/playlist/format.ts @@ -82,7 +82,7 @@ async function main() { logger.info('adding the missing quality...') const progressBar = new cliProgress.SingleBar({ clearOnComplete: true, - format: `[{bar}] {percentage}% | {value}/{total}` + format: '[{bar}] {percentage}% | {value}/{total}' }) progressBar.start(streams.count(), 0) await eachLimit(streams.all(), options.parallel, async (stream: Stream) => { diff --git a/scripts/commands/playlist/update.ts b/scripts/commands/playlist/update.ts index 38dd69b7e0..5165b0b2b0 100644 --- a/scripts/commands/playlist/update.ts +++ b/scripts/commands/playlist/update.ts @@ -1,20 +1,19 @@ -import { IssueLoader, PlaylistParser } from '../../core' +import { isURI, getStreamInfo, loadIssues } from '../../utils' import { Playlist, Issue, Stream } from '../../models' import { loadData, data as apiData } from '../../api' import { Logger, Collection } from '@freearhey/core' -import { isURI, getStreamInfo } from '../../utils' import { Storage } from '@freearhey/storage-js' import { STREAMS_DIR } from '../../constants' +import { PlaylistParser } from '../../core' import * as sdk from '@iptv-org/sdk' const processedIssues = new Collection() async function main() { const logger = new Logger({ level: -999 }) - const issueLoader = new IssueLoader() logger.info('loading issues...') - const issues = await issueLoader.load() + const issues = await loadIssues() logger.info('loading data from api...') await loadData() diff --git a/scripts/commands/report/create.ts b/scripts/commands/report/create.ts index 93488471d3..b1af2f4fb6 100644 --- a/scripts/commands/report/create.ts +++ b/scripts/commands/report/create.ts @@ -1,9 +1,9 @@ +import { isURI, truncate, loadIssues, loadDiscussions } from '../../utils' import { Logger, Collection, Dictionary } from '@freearhey/core' -import { IssueLoader, PlaylistParser } from '../../core' import { Storage } from '@freearhey/storage-js' -import { isURI, truncate } from '../../utils' import { STREAMS_DIR } from '../../constants' -import { Issue, Stream } from '../../models' +import { Discussion, Issue, Stream } from '../../models' +import { PlaylistParser } from '../../core' import { data, loadData } from '../../api' const status = { @@ -22,11 +22,13 @@ const status = { async function main() { const logger = new Logger() - const issueLoader = new IssueLoader() let report = new Collection() logger.info('loading issues...') - const issues = await issueLoader.load() + const issues = await loadIssues() + + logger.info('loading discussions...') + const discussions = await loadDiscussions() logger.info('loading data from api...') await loadData() @@ -135,16 +137,17 @@ async function main() { }) logger.info('checking channel search requests...') - const channelSearchRequests = issues.filter(issue => - issue.labels.find((label: string) => label === 'channel search') + const channelSearchRequests = discussions.filter( + (discussion: Discussion) => discussion.category === 'Channel Search' ) const channelSearchRequestsBuffer = new Dictionary() - channelSearchRequests.forEach((issue: Issue) => { - const streamId = issue.data.getString('stream_id') || issue.data.getString('channel_id') || '' + channelSearchRequests.forEach((discussion: Discussion) => { + const streamId = + discussion.data.getString('stream_id') || discussion.data.getString('channel_id') || '' const [channelId, feedId] = streamId.split('@') const result = { - issueNumber: issue.number, + issueNumber: discussion.number, type: 'channel search', streamId: streamId || undefined, streamUrl: undefined, diff --git a/scripts/core/issueData.ts b/scripts/core/dataSet.ts similarity index 92% rename from scripts/core/issueData.ts rename to scripts/core/dataSet.ts index a38209250a..14dd1771f4 100644 --- a/scripts/core/issueData.ts +++ b/scripts/core/dataSet.ts @@ -1,6 +1,6 @@ import { Dictionary } from '@freearhey/core' -export class IssueData { +export class DataSet { _data: Dictionary constructor(data: Dictionary) { this._data = data diff --git a/scripts/core/index.ts b/scripts/core/index.ts index 850b9d9cfb..c273006509 100644 --- a/scripts/core/index.ts +++ b/scripts/core/index.ts @@ -1,8 +1,6 @@ export * from './cliTable' export * from './htmlTable' -export * from './issueData' -export * from './issueLoader' -export * from './issueParser' +export * from './dataSet' export * from './logParser' export * from './markdown' export * from './numberParser' diff --git a/scripts/core/issueLoader.ts b/scripts/core/issueLoader.ts deleted file mode 100644 index 43fb2185b6..0000000000 --- a/scripts/core/issueLoader.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods' -import { paginateRest } from '@octokit/plugin-paginate-rest' -import { TESTING, OWNER, REPO } from '../constants' -import { Collection } from '@freearhey/core' -import { Octokit } from '@octokit/core' -import { IssueParser } from './' - -const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods) -const octokit = new CustomOctokit() - -export class IssueLoader { - async load(props?: { labels: string | string[] }) { - let labels = '' - if (props && props.labels) { - labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels - } - let issues: object[] = [] - if (TESTING) { - issues = (await import('../../tests/__data__/input/issues.js')).default - } else { - issues = await octokit.paginate(octokit.rest.issues.listForRepo, { - owner: OWNER, - repo: REPO, - per_page: 100, - labels, - status: 'open', - headers: { - 'X-GitHub-Api-Version': '2022-11-28' - } - }) - } - - const parser = new IssueParser() - - return new Collection(issues).map(parser.parse) - } -} diff --git a/scripts/core/issueParser.ts b/scripts/core/issueParser.ts deleted file mode 100644 index 54ebd694b1..0000000000 --- a/scripts/core/issueParser.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Dictionary } from '@freearhey/core' -import { IssueData } from './issueData' -import { Issue } from '../models' - -const FIELDS = new Dictionary({ - 'Stream ID': 'stream_id', - 'Channel ID': 'channel_id', - 'Feed ID': 'feed_id', - 'Stream URL': 'stream_url', - Label: 'label', - Quality: 'quality', - 'HTTP User-Agent': 'http_user_agent', - 'HTTP User Agent': 'http_user_agent', - 'HTTP Referrer': 'http_referrer', - 'What happened to the stream?': 'reason', - Reason: 'reason', - Notes: 'notes' -}) - -export class IssueParser { - parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue { - const fields = typeof issue.body === 'string' ? issue.body.split('###') : [] - - const data = new Dictionary() - fields.forEach((field: string) => { - const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : [] - let _label = parsed.shift() - _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : '' - let _value = parsed.join('\r\n') - _value = _value ? _value.trim() : '' - - if (!_label || !_value) return data - - const id = FIELDS.get(_label) - const value: string = _value === '_No response_' || _value === 'None' ? '' : _value - - if (!id) return - - data.set(id, value) - }) - - const labels = issue.labels.map(label => label.name) - - return new Issue({ number: issue.number, labels, data: new IssueData(data) }) - } -} diff --git a/scripts/models/discussion.ts b/scripts/models/discussion.ts new file mode 100644 index 0000000000..b78f8d158e --- /dev/null +++ b/scripts/models/discussion.ts @@ -0,0 +1,19 @@ +import { DataSet } from '../core' + +type DiscussionProps = { + number: number + category: string + data: DataSet +} + +export class Discussion { + number: number + category: string + data: DataSet + + constructor({ number, category, data }: DiscussionProps) { + this.number = number + this.category = category + this.data = data + } +} diff --git a/scripts/models/index.ts b/scripts/models/index.ts index f4b06f6dd5..f0ea5094a0 100644 --- a/scripts/models/index.ts +++ b/scripts/models/index.ts @@ -1,3 +1,4 @@ export * from './issue' export * from './playlist' export * from './stream' +export * from './discussion' diff --git a/scripts/utils.ts b/scripts/utils.ts index eb3b4e2f0b..439a143363 100644 --- a/scripts/utils.ts +++ b/scripts/utils.ts @@ -1,10 +1,16 @@ +import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods' import axios, { AxiosProxyConfig, AxiosRequestConfig } from 'axios' +import { paginateGraphQL } from '@octokit/plugin-paginate-graphql' import { parse as parsePlaylist, setOptions } from 'hls-parser' -import { parse as parseManifest } from 'mpd-parser' +import { paginateRest } from '@octokit/plugin-paginate-rest' +import { Collection, Dictionary } from '@freearhey/core' import { SocksProxyAgent } from 'socks-proxy-agent' -import { ProxyParser } from './core/proxyParser.js' -import { TESTING } from './constants.js' +import { parse as parseManifest } from 'mpd-parser' +import { TESTING, OWNER, REPO } from './constants' +import { ProxyParser, DataSet } from './core' +import { Discussion, Issue } from './models' import normalizeUrl from 'normalize-url' +import { Octokit } from '@octokit/core' import { orderBy } from 'es-toolkit' import path from 'node:path' import fs from 'node:fs' @@ -90,7 +96,9 @@ export async function getStreamInfo( const response = await axios(url, request) data = response.data - } catch {} + } catch { + // do nothing + } } if (!data) return undefined @@ -115,7 +123,9 @@ export async function getStreamInfo( } } } - } catch {} + } catch { + // do nothing + } } else if (url.includes('.mpd')) { const manifest = parseManifest(data, { manifestUri: url, @@ -138,3 +148,148 @@ export async function getStreamInfo( return info } + +export async function loadIssues(props?: { labels: string | string[] }) { + const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods) + const octokit = new CustomOctokit() + + let labels = '' + if (props && props.labels) { + labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels + } + let issues: object[] = [] + if (TESTING) { + issues = (await import('../tests/__data__/input/issues.js')).default + } else { + issues = await octokit.paginate(octokit.rest.issues.listForRepo, { + owner: OWNER, + repo: REPO, + per_page: 100, + labels, + status: 'open', + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + } + + return new Collection(issues).map(parseIssue) +} + +function parseIssue(issue: { number: number; body: string; labels: { name: string }[] }): Issue { + const FIELDS = new Dictionary({ + 'Stream ID': 'stream_id', + 'Channel ID': 'channel_id', + 'Feed ID': 'feed_id', + 'Stream URL': 'stream_url', + Label: 'label', + Quality: 'quality', + 'HTTP User-Agent': 'http_user_agent', + 'HTTP User Agent': 'http_user_agent', + 'HTTP Referrer': 'http_referrer', + 'What happened to the stream?': 'reason', + Reason: 'reason', + Notes: 'notes' + }) + + const fields = typeof issue.body === 'string' ? issue.body.split('###') : [] + + const data = new Dictionary() + fields.forEach((field: string) => { + const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : [] + let _label = parsed.shift() + _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : '' + let _value = parsed.join('\r\n') + _value = _value ? _value.trim() : '' + + if (!_label || !_value) return data + + const id = FIELDS.get(_label) + const value: string = _value === '_No response_' || _value === 'None' ? '' : _value + + if (!id) return + + data.set(id, value) + }) + + const labels = issue.labels.map(label => label.name) + + return new Issue({ number: issue.number, labels, data: new DataSet(data) }) +} + +export async function loadDiscussions() { + let discussions: object[] = [] + if (TESTING) { + discussions = (await import('../tests/__data__/input/discussions.js')).default + } else { + const CustomOctokit = Octokit.plugin(paginateGraphQL) + const octokit = new CustomOctokit({ + auth: process.env.GITHUB_TOKEN + }) + + const query = ` + query ($owner: String!, $repo: String!, $cursor: String) { + repository(owner: $owner, name: $repo) { + discussions(first: 100, after: $cursor, states: OPEN) { + nodes { + number + body + category { + name + } + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + ` + + const result = await octokit.graphql.paginate(query, { + owner: 'iptv-org', + repo: 'iptv' + }) + + discussions = result.repository.discussions.nodes + } + + return new Collection(discussions).map(parseDiscussion) +} + +function parseDiscussion(discussion: { + number: number + category: { name: string } + body: string +}): Discussion { + const FIELDS = new Dictionary({ + 'Stream ID': 'stream_id' + }) + + const fields = typeof discussion.body === 'string' ? discussion.body.split('###') : [] + + const data = new Dictionary() + fields.forEach((field: string) => { + const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : [] + let _label = parsed.shift() + _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : '' + let _value = parsed.join('\r\n') + _value = _value ? _value.trim() : '' + + if (!_label || !_value) return data + + const id = FIELDS.get(_label) + const value: string = _value === '_No response_' || _value === 'None' ? '' : _value + + if (!id) return + + data.set(id, value) + }) + + return new Discussion({ + number: discussion.number, + category: discussion.category.name, + data: new DataSet(data) + }) +}