Update scripts

scripts/commands/playlist/format.ts

@@ -82,7 +82,7 @@ async function main() {
   logger.info('adding the missing quality...')
   const progressBar = new cliProgress.SingleBar({
     clearOnComplete: true,
-    format: `[{bar}] {percentage}% | {value}/{total}`
+    format: '[{bar}] {percentage}% | {value}/{total}'
   })
   progressBar.start(streams.count(), 0)
   await eachLimit(streams.all(), options.parallel, async (stream: Stream) => {

scripts/commands/playlist/update.ts

@@ -1,20 +1,19 @@
-import { IssueLoader, PlaylistParser } from '../../core'
+import { isURI, getStreamInfo, loadIssues } from '../../utils'
 import { Playlist, Issue, Stream } from '../../models'
 import { loadData, data as apiData } from '../../api'
 import { Logger, Collection } from '@freearhey/core'
-import { isURI, getStreamInfo } from '../../utils'
 import { Storage } from '@freearhey/storage-js'
 import { STREAMS_DIR } from '../../constants'
+import { PlaylistParser } from '../../core'
 import * as sdk from '@iptv-org/sdk'
 
 const processedIssues = new Collection()
 
 async function main() {
   const logger = new Logger({ level: -999 })
-  const issueLoader = new IssueLoader()
 
   logger.info('loading issues...')
-  const issues = await issueLoader.load()
+  const issues = await loadIssues()
 
   logger.info('loading data from api...')
   await loadData()

scripts/commands/report/create.ts

@@ -1,9 +1,9 @@
+import { isURI, truncate, loadIssues, loadDiscussions } from '../../utils'
 import { Logger, Collection, Dictionary } from '@freearhey/core'
-import { IssueLoader, PlaylistParser } from '../../core'
 import { Storage } from '@freearhey/storage-js'
-import { isURI, truncate } from '../../utils'
 import { STREAMS_DIR } from '../../constants'
-import { Issue, Stream } from '../../models'
+import { Discussion, Issue, Stream } from '../../models'
+import { PlaylistParser } from '../../core'
 import { data, loadData } from '../../api'
 
 const status = {
@@ -22,11 +22,13 @@ const status = {
 
 async function main() {
   const logger = new Logger()
-  const issueLoader = new IssueLoader()
   let report = new Collection()
 
   logger.info('loading issues...')
-  const issues = await issueLoader.load()
+  const issues = await loadIssues()
+
+  logger.info('loading discussions...')
+  const discussions = await loadDiscussions()
 
   logger.info('loading data from api...')
   await loadData()
@@ -135,16 +137,17 @@ async function main() {
   })
 
   logger.info('checking channel search requests...')
-  const channelSearchRequests = issues.filter(issue =>
+  const channelSearchRequests = discussions.filter(
-    issue.labels.find((label: string) => label === 'channel search')
+    (discussion: Discussion) => discussion.category === 'Channel Search'
   )
   const channelSearchRequestsBuffer = new Dictionary()
-  channelSearchRequests.forEach((issue: Issue) => {
+  channelSearchRequests.forEach((discussion: Discussion) => {
-    const streamId = issue.data.getString('stream_id') || issue.data.getString('channel_id') || ''
+    const streamId =
+      discussion.data.getString('stream_id') || discussion.data.getString('channel_id') || ''
     const [channelId, feedId] = streamId.split('@')
 
     const result = {
-      issueNumber: issue.number,
+      issueNumber: discussion.number,
       type: 'channel search',
       streamId: streamId || undefined,
       streamUrl: undefined,

scripts/core/dataSet.ts

@@ -1,6 +1,6 @@
 import { Dictionary } from '@freearhey/core'
 
-export class IssueData {
+export class DataSet {
   _data: Dictionary<string>
   constructor(data: Dictionary<string>) {
     this._data = data

scripts/core/index.ts

@@ -1,8 +1,6 @@
 export * from './cliTable'
 export * from './htmlTable'
-export * from './issueData'
+export * from './dataSet'
-export * from './issueLoader'
-export * from './issueParser'
 export * from './logParser'
 export * from './markdown'
 export * from './numberParser'

scripts/core/issueLoader.ts

@@ -1,37 +0,0 @@
-import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
-import { paginateRest } from '@octokit/plugin-paginate-rest'
-import { TESTING, OWNER, REPO } from '../constants'
-import { Collection } from '@freearhey/core'
-import { Octokit } from '@octokit/core'
-import { IssueParser } from './'
-
-const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
-const octokit = new CustomOctokit()
-
-export class IssueLoader {
-  async load(props?: { labels: string | string[] }) {
-    let labels = ''
-    if (props && props.labels) {
-      labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels
-    }
-    let issues: object[] = []
-    if (TESTING) {
-      issues = (await import('../../tests/__data__/input/issues.js')).default
-    } else {
-      issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
-        owner: OWNER,
-        repo: REPO,
-        per_page: 100,
-        labels,
-        status: 'open',
-        headers: {
-          'X-GitHub-Api-Version': '2022-11-28'
-        }
-      })
-    }
-
-    const parser = new IssueParser()
-
-    return new Collection(issues).map(parser.parse)
-  }
-}

scripts/core/issueParser.ts

@@ -1,46 +0,0 @@
-import { Dictionary } from '@freearhey/core'
-import { IssueData } from './issueData'
-import { Issue } from '../models'
-
-const FIELDS = new Dictionary({
-  'Stream ID': 'stream_id',
-  'Channel ID': 'channel_id',
-  'Feed ID': 'feed_id',
-  'Stream URL': 'stream_url',
-  Label: 'label',
-  Quality: 'quality',
-  'HTTP User-Agent': 'http_user_agent',
-  'HTTP User Agent': 'http_user_agent',
-  'HTTP Referrer': 'http_referrer',
-  'What happened to the stream?': 'reason',
-  Reason: 'reason',
-  Notes: 'notes'
-})
-
-export class IssueParser {
-  parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue {
-    const fields = typeof issue.body === 'string' ? issue.body.split('###') : []
-
-    const data = new Dictionary<string>()
-    fields.forEach((field: string) => {
-      const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : []
-      let _label = parsed.shift()
-      _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : ''
-      let _value = parsed.join('\r\n')
-      _value = _value ? _value.trim() : ''
-
-      if (!_label || !_value) return data
-
-      const id = FIELDS.get(_label)
-      const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
-
-      if (!id) return
-
-      data.set(id, value)
-    })
-
-    const labels = issue.labels.map(label => label.name)
-
-    return new Issue({ number: issue.number, labels, data: new IssueData(data) })
-  }
-}

scripts/models/discussion.ts

@@ -0,0 +1,19 @@
+import { DataSet } from '../core'
+
+type DiscussionProps = {
+  number: number
+  category: string
+  data: DataSet
+}
+
+export class Discussion {
+  number: number
+  category: string
+  data: DataSet
+
+  constructor({ number, category, data }: DiscussionProps) {
+    this.number = number
+    this.category = category
+    this.data = data
+  }
+}

scripts/models/index.ts

@@ -1,3 +1,4 @@
 export * from './issue'
 export * from './playlist'
 export * from './stream'
+export * from './discussion'

scripts/utils.ts

@@ -1,10 +1,16 @@
+import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
 import axios, { AxiosProxyConfig, AxiosRequestConfig } from 'axios'
+import { paginateGraphQL } from '@octokit/plugin-paginate-graphql'
 import { parse as parsePlaylist, setOptions } from 'hls-parser'
-import { parse as parseManifest } from 'mpd-parser'
+import { paginateRest } from '@octokit/plugin-paginate-rest'
+import { Collection, Dictionary } from '@freearhey/core'
 import { SocksProxyAgent } from 'socks-proxy-agent'
-import { ProxyParser } from './core/proxyParser.js'
+import { parse as parseManifest } from 'mpd-parser'
-import { TESTING } from './constants.js'
+import { TESTING, OWNER, REPO } from './constants'
+import { ProxyParser, DataSet } from './core'
+import { Discussion, Issue } from './models'
 import normalizeUrl from 'normalize-url'
+import { Octokit } from '@octokit/core'
 import { orderBy } from 'es-toolkit'
 import path from 'node:path'
 import fs from 'node:fs'
@@ -90,7 +96,9 @@ export async function getStreamInfo(
       const response = await axios(url, request)
 
       data = response.data
-    } catch {}
+    } catch {
+      // do nothing
+    }
   }
 
   if (!data) return undefined
@@ -115,7 +123,9 @@ export async function getStreamInfo(
           }
         }
       }
-    } catch {}
+    } catch {
+      // do nothing
+    }
   } else if (url.includes('.mpd')) {
     const manifest = parseManifest(data, {
       manifestUri: url,
@@ -138,3 +148,148 @@ export async function getStreamInfo(
 
   return info
 }
+
+export async function loadIssues(props?: { labels: string | string[] }) {
+  const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
+  const octokit = new CustomOctokit()
+
+  let labels = ''
+  if (props && props.labels) {
+    labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels
+  }
+  let issues: object[] = []
+  if (TESTING) {
+    issues = (await import('../tests/__data__/input/issues.js')).default
+  } else {
+    issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
+      owner: OWNER,
+      repo: REPO,
+      per_page: 100,
+      labels,
+      status: 'open',
+      headers: {
+        'X-GitHub-Api-Version': '2022-11-28'
+      }
+    })
+  }
+
+  return new Collection(issues).map(parseIssue)
+}
+
+function parseIssue(issue: { number: number; body: string; labels: { name: string }[] }): Issue {
+  const FIELDS = new Dictionary({
+    'Stream ID': 'stream_id',
+    'Channel ID': 'channel_id',
+    'Feed ID': 'feed_id',
+    'Stream URL': 'stream_url',
+    Label: 'label',
+    Quality: 'quality',
+    'HTTP User-Agent': 'http_user_agent',
+    'HTTP User Agent': 'http_user_agent',
+    'HTTP Referrer': 'http_referrer',
+    'What happened to the stream?': 'reason',
+    Reason: 'reason',
+    Notes: 'notes'
+  })
+
+  const fields = typeof issue.body === 'string' ? issue.body.split('###') : []
+
+  const data = new Dictionary<string>()
+  fields.forEach((field: string) => {
+    const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : []
+    let _label = parsed.shift()
+    _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : ''
+    let _value = parsed.join('\r\n')
+    _value = _value ? _value.trim() : ''
+
+    if (!_label || !_value) return data
+
+    const id = FIELDS.get(_label)
+    const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
+
+    if (!id) return
+
+    data.set(id, value)
+  })
+
+  const labels = issue.labels.map(label => label.name)
+
+  return new Issue({ number: issue.number, labels, data: new DataSet(data) })
+}
+
+export async function loadDiscussions() {
+  let discussions: object[] = []
+  if (TESTING) {
+    discussions = (await import('../tests/__data__/input/discussions.js')).default
+  } else {
+    const CustomOctokit = Octokit.plugin(paginateGraphQL)
+    const octokit = new CustomOctokit({
+      auth: process.env.GITHUB_TOKEN
+    })
+
+    const query = `
+      query ($owner: String!, $repo: String!, $cursor: String) {
+        repository(owner: $owner, name: $repo) {
+          discussions(first: 100, after: $cursor, states: OPEN) {
+            nodes {
+              number
+              body
+              category {
+                name
+              }
+            }
+            pageInfo {
+              endCursor
+              hasNextPage
+            }
+          }
+        }
+      }
+    `
+
+    const result = await octokit.graphql.paginate(query, {
+      owner: 'iptv-org',
+      repo: 'iptv'
+    })
+
+    discussions = result.repository.discussions.nodes
+  }
+
+  return new Collection(discussions).map(parseDiscussion)
+}
+
+function parseDiscussion(discussion: {
+  number: number
+  category: { name: string }
+  body: string
+}): Discussion {
+  const FIELDS = new Dictionary({
+    'Stream ID': 'stream_id'
+  })
+
+  const fields = typeof discussion.body === 'string' ? discussion.body.split('###') : []
+
+  const data = new Dictionary<string>()
+  fields.forEach((field: string) => {
+    const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : []
+    let _label = parsed.shift()
+    _label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : ''
+    let _value = parsed.join('\r\n')
+    _value = _value ? _value.trim() : ''
+
+    if (!_label || !_value) return data
+
+    const id = FIELDS.get(_label)
+    const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
+
+    if (!id) return
+
+    data.set(id, value)
+  })
+
+  return new Discussion({
+    number: discussion.number,
+    category: discussion.category.name,
+    data: new DataSet(data)
+  })
+}