Create scripts/core

scripts/core/checker.js

@@ -0,0 +1,19 @@
+const IPTVChecker = require('iptv-checker')
+
+const checker = {}
+
+checker.check = async function (item, config) {
+  const ic = new IPTVChecker(config)
+  const result = await ic.checkStream({ url: item.url, http: item.http })
+
+  return {
+    _id: item._id,
+    url: item.url,
+    http: item.http,
+    error: !result.status.ok ? result.status.reason : null,
+    streams: result.status.ok ? result.status.metadata.streams : [],
+    requests: result.status.ok ? result.status.metadata.requests : []
+  }
+}
+
+module.exports = checker

scripts/core/db.js

@@ -0,0 +1,61 @@
+const Database = require('nedb-promises')
+const file = require('./file')
+
+const DB_FILEPATH = process.env.DB_FILEPATH || './scripts/channels.db'
+
+const nedb = Database.create({
+  filename: file.resolve(DB_FILEPATH),
+  autoload: true,
+  onload(err) {
+    if (err) console.error(err)
+  },
+  compareStrings: (a, b) => {
+    a = a.replace(/\s/g, '_')
+    b = b.replace(/\s/g, '_')
+
+    return a.localeCompare(b, undefined, {
+      sensitivity: 'accent',
+      numeric: true
+    })
+  }
+})
+
+const db = {}
+
+db.removeIndex = function (field) {
+  return nedb.removeIndex(field)
+}
+
+db.addIndex = function (options) {
+  return nedb.ensureIndex(options)
+}
+
+db.compact = function () {
+  return nedb.persistence.compactDatafile()
+}
+
+db.reset = function () {
+  return file.clear(DB_FILEPATH)
+}
+
+db.count = function (query) {
+  return nedb.count(query)
+}
+
+db.insert = function (doc) {
+  return nedb.insert(doc)
+}
+
+db.update = function (query, update) {
+  return nedb.update(query, update)
+}
+
+db.find = function (query) {
+  return nedb.find(query)
+}
+
+db.remove = function (query, options) {
+  return nedb.remove(query, options)
+}
+
+module.exports = db

scripts/core/file.js

@@ -0,0 +1,67 @@
+const path = require('path')
+const glob = require('glob')
+const fs = require('mz/fs')
+
+const file = {}
+
+file.list = function (pattern) {
+  return new Promise(resolve => {
+    glob(pattern, function (err, files) {
+      resolve(files)
+    })
+  })
+}
+
+file.getFilename = function (filepath) {
+  return path.parse(filepath).name
+}
+
+file.createDir = async function (dir) {
+  if (await file.exists(dir)) return
+
+  return fs.mkdir(dir, { recursive: true }).catch(console.error)
+}
+
+file.exists = function (filepath) {
+  return fs.exists(path.resolve(filepath))
+}
+
+file.read = function (filepath) {
+  return fs.readFile(path.resolve(filepath), { encoding: 'utf8' }).catch(console.error)
+}
+
+file.append = function (filepath, data) {
+  return fs.appendFile(path.resolve(filepath), data).catch(console.error)
+}
+
+file.create = function (filepath, data = '') {
+  filepath = path.resolve(filepath)
+  const dir = path.dirname(filepath)
+
+  return file
+    .createDir(dir)
+    .then(() => fs.writeFile(filepath, data, { encoding: 'utf8', flag: 'w' }))
+    .catch(console.error)
+}
+
+file.write = function (filepath, data = '') {
+  return fs.writeFile(path.resolve(filepath), data).catch(console.error)
+}
+
+file.clear = function (filepath) {
+  return file.write(filepath, '')
+}
+
+file.resolve = function (filepath) {
+  return path.resolve(filepath)
+}
+
+file.dirname = function (filepath) {
+  return path.dirname(filepath)
+}
+
+file.basename = function (filepath) {
+  return path.basename(filepath)
+}
+
+module.exports = file

scripts/core/generator.js

@@ -0,0 +1,114 @@
+const { create: createPlaylist } = require('./playlist')
+const store = require('./store')
+const file = require('./file')
+const logger = require('./logger')
+const db = require('./db')
+const _ = require('lodash')
+
+const generator = {}
+
+generator.generate = async function (filepath, query = {}, options = {}) {
+  options = {
+    ...{
+      format: 'm3u',
+      saveEmpty: false,
+      includeNSFW: false,
+      includeGuides: true,
+      includeBroken: false,
+      onLoad: r => r,
+      uniqBy: item => item.id || _.uniqueId(),
+      sortBy: null
+    },
+    ...options
+  }
+
+  query['is_nsfw'] = options.includeNSFW ? { $in: [true, false] } : false
+  query['is_broken'] = options.includeBroken ? { $in: [true, false] } : false
+
+  let items = await db
+    .find(query)
+    .sort({ name: 1, 'status.level': 1, 'resolution.height': -1, url: 1 })
+
+  items = _.uniqBy(items, 'url')
+  if (!options.saveEmpty && !items.length) return { filepath, query, options, count: 0 }
+  if (options.uniqBy) items = _.uniqBy(items, options.uniqBy)
+
+  items = options.onLoad(items)
+
+  if (options.sortBy) items = _.sortBy(items, options.sortBy)
+
+  switch (options.format) {
+    case 'json':
+      await saveAsJSON(filepath, items, options)
+      break
+    case 'm3u':
+    default:
+      await saveAsM3U(filepath, items, options)
+      break
+  }
+
+  return { filepath, query, options, count: items.length }
+}
+
+async function saveAsM3U(filepath, items, options) {
+  const playlist = await createPlaylist(filepath)
+
+  const header = {}
+  if (options.includeGuides) {
+    let guides = items.map(item => item.guides)
+    guides = _.uniq(_.flatten(guides)).sort().join(',')
+
+    header['x-tvg-url'] = guides
+  }
+
+  await playlist.header(header)
+  for (const item of items) {
+    const stream = store.create(item)
+    await playlist.link(
+      stream.get('url'),
+      stream.get('title'),
+      {
+        'tvg-id': stream.get('tvg_id'),
+        'tvg-country': stream.get('tvg_country'),
+        'tvg-language': stream.get('tvg_language'),
+        'tvg-logo': stream.get('tvg_logo'),
+        // 'tvg-url': stream.get('tvg_url') || undefined,
+        'user-agent': stream.get('http.user-agent') || undefined,
+        'group-title': stream.get('group_title')
+      },
+      {
+        'http-referrer': stream.get('http.referrer') || undefined,
+        'http-user-agent': stream.get('http.user-agent') || undefined
+      }
+    )
+  }
+}
+
+async function saveAsJSON(filepath, items, options) {
+  const output = items.map(item => {
+    const stream = store.create(item)
+    const categories = stream.get('categories').map(c => ({ name: c.name, slug: c.slug }))
+    const countries = stream.get('countries').map(c => ({ name: c.name, code: c.code }))
+
+    return {
+      name: stream.get('name'),
+      logo: stream.get('logo'),
+      url: stream.get('url'),
+      categories,
+      countries,
+      languages: stream.get('languages'),
+      tvg: {
+        id: stream.get('tvg_id'),
+        name: stream.get('name'),
+        url: stream.get('tvg_url')
+      }
+    }
+  })
+
+  await file.create(filepath, JSON.stringify(output))
+}
+
+generator.saveAsM3U = saveAsM3U
+generator.saveAsJSON = saveAsJSON
+
+module.exports = generator

scripts/core/index.js

@@ -0,0 +1,10 @@
+exports.db = require('./db')
+exports.logger = require('./logger')
+exports.file = require('./file')
+exports.timer = require('./timer')
+exports.parser = require('./parser')
+exports.checker = require('./checker')
+exports.generator = require('./generator')
+exports.playlist = require('./playlist')
+exports.store = require('./store')
+exports.markdown = require('./markdown')

scripts/core/logger.js

@@ -0,0 +1,42 @@
+const { createLogger, format, transports, addColors } = require('winston')
+const { combine, timestamp, printf } = format
+
+const consoleFormat = ({ level, message, timestamp }) => {
+  if (typeof message === 'object') return JSON.stringify(message)
+  return message
+}
+
+const config = {
+  levels: {
+    error: 0,
+    warn: 1,
+    info: 2,
+    failed: 3,
+    success: 4,
+    http: 5,
+    verbose: 6,
+    debug: 7,
+    silly: 8
+  },
+  colors: {
+    info: 'white',
+    success: 'green',
+    failed: 'red'
+  }
+}
+
+const t = [
+  new transports.Console({
+    format: format.combine(format.printf(consoleFormat))
+  })
+]
+
+const logger = createLogger({
+  transports: t,
+  levels: config.levels,
+  level: 'verbose'
+})
+
+addColors(config.colors)
+
+module.exports = logger

scripts/core/markdown.js

@@ -0,0 +1,39 @@
+const markdownInclude = require('markdown-include')
+const file = require('./file')
+
+const markdown = {}
+
+markdown.createTable = function (data, cols) {
+  let output = '<table>\n'
+
+  output += '  <thead>\n    <tr>'
+  for (let column of cols) {
+    output += `<th align="${column.align}">${column.name}</th>`
+  }
+  output += '</tr>\n  </thead>\n'
+
+  output += '  <tbody>\n'
+  for (let item of data) {
+    output += '    <tr>'
+    let i = 0
+    for (let prop in item) {
+      const column = cols[i]
+      let nowrap = column.nowrap
+      let align = column.align
+      output += `<td align="${align}"${nowrap ? ' nowrap' : ''}>${item[prop]}</td>`
+      i++
+    }
+    output += '</tr>\n'
+  }
+  output += '  </tbody>\n'
+
+  output += '</table>'
+
+  return output
+}
+
+markdown.compile = function (filepath) {
+  markdownInclude.compileFiles(file.resolve(filepath))
+}
+
+module.exports = markdown

scripts/core/parser.js

@@ -0,0 +1,31 @@
+const ipp = require('iptv-playlist-parser')
+const logger = require('./logger')
+const file = require('./file')
+
+const parser = {}
+
+parser.parsePlaylist = async function (filepath) {
+  const content = await file.read(filepath)
+  const playlist = ipp.parse(content)
+
+  return playlist.items
+}
+
+parser.parseLogs = async function (filepath) {
+  const content = await file.read(filepath)
+  if (!content) return []
+  const lines = content.split('\n')
+
+  return lines.map(line => (line ? JSON.parse(line) : null)).filter(l => l)
+}
+
+parser.parseNumber = function (string) {
+  const parsed = parseInt(string)
+  if (isNaN(parsed)) {
+    logger.error('Not a number')
+  }
+
+  return parsed
+}
+
+module.exports = parser

scripts/core/playlist.js

@@ -0,0 +1,49 @@
+const file = require('./file')
+
+const playlist = {}
+
+playlist.create = async function (filepath) {
+  playlist.filepath = filepath
+  const dir = file.dirname(filepath)
+  file.createDir(dir)
+  await file.create(filepath, '')
+
+  return playlist
+}
+
+playlist.header = async function (attrs) {
+  let header = `#EXTM3U`
+  for (const name in attrs) {
+    const value = attrs[name]
+    header += ` ${name}="${value}"`
+  }
+  header += `\n`
+
+  await file.append(playlist.filepath, header)
+
+  return playlist
+}
+
+playlist.link = async function (url, title, attrs, vlcOpts) {
+  let link = `#EXTINF:-1`
+  for (const name in attrs) {
+    const value = attrs[name]
+    if (value !== undefined) {
+      link += ` ${name}="${value}"`
+    }
+  }
+  link += `,${title}\n`
+  for (const name in vlcOpts) {
+    const value = vlcOpts[name]
+    if (value !== undefined) {
+      link += `#EXTVLCOPT:${name}=${value}\n`
+    }
+  }
+  link += `${url}\n`
+
+  await file.append(playlist.filepath, link)
+
+  return playlist
+}
+
+module.exports = playlist

scripts/core/store.js

@@ -0,0 +1,56 @@
+const _ = require('lodash')
+const logger = require('./logger')
+const setters = require('../store/setters')
+const getters = require('../store/getters')
+
+module.exports = {
+  create(state = {}) {
+    return {
+      state,
+      changed: false,
+      set: function (prop, value) {
+        const prevState = JSON.stringify(this.state)
+
+        const setter = setters[prop]
+        if (typeof setter === 'function') {
+          try {
+            this.state[prop] = setter.bind()(value)
+          } catch (error) {
+            logger.error(`store/setters/${prop}.js: ${error.message}`)
+          }
+        } else if (typeof value === 'object') {
+          this.state[prop] = value[prop]
+        } else {
+          this.state[prop] = value
+        }
+
+        const newState = JSON.stringify(this.state)
+        if (prevState !== newState) {
+          this.changed = true
+        }
+
+        return this
+      },
+      get: function (prop) {
+        const getter = getters[prop]
+        if (typeof getter === 'function') {
+          try {
+            return getter.bind(this.state)()
+          } catch (error) {
+            logger.error(`store/getters/${prop}.js: ${error.message}`)
+          }
+        } else {
+          return prop.split('.').reduce((o, i) => (o ? o[i] : undefined), this.state)
+        }
+      },
+      has: function (prop) {
+        const value = this.get(prop)
+
+        return !_.isEmpty(value)
+      },
+      data: function () {
+        return this.state
+      }
+    }
+  }
+}

scripts/core/timer.js

@@ -0,0 +1,29 @@
+const { performance } = require('perf_hooks')
+const dayjs = require('dayjs')
+const duration = require('dayjs/plugin/duration')
+const relativeTime = require('dayjs/plugin/relativeTime')
+
+dayjs.extend(relativeTime)
+dayjs.extend(duration)
+
+const timer = {}
+
+let t0 = 0
+
+timer.start = function () {
+  t0 = performance.now()
+}
+
+timer.format = function (f) {
+  let t1 = performance.now()
+
+  return dayjs.duration(t1 - t0).format(f)
+}
+
+timer.humanize = function (suffix = true) {
+  let t1 = performance.now()
+
+  return dayjs.duration(t1 - t0).humanize(suffix)
+}
+
+module.exports = timer