From c6c31545228ab9011992af456fe11424c97a5a0d Mon Sep 17 00:00:00 2001 From: Aleksandr Statciuk Date: Sun, 12 Dec 2021 07:10:18 +0300 Subject: [PATCH] Create scripts/core --- scripts/core/checker.js | 19 +++++++ scripts/core/db.js | 61 ++++++++++++++++++++ scripts/core/file.js | 67 ++++++++++++++++++++++ scripts/core/generator.js | 114 ++++++++++++++++++++++++++++++++++++++ scripts/core/index.js | 10 ++++ scripts/core/logger.js | 42 ++++++++++++++ scripts/core/markdown.js | 39 +++++++++++++ scripts/core/parser.js | 31 +++++++++++ scripts/core/playlist.js | 49 ++++++++++++++++ scripts/core/store.js | 56 +++++++++++++++++++ scripts/core/timer.js | 29 ++++++++++ 11 files changed, 517 insertions(+) create mode 100644 scripts/core/checker.js create mode 100644 scripts/core/db.js create mode 100644 scripts/core/file.js create mode 100644 scripts/core/generator.js create mode 100644 scripts/core/index.js create mode 100644 scripts/core/logger.js create mode 100644 scripts/core/markdown.js create mode 100644 scripts/core/parser.js create mode 100644 scripts/core/playlist.js create mode 100644 scripts/core/store.js create mode 100644 scripts/core/timer.js diff --git a/scripts/core/checker.js b/scripts/core/checker.js new file mode 100644 index 000000000..552110072 --- /dev/null +++ b/scripts/core/checker.js @@ -0,0 +1,19 @@ +const IPTVChecker = require('iptv-checker') + +const checker = {} + +checker.check = async function (item, config) { + const ic = new IPTVChecker(config) + const result = await ic.checkStream({ url: item.url, http: item.http }) + + return { + _id: item._id, + url: item.url, + http: item.http, + error: !result.status.ok ? result.status.reason : null, + streams: result.status.ok ? result.status.metadata.streams : [], + requests: result.status.ok ? result.status.metadata.requests : [] + } +} + +module.exports = checker diff --git a/scripts/core/db.js b/scripts/core/db.js new file mode 100644 index 000000000..27360a839 --- /dev/null +++ b/scripts/core/db.js @@ -0,0 +1,61 @@ +const Database = require('nedb-promises') +const file = require('./file') + +const DB_FILEPATH = process.env.DB_FILEPATH || './scripts/channels.db' + +const nedb = Database.create({ + filename: file.resolve(DB_FILEPATH), + autoload: true, + onload(err) { + if (err) console.error(err) + }, + compareStrings: (a, b) => { + a = a.replace(/\s/g, '_') + b = b.replace(/\s/g, '_') + + return a.localeCompare(b, undefined, { + sensitivity: 'accent', + numeric: true + }) + } +}) + +const db = {} + +db.removeIndex = function (field) { + return nedb.removeIndex(field) +} + +db.addIndex = function (options) { + return nedb.ensureIndex(options) +} + +db.compact = function () { + return nedb.persistence.compactDatafile() +} + +db.reset = function () { + return file.clear(DB_FILEPATH) +} + +db.count = function (query) { + return nedb.count(query) +} + +db.insert = function (doc) { + return nedb.insert(doc) +} + +db.update = function (query, update) { + return nedb.update(query, update) +} + +db.find = function (query) { + return nedb.find(query) +} + +db.remove = function (query, options) { + return nedb.remove(query, options) +} + +module.exports = db diff --git a/scripts/core/file.js b/scripts/core/file.js new file mode 100644 index 000000000..56da4db96 --- /dev/null +++ b/scripts/core/file.js @@ -0,0 +1,67 @@ +const path = require('path') +const glob = require('glob') +const fs = require('mz/fs') + +const file = {} + +file.list = function (pattern) { + return new Promise(resolve => { + glob(pattern, function (err, files) { + resolve(files) + }) + }) +} + +file.getFilename = function (filepath) { + return path.parse(filepath).name +} + +file.createDir = async function (dir) { + if (await file.exists(dir)) return + + return fs.mkdir(dir, { recursive: true }).catch(console.error) +} + +file.exists = function (filepath) { + return fs.exists(path.resolve(filepath)) +} + +file.read = function (filepath) { + return fs.readFile(path.resolve(filepath), { encoding: 'utf8' }).catch(console.error) +} + +file.append = function (filepath, data) { + return fs.appendFile(path.resolve(filepath), data).catch(console.error) +} + +file.create = function (filepath, data = '') { + filepath = path.resolve(filepath) + const dir = path.dirname(filepath) + + return file + .createDir(dir) + .then(() => fs.writeFile(filepath, data, { encoding: 'utf8', flag: 'w' })) + .catch(console.error) +} + +file.write = function (filepath, data = '') { + return fs.writeFile(path.resolve(filepath), data).catch(console.error) +} + +file.clear = function (filepath) { + return file.write(filepath, '') +} + +file.resolve = function (filepath) { + return path.resolve(filepath) +} + +file.dirname = function (filepath) { + return path.dirname(filepath) +} + +file.basename = function (filepath) { + return path.basename(filepath) +} + +module.exports = file diff --git a/scripts/core/generator.js b/scripts/core/generator.js new file mode 100644 index 000000000..046ac15fd --- /dev/null +++ b/scripts/core/generator.js @@ -0,0 +1,114 @@ +const { create: createPlaylist } = require('./playlist') +const store = require('./store') +const file = require('./file') +const logger = require('./logger') +const db = require('./db') +const _ = require('lodash') + +const generator = {} + +generator.generate = async function (filepath, query = {}, options = {}) { + options = { + ...{ + format: 'm3u', + saveEmpty: false, + includeNSFW: false, + includeGuides: true, + includeBroken: false, + onLoad: r => r, + uniqBy: item => item.id || _.uniqueId(), + sortBy: null + }, + ...options + } + + query['is_nsfw'] = options.includeNSFW ? { $in: [true, false] } : false + query['is_broken'] = options.includeBroken ? { $in: [true, false] } : false + + let items = await db + .find(query) + .sort({ name: 1, 'status.level': 1, 'resolution.height': -1, url: 1 }) + + items = _.uniqBy(items, 'url') + if (!options.saveEmpty && !items.length) return { filepath, query, options, count: 0 } + if (options.uniqBy) items = _.uniqBy(items, options.uniqBy) + + items = options.onLoad(items) + + if (options.sortBy) items = _.sortBy(items, options.sortBy) + + switch (options.format) { + case 'json': + await saveAsJSON(filepath, items, options) + break + case 'm3u': + default: + await saveAsM3U(filepath, items, options) + break + } + + return { filepath, query, options, count: items.length } +} + +async function saveAsM3U(filepath, items, options) { + const playlist = await createPlaylist(filepath) + + const header = {} + if (options.includeGuides) { + let guides = items.map(item => item.guides) + guides = _.uniq(_.flatten(guides)).sort().join(',') + + header['x-tvg-url'] = guides + } + + await playlist.header(header) + for (const item of items) { + const stream = store.create(item) + await playlist.link( + stream.get('url'), + stream.get('title'), + { + 'tvg-id': stream.get('tvg_id'), + 'tvg-country': stream.get('tvg_country'), + 'tvg-language': stream.get('tvg_language'), + 'tvg-logo': stream.get('tvg_logo'), + // 'tvg-url': stream.get('tvg_url') || undefined, + 'user-agent': stream.get('http.user-agent') || undefined, + 'group-title': stream.get('group_title') + }, + { + 'http-referrer': stream.get('http.referrer') || undefined, + 'http-user-agent': stream.get('http.user-agent') || undefined + } + ) + } +} + +async function saveAsJSON(filepath, items, options) { + const output = items.map(item => { + const stream = store.create(item) + const categories = stream.get('categories').map(c => ({ name: c.name, slug: c.slug })) + const countries = stream.get('countries').map(c => ({ name: c.name, code: c.code })) + + return { + name: stream.get('name'), + logo: stream.get('logo'), + url: stream.get('url'), + categories, + countries, + languages: stream.get('languages'), + tvg: { + id: stream.get('tvg_id'), + name: stream.get('name'), + url: stream.get('tvg_url') + } + } + }) + + await file.create(filepath, JSON.stringify(output)) +} + +generator.saveAsM3U = saveAsM3U +generator.saveAsJSON = saveAsJSON + +module.exports = generator diff --git a/scripts/core/index.js b/scripts/core/index.js new file mode 100644 index 000000000..948ff6c6d --- /dev/null +++ b/scripts/core/index.js @@ -0,0 +1,10 @@ +exports.db = require('./db') +exports.logger = require('./logger') +exports.file = require('./file') +exports.timer = require('./timer') +exports.parser = require('./parser') +exports.checker = require('./checker') +exports.generator = require('./generator') +exports.playlist = require('./playlist') +exports.store = require('./store') +exports.markdown = require('./markdown') diff --git a/scripts/core/logger.js b/scripts/core/logger.js new file mode 100644 index 000000000..a109a050b --- /dev/null +++ b/scripts/core/logger.js @@ -0,0 +1,42 @@ +const { createLogger, format, transports, addColors } = require('winston') +const { combine, timestamp, printf } = format + +const consoleFormat = ({ level, message, timestamp }) => { + if (typeof message === 'object') return JSON.stringify(message) + return message +} + +const config = { + levels: { + error: 0, + warn: 1, + info: 2, + failed: 3, + success: 4, + http: 5, + verbose: 6, + debug: 7, + silly: 8 + }, + colors: { + info: 'white', + success: 'green', + failed: 'red' + } +} + +const t = [ + new transports.Console({ + format: format.combine(format.printf(consoleFormat)) + }) +] + +const logger = createLogger({ + transports: t, + levels: config.levels, + level: 'verbose' +}) + +addColors(config.colors) + +module.exports = logger diff --git a/scripts/core/markdown.js b/scripts/core/markdown.js new file mode 100644 index 000000000..32dc1110e --- /dev/null +++ b/scripts/core/markdown.js @@ -0,0 +1,39 @@ +const markdownInclude = require('markdown-include') +const file = require('./file') + +const markdown = {} + +markdown.createTable = function (data, cols) { + let output = '\n' + + output += ' \n ' + for (let column of cols) { + output += `` + } + output += '\n \n' + + output += ' \n' + for (let item of data) { + output += ' ' + let i = 0 + for (let prop in item) { + const column = cols[i] + let nowrap = column.nowrap + let align = column.align + output += `` + i++ + } + output += '\n' + } + output += ' \n' + + output += '
${column.name}
${item[prop]}
' + + return output +} + +markdown.compile = function (filepath) { + markdownInclude.compileFiles(file.resolve(filepath)) +} + +module.exports = markdown diff --git a/scripts/core/parser.js b/scripts/core/parser.js new file mode 100644 index 000000000..009d274eb --- /dev/null +++ b/scripts/core/parser.js @@ -0,0 +1,31 @@ +const ipp = require('iptv-playlist-parser') +const logger = require('./logger') +const file = require('./file') + +const parser = {} + +parser.parsePlaylist = async function (filepath) { + const content = await file.read(filepath) + const playlist = ipp.parse(content) + + return playlist.items +} + +parser.parseLogs = async function (filepath) { + const content = await file.read(filepath) + if (!content) return [] + const lines = content.split('\n') + + return lines.map(line => (line ? JSON.parse(line) : null)).filter(l => l) +} + +parser.parseNumber = function (string) { + const parsed = parseInt(string) + if (isNaN(parsed)) { + logger.error('Not a number') + } + + return parsed +} + +module.exports = parser diff --git a/scripts/core/playlist.js b/scripts/core/playlist.js new file mode 100644 index 000000000..fb8077b27 --- /dev/null +++ b/scripts/core/playlist.js @@ -0,0 +1,49 @@ +const file = require('./file') + +const playlist = {} + +playlist.create = async function (filepath) { + playlist.filepath = filepath + const dir = file.dirname(filepath) + file.createDir(dir) + await file.create(filepath, '') + + return playlist +} + +playlist.header = async function (attrs) { + let header = `#EXTM3U` + for (const name in attrs) { + const value = attrs[name] + header += ` ${name}="${value}"` + } + header += `\n` + + await file.append(playlist.filepath, header) + + return playlist +} + +playlist.link = async function (url, title, attrs, vlcOpts) { + let link = `#EXTINF:-1` + for (const name in attrs) { + const value = attrs[name] + if (value !== undefined) { + link += ` ${name}="${value}"` + } + } + link += `,${title}\n` + for (const name in vlcOpts) { + const value = vlcOpts[name] + if (value !== undefined) { + link += `#EXTVLCOPT:${name}=${value}\n` + } + } + link += `${url}\n` + + await file.append(playlist.filepath, link) + + return playlist +} + +module.exports = playlist diff --git a/scripts/core/store.js b/scripts/core/store.js new file mode 100644 index 000000000..c41594403 --- /dev/null +++ b/scripts/core/store.js @@ -0,0 +1,56 @@ +const _ = require('lodash') +const logger = require('./logger') +const setters = require('../store/setters') +const getters = require('../store/getters') + +module.exports = { + create(state = {}) { + return { + state, + changed: false, + set: function (prop, value) { + const prevState = JSON.stringify(this.state) + + const setter = setters[prop] + if (typeof setter === 'function') { + try { + this.state[prop] = setter.bind()(value) + } catch (error) { + logger.error(`store/setters/${prop}.js: ${error.message}`) + } + } else if (typeof value === 'object') { + this.state[prop] = value[prop] + } else { + this.state[prop] = value + } + + const newState = JSON.stringify(this.state) + if (prevState !== newState) { + this.changed = true + } + + return this + }, + get: function (prop) { + const getter = getters[prop] + if (typeof getter === 'function') { + try { + return getter.bind(this.state)() + } catch (error) { + logger.error(`store/getters/${prop}.js: ${error.message}`) + } + } else { + return prop.split('.').reduce((o, i) => (o ? o[i] : undefined), this.state) + } + }, + has: function (prop) { + const value = this.get(prop) + + return !_.isEmpty(value) + }, + data: function () { + return this.state + } + } + } +} diff --git a/scripts/core/timer.js b/scripts/core/timer.js new file mode 100644 index 000000000..6e5f381d9 --- /dev/null +++ b/scripts/core/timer.js @@ -0,0 +1,29 @@ +const { performance } = require('perf_hooks') +const dayjs = require('dayjs') +const duration = require('dayjs/plugin/duration') +const relativeTime = require('dayjs/plugin/relativeTime') + +dayjs.extend(relativeTime) +dayjs.extend(duration) + +const timer = {} + +let t0 = 0 + +timer.start = function () { + t0 = performance.now() +} + +timer.format = function (f) { + let t1 = performance.now() + + return dayjs.duration(t1 - t0).format(f) +} + +timer.humanize = function (suffix = true) { + let t1 = performance.now() + + return dayjs.duration(t1 - t0).humanize(suffix) +} + +module.exports = timer