Update scripts

pull/28852/head
freearhey 1 month ago
parent 5887997bb3
commit adec9dd131

@ -0,0 +1,151 @@
import { Collection, Dictionary } from '@freearhey/core'
import { DATA_DIR } from './constants'
import cliProgress from 'cli-progress'
import * as sdk from '@iptv-org/sdk'
const data = {
categoriesKeyById: new Dictionary<sdk.Models.Category>(),
countriesKeyByCode: new Dictionary<sdk.Models.Country>(),
subdivisionsKeyByCode: new Dictionary<sdk.Models.Subdivision>(),
citiesKeyByCode: new Dictionary<sdk.Models.City>(),
regionsKeyByCode: new Dictionary<sdk.Models.Region>(),
languagesKeyByCode: new Dictionary<sdk.Models.Language>(),
channelsKeyById: new Dictionary<sdk.Models.Channel>(),
feedsKeyByStreamId: new Dictionary<sdk.Models.Feed>(),
feedsGroupedByChannel: new Dictionary<sdk.Models.Feed[]>(),
blocklistRecordsGroupedByChannel: new Dictionary<sdk.Models.BlocklistRecord[]>(),
categories: new Collection<sdk.Models.Category>(),
countries: new Collection<sdk.Models.Country>(),
subdivisions: new Collection<sdk.Models.Subdivision>(),
cities: new Collection<sdk.Models.City>(),
regions: new Collection<sdk.Models.Region>()
}
let searchIndex
async function loadData() {
const dataManager = new sdk.DataManager({ dataDir: DATA_DIR })
await dataManager.loadFromDisk()
dataManager.processData()
const {
channels,
feeds,
categories,
languages,
countries,
subdivisions,
cities,
regions,
blocklist
} = dataManager.getProcessedData()
searchIndex = sdk.SearchEngine.createIndex<sdk.Models.Channel>(channels)
data.categoriesKeyById = categories.keyBy((category: sdk.Models.Category) => category.id)
data.countriesKeyByCode = countries.keyBy((country: sdk.Models.Country) => country.code)
data.subdivisionsKeyByCode = subdivisions.keyBy(
(subdivision: sdk.Models.Subdivision) => subdivision.code
)
data.citiesKeyByCode = cities.keyBy((city: sdk.Models.City) => city.code)
data.regionsKeyByCode = regions.keyBy((region: sdk.Models.Region) => region.code)
data.languagesKeyByCode = languages.keyBy((language: sdk.Models.Language) => language.code)
data.channelsKeyById = channels.keyBy((channel: sdk.Models.Channel) => channel.id)
data.feedsKeyByStreamId = feeds.keyBy((feed: sdk.Models.Feed) => feed.getStreamId())
data.feedsGroupedByChannel = feeds.groupBy((feed: sdk.Models.Feed) => feed.channel)
data.blocklistRecordsGroupedByChannel = blocklist.groupBy(
(blocklistRecord: sdk.Models.BlocklistRecord) => blocklistRecord.channel
)
data.categories = categories
data.countries = countries
data.subdivisions = subdivisions
data.cities = cities
data.regions = regions
}
async function downloadData() {
function formatBytes(bytes: number) {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]
}
const files = [
'blocklist',
'categories',
'channels',
'cities',
'countries',
'feeds',
'guides',
'languages',
'logos',
'regions',
'streams',
'subdivisions',
'timezones'
]
const multiBar = new cliProgress.MultiBar({
stopOnComplete: true,
hideCursor: true,
forceRedraw: true,
barsize: 36,
format(options, params, payload) {
const filename = payload.filename.padEnd(18, ' ')
const barsize = options.barsize || 40
const percent = (params.progress * 100).toFixed(2)
const speed = payload.speed ? formatBytes(payload.speed) + '/s' : 'N/A'
const total = formatBytes(params.total)
const completeSize = Math.round(params.progress * barsize)
const incompleteSize = barsize - completeSize
const bar =
options.barCompleteString && options.barIncompleteString
? options.barCompleteString.substr(0, completeSize) +
options.barGlue +
options.barIncompleteString.substr(0, incompleteSize)
: '-'.repeat(barsize)
return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
}
})
const dataManager = new sdk.DataManager({ dataDir: DATA_DIR })
const requests: Promise<unknown>[] = []
for (const basename of files) {
const filename = `${basename}.json`
const progressBar = multiBar.create(0, 0, { filename })
const request = dataManager.downloadFileToDisk(basename, {
onDownloadProgress({ total, loaded, rate }) {
if (total) progressBar.setTotal(total)
progressBar.update(loaded, { speed: rate })
}
})
requests.push(request)
}
await Promise.allSettled(requests).catch(console.error)
}
function searchChannels(query: string): Collection<sdk.Models.Channel> {
if (!searchIndex) return new Collection<sdk.Models.Channel>()
const results = searchIndex.search(query)
const channels = new Collection<sdk.Models.Channel>()
new Collection<sdk.Types.ChannelSearchableData>(results).forEach(
(item: sdk.Types.ChannelSearchableData) => {
const channel = data.channelsKeyById.get(item.id)
if (channel) channels.add(channel)
}
)
return channels
}
export { data, loadData, downloadData, searchChannels }

@ -1,39 +1,31 @@
import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
import type { DataProcessorData } from '../../types/dataProcessor'
import { API_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
import type { DataLoaderData } from '../../types/dataLoader'
import { Logger, Storage } from '@freearhey/core'
import { API_DIR, STREAMS_DIR } from '../../constants'
import { Storage } from '@freearhey/storage-js'
import { PlaylistParser } from '../../core'
import { Logger } from '@freearhey/core'
import { Stream } from '../../models'
import { loadData } from '../../api'
async function main() {
const logger = new Logger()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const dataLoader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await dataLoader.load()
const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData =
processor.process(data)
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
channelsKeyById,
logosGroupedByStreamId,
feedsGroupedByChannelId
storage: streamsStorage
})
const files = await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
streams = streams
.orderBy((stream: Stream) => stream.getId())
.map((stream: Stream) => stream.toJSON())
logger.info(`found ${streams.count()} streams`)
const parsed = await parser.parse(files)
const _streams = parsed
.sortBy((stream: Stream) => stream.getId())
.map((stream: Stream) => stream.toObject())
logger.info(`found ${_streams.count()} streams`)
logger.info('saving to .api/streams.json...')
const apiStorage = new Storage(API_DIR)
await apiStorage.save('streams.json', streams.toJSON())
await apiStorage.save('streams.json', _streams.toJSON())
}
main()

@ -1,26 +1,7 @@
import { DATA_DIR } from '../../constants'
import { Storage } from '@freearhey/core'
import { DataLoader } from '../../core'
import { downloadData } from '../../api'
async function main() {
const storage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage })
await Promise.all([
loader.download('blocklist.json'),
loader.download('categories.json'),
loader.download('channels.json'),
loader.download('countries.json'),
loader.download('languages.json'),
loader.download('regions.json'),
loader.download('subdivisions.json'),
loader.download('feeds.json'),
loader.download('logos.json'),
loader.download('timezones.json'),
loader.download('guides.json'),
loader.download('streams.json'),
loader.download('cities.json')
])
await downloadData()
}
main()

@ -1,17 +1,16 @@
import { Storage, Collection, Logger, Dictionary } from '@freearhey/core'
import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
import type { ChannelSearchableData } from '../../types/channel'
import { Channel, Feed, Playlist, Stream } from '../../models'
import { DataProcessorData } from '../../types/dataProcessor'
import { DataLoaderData } from '../../types/dataLoader'
import { loadData, data, searchChannels } from '../../api'
import { Collection, Logger } from '@freearhey/core'
import { select, input } from '@inquirer/prompts'
import { DATA_DIR } from '../../constants'
import { Playlist, Stream } from '../../models'
import { Storage } from '@freearhey/storage-js'
import { PlaylistParser } from '../../core'
import nodeCleanup from 'node-cleanup'
import sjs from '@freearhey/search-js'
import * as sdk from '@iptv-org/sdk'
import { truncate } from '../../utils'
import { Command } from 'commander'
import readline from 'readline'
type ChoiceValue = { type: string; value?: Feed | Channel }
type ChoiceValue = { type: string; value?: sdk.Models.Feed | sdk.Models.Channel }
type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }
if (process.platform === 'win32') {
@ -32,7 +31,7 @@ program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(proc
const filepath = program.args[0]
const logger = new Logger()
const storage = new Storage()
let parsedStreams = new Collection()
let parsedStreams = new Collection<Stream>()
main(filepath)
nodeCleanup(() => {
@ -45,42 +44,24 @@ export default async function main(filepath: string) {
}
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const {
channels,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
}: DataProcessorData = processor.process(data)
await loadData()
logger.info('loading streams...')
const parser = new PlaylistParser({
storage,
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById
storage
})
parsedStreams = await parser.parseFile(filepath)
const streamsWithoutId = parsedStreams.filter((stream: Stream) => !stream.id)
const streamsWithoutId = parsedStreams.filter((stream: Stream) => !stream.tvgId)
logger.info(
`found ${parsedStreams.count()} streams (including ${streamsWithoutId.count()} without ID)`
)
logger.info('creating search index...')
const items = channels.map((channel: Channel) => channel.getSearchable()).all()
const searchIndex = sjs.createIndex(items, {
searchable: ['name', 'altNames', 'guideNames', 'streamTitles', 'feedFullNames']
})
logger.info('starting...\n')
for (const stream of streamsWithoutId.all()) {
try {
stream.id = await selectChannel(stream, searchIndex, feedsGroupedByChannelId, channelsKeyById)
stream.tvgId = await selectChannel(stream)
} catch (err) {
logger.info(err.message)
break
@ -88,28 +69,20 @@ export default async function main(filepath: string) {
}
streamsWithoutId.forEach((stream: Stream) => {
if (stream.id === '-') {
stream.id = ''
if (stream.channel === '-') {
stream.channel = ''
}
})
}
async function selectChannel(
stream: Stream,
searchIndex,
feedsGroupedByChannelId: Dictionary,
channelsKeyById: Dictionary
): Promise<string> {
const query = escapeRegex(stream.getTitle())
const similarChannels = searchIndex
.search(query)
.map((item: ChannelSearchableData) => channelsKeyById.get(item.id))
const url = stream.url.length > 50 ? stream.url.slice(0, 50) + '...' : stream.url
async function selectChannel(stream: Stream): Promise<string> {
const query = escapeRegex(stream.title)
const similarChannels = searchChannels(query)
const url = truncate(stream.url, 50)
const selected: ChoiceValue = await select({
message: `Select channel ID for "${stream.title}" (${url}):`,
choices: getChannelChoises(new Collection(similarChannels)),
choices: getChannelChoises(similarChannels),
pageSize: 10
})
@ -119,14 +92,14 @@ async function selectChannel(
case 'type': {
const typedChannelId = await input({ message: ' Channel ID:' })
if (!typedChannelId) return ''
const selectedFeedId = await selectFeed(typedChannelId, feedsGroupedByChannelId)
const selectedFeedId = await selectFeed(typedChannelId)
if (selectedFeedId === '-') return typedChannelId
return [typedChannelId, selectedFeedId].join('@')
}
case 'channel': {
const selectedChannel = selected.value
if (!selectedChannel) return ''
const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
const selectedFeedId = await selectFeed(selectedChannel.id)
if (selectedFeedId === '-') return selectedChannel.id
return [selectedChannel.id, selectedFeedId].join('@')
}
@ -135,8 +108,8 @@ async function selectChannel(
return ''
}
async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise<string> {
const channelFeeds = new Collection(feedsGroupedByChannelId.get(channelId))
async function selectFeed(channelId: string): Promise<string> {
const channelFeeds = new Collection(data.feedsGroupedByChannel.get(channelId))
const choices = getFeedChoises(channelFeeds)
const selected: ChoiceValue = await select({
@ -159,11 +132,11 @@ async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary
return ''
}
function getChannelChoises(channels: Collection): Choice[] {
function getChannelChoises(channels: Collection<sdk.Models.Channel>): Choice[] {
const choises: Choice[] = []
channels.forEach((channel: Channel) => {
const names = new Collection([channel.name, ...channel.altNames.all()]).uniq().join(', ')
channels.forEach((channel: sdk.Models.Channel) => {
const names = new Collection([channel.name, ...channel.alt_names]).uniq().join(', ')
choises.push({
value: {
@ -181,19 +154,19 @@ function getChannelChoises(channels: Collection): Choice[] {
return choises
}
function getFeedChoises(feeds: Collection): Choice[] {
function getFeedChoises(feeds: Collection<sdk.Models.Feed>): Choice[] {
const choises: Choice[] = []
feeds.forEach((feed: Feed) => {
feeds.forEach((feed: sdk.Models.Feed) => {
let name = `${feed.id} (${feed.name})`
if (feed.isMain) name += ' [main]'
if (feed.is_main) name += ' [main]'
choises.push({
value: {
type: 'feed',
value: feed
},
default: feed.isMain,
default: feed.is_main,
name,
short: feed.id
})

@ -1,10 +1,10 @@
import { Logger, Storage } from '@freearhey/core'
import { STREAMS_DIR, DATA_DIR } from '../../constants'
import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
import { Collection, Logger } from '@freearhey/core'
import { Stream, Playlist } from '../../models'
import { Storage } from '@freearhey/storage-js'
import { STREAMS_DIR } from '../../constants'
import { PlaylistParser } from '../../core'
import { loadData } from '../../api'
import { program } from 'commander'
import { DataLoaderData } from '../../types/dataLoader'
import { DataProcessorData } from '../../types/dataProcessor'
import path from 'node:path'
program.argument('[filepath...]', 'Path to file to format').parse(process.argv)
@ -13,20 +13,12 @@ async function main() {
const logger = new Logger()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData =
processor.process(data)
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
storage: streamsStorage
})
let files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
files = files.map((filepath: string) => path.basename(filepath))
@ -45,19 +37,33 @@ async function main() {
logger.info('removing wrong id...')
streams = streams.map((stream: Stream) => {
if (!stream.channel || channelsKeyById.missing(stream.channel.id)) {
stream.id = ''
const channel = stream.getChannel()
if (channel) return stream
stream.tvgId = ''
stream.channel = ''
stream.feed = ''
return stream
})
logger.info('adding the missing feed id...')
streams = streams.map((stream: Stream) => {
const feed = stream.getFeed()
if (feed) {
stream.feed = feed.id
stream.tvgId = stream.getId()
}
return stream
})
logger.info('sorting links...')
streams = streams.orderBy(
streams = streams.sortBy(
[
(stream: Stream) => stream.title,
(stream: Stream) => stream.getVerticalResolution(),
(stream: Stream) => stream.getLabel(),
(stream: Stream) => stream.label,
(stream: Stream) => stream.url
],
['asc', 'desc', 'asc', 'asc']
@ -66,9 +72,9 @@ async function main() {
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath) || []
const streams = new Collection(groupedStreams.get(filepath))
if (!streams.length) return
if (streams.isEmpty()) return
const playlist = new Playlist(streams, { public: false })
await streamsStorage.save(filepath, playlist.toString())

@ -1,10 +1,10 @@
import { PlaylistParser, DataProcessor, DataLoader } from '../../core'
import type { DataProcessorData } from '../../types/dataProcessor'
import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
import type { DataLoaderData } from '../../types/dataLoader'
import { Logger, Storage, File } from '@freearhey/core'
import { Stream } from '../../models'
import { LOGS_DIR, STREAMS_DIR } from '../../constants'
import { Storage, File } from '@freearhey/storage-js'
import { PlaylistParser } from '../../core'
import { loadData, data } from '../../api'
import { Logger } from '@freearhey/core'
import uniqueId from 'lodash.uniqueid'
import { Stream } from '../../models'
import {
IndexCategoryGenerator,
IndexLanguageGenerator,
@ -25,28 +25,12 @@ async function main() {
const logFile = new File('generators.log')
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const {
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById,
subdivisions,
categories,
countries,
regions,
cities
}: DataProcessorData = processor.process(data)
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById
storage: streamsStorage
})
const files = await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
@ -57,20 +41,20 @@ async function main() {
await new RawGenerator({ streams, logFile }).generate()
logger.info('filtering streams...')
streams = streams.uniqBy((stream: Stream) =>
stream.hasId() ? stream.getChannelId() + stream.getFeedId() : uniqueId()
)
streams = streams.uniqBy((stream: Stream) => stream.getId() || uniqueId())
logger.info('sorting streams...')
streams = streams.orderBy(
streams = streams.sortBy(
[
(stream: Stream) => stream.getId(),
(stream: Stream) => stream.getVerticalResolution(),
(stream: Stream) => stream.getLabel()
(stream: Stream) => stream.label
],
['asc', 'asc', 'desc']
)
const { categories, countries, subdivisions, cities, regions } = data
logger.info('generating categories/...')
await new CategoriesGenerator({ categories, streams, logFile }).generate()

@ -1,15 +1,16 @@
import { Logger, Storage, Collection } from '@freearhey/core'
import { ROOT_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
import { PlaylistParser, StreamTester, CliTable, DataProcessor, DataLoader } from '../../core'
import { PlaylistParser, StreamTester, CliTable } from '../../core'
import type { TestResult } from '../../core/streamTester'
import { Stream } from '../../models'
import { ROOT_DIR, STREAMS_DIR } from '../../constants'
import { Logger, Collection } from '@freearhey/core'
import { program, OptionValues } from 'commander'
import { eachLimit } from 'async-es'
import { Storage } from '@freearhey/storage-js'
import { Stream } from '../../models'
import { loadData } from '../../api'
import { eachLimit } from 'async'
import dns from 'node:dns'
import chalk from 'chalk'
import os from 'node:os'
import dns from 'node:dns'
import type { DataLoaderData } from '../../types/dataLoader'
import type { DataProcessorData } from '../../types/dataProcessor'
import { truncate } from '../../utils'
const LIVE_UPDATE_INTERVAL = 5000
const LIVE_UPDATE_MAX_STREAMS = 100
@ -18,7 +19,7 @@ let errors = 0
let warnings = 0
const results: { [key: string]: string } = {}
let interval: string | number | NodeJS.Timeout | undefined
let streams = new Collection()
let streams = new Collection<Stream>()
let isLiveUpdateEnabled = true
program
@ -50,20 +51,12 @@ async function main() {
}
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData =
processor.process(data)
await loadData()
logger.info('loading streams...')
const rootStorage = new Storage(ROOT_DIR)
const parser = new PlaylistParser({
storage: rootStorage,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
storage: rootStorage
})
const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`)
streams = await parser.parse(files)
@ -79,7 +72,7 @@ async function main() {
}, LIVE_UPDATE_INTERVAL)
}
await eachLimit(
eachLimit(
streams.all(),
options.parallel,
async (stream: Stream) => {
@ -96,7 +89,7 @@ async function main() {
main()
async function runTest(stream: Stream) {
const key = stream.filepath + stream.getId() + stream.url
const key = stream.getUniqKey()
results[key] = chalk.white('LOADING...')
const result: TestResult = await tester.test(stream)
@ -121,7 +114,7 @@ function drawTable() {
const streamsGrouped = streams.groupBy((stream: Stream) => stream.filepath)
for (const filepath of streamsGrouped.keys()) {
const streams: Stream[] = streamsGrouped.get(filepath)
const streams: Stream[] = streamsGrouped.get(filepath) || []
const table = new CliTable({
columns: [
@ -132,12 +125,14 @@ function drawTable() {
]
})
streams.forEach((stream: Stream, index: number) => {
const status = results[stream.filepath + stream.getId() + stream.url] || chalk.gray('PENDING')
const key = stream.getUniqKey()
const status = results[key] || chalk.gray('PENDING')
const tvgId = stream.getTvgId()
const row = {
'': index,
'tvg-id': stream.getId().length > 25 ? stream.getId().slice(0, 22) + '...' : stream.getId(),
url: stream.url.length > 100 ? stream.url.slice(0, 97) + '...' : stream.url,
'tvg-id': truncate(tvgId, 25),
url: truncate(stream.url, 100),
status
}
table.append(row)
@ -149,7 +144,7 @@ function drawTable() {
}
}
function onFinish(error: any) {
function onFinish(error: Error) {
clearInterval(interval)
if (error) {

@ -1,9 +1,10 @@
import { DataLoader, DataProcessor, IssueLoader, PlaylistParser } from '../../core'
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import type { DataProcessorData } from '../../types/dataProcessor'
import { Stream, Playlist, Channel, Issue } from '../../models'
import type { DataLoaderData } from '../../types/dataLoader'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { IssueLoader, PlaylistParser } from '../../core'
import { Playlist, Issue, Stream } from '../../models'
import { loadData, data as apiData } from '../../api'
import { Logger, Collection } from '@freearhey/core'
import { Storage } from '@freearhey/storage-js'
import { STREAMS_DIR } from '../../constants'
import * as sdk from '@iptv-org/sdk'
import { isURI } from '../../utils'
const processedIssues = new Collection()
@ -16,20 +17,12 @@ async function main() {
const issues = await issueLoader.load()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const dataLoader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await dataLoader.load()
const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData =
processor.process(data)
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById
storage: streamsStorage
})
const files = await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
@ -40,23 +33,19 @@ async function main() {
logger.info('edit stream description...')
await editStreams({
streams,
issues,
channelsKeyById,
feedsGroupedByChannelId
issues
})
logger.info('add new streams...')
await addStreams({
streams,
issues,
channelsKeyById,
feedsGroupedByChannelId
issues
})
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of groupedStreams.keys()) {
let streams = groupedStreams.get(filepath) || []
let streams = new Collection(groupedStreams.get(filepath))
streams = streams.filter((stream: Stream) => stream.removed === false)
const playlist = new Playlist(streams, { public: false })
@ -69,10 +58,17 @@ async function main() {
main()
async function removeStreams({ streams, issues }: { streams: Collection; issues: Collection }) {
async function removeStreams({
streams,
issues
}: {
streams: Collection<Stream>
issues: Collection<Issue>
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved')
)
requests.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('streamUrl')) return
@ -97,14 +93,10 @@ async function removeStreams({ streams, issues }: { streams: Collection; issues:
async function editStreams({
streams,
issues,
channelsKeyById,
feedsGroupedByChannelId
issues
}: {
streams: Collection
issues: Collection
channelsKeyById: Dictionary
feedsGroupedByChannelId: Dictionary
streams: Collection<Stream>
issues: Collection<Issue>
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved')
@ -123,17 +115,12 @@ async function editStreams({
const [channelId, feedId] = streamId.split('@')
if (channelId) {
stream
.setChannelId(channelId)
.setFeedId(feedId)
.withChannel(channelsKeyById)
.withFeed(feedsGroupedByChannelId)
.updateId()
.updateTitle()
.updateFilepath()
stream.channel = channelId
stream.feed = feedId
stream.updateTvgId().updateTitle().updateFilepath()
}
stream.update(data)
stream.updateWithIssue(data)
processedIssues.add(issue.number)
})
@ -141,14 +128,10 @@ async function editStreams({
async function addStreams({
streams,
issues,
channelsKeyById,
feedsGroupedByChannelId
issues
}: {
streams: Collection
issues: Collection
channelsKeyById: Dictionary
feedsGroupedByChannelId: Dictionary
streams: Collection<Stream>
issues: Collection<Issue>
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:add') && issue.labels.includes('approved')
@ -163,30 +146,27 @@ async function addStreams({
const streamId = data.getString('streamId') || ''
const [channelId, feedId] = streamId.split('@')
const channel: Channel = channelsKeyById.get(channelId)
const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId)
if (!channel) return
const label = data.getString('label') || null
const label = data.getString('label') || ''
const quality = data.getString('quality') || null
const httpUserAgent = data.getString('httpUserAgent') || null
const httpReferrer = data.getString('httpReferrer') || null
const directives = data.getArray('directives') || []
const stream = new Stream({
channelId,
feedId,
channel: channelId,
feed: feedId,
title: channel.name,
url: streamUrl,
userAgent: httpUserAgent,
user_agent: httpUserAgent,
referrer: httpReferrer,
directives,
quality,
label
quality
})
.withChannel(channelsKeyById)
.withFeed(feedsGroupedByChannelId)
.updateTitle()
.updateFilepath()
stream.label = label
stream.setDirectives(directives).updateTitle().updateFilepath()
streams.add(stream)
processedIssues.add(issue.number)

@ -1,9 +1,10 @@
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
import { DataProcessorData } from '../../types/dataProcessor'
import { DATA_DIR, ROOT_DIR } from '../../constants'
import { DataLoaderData } from '../../types/dataLoader'
import { BlocklistRecord, Stream } from '../../models'
import { Logger, Collection, Dictionary } from '@freearhey/core'
import { Storage } from '@freearhey/storage-js'
import { PlaylistParser } from '../../core'
import { data, loadData } from '../../api'
import { ROOT_DIR } from '../../constants'
import { Stream } from '../../models'
import * as sdk from '@iptv-org/sdk'
import { program } from 'commander'
import chalk from 'chalk'
@ -19,24 +20,12 @@ async function main() {
const logger = new Logger()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const {
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId,
blocklistRecordsGroupedByChannelId
}: DataProcessorData = processor.process(data)
await loadData()
logger.info('loading streams...')
const rootStorage = new Storage(ROOT_DIR)
const parser = new PlaylistParser({
storage: rootStorage,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
storage: rootStorage
})
const files = program.args.length ? program.args : await rootStorage.list('streams/**/*.m3u')
const streams = await parser.parse(files)
@ -49,16 +38,16 @@ async function main() {
const streams = streamsGroupedByFilepath.get(filepath)
if (!streams) continue
const log = new Collection()
const buffer = new Dictionary()
const log = new Collection<LogItem>()
const buffer = new Dictionary<boolean>()
streams.forEach((stream: Stream) => {
if (stream.channelId) {
const channel = channelsKeyById.get(stream.channelId)
if (stream.channel) {
const channel = data.channelsKeyById.get(stream.channel)
if (!channel) {
log.add({
type: 'warning',
line: stream.getLine(),
message: `"${stream.id}" is not in the database`
message: `"${stream.tvgId}" is not in the database`
})
}
}
@ -74,28 +63,30 @@ async function main() {
buffer.set(stream.url, true)
}
const blocklistRecords = stream.channel
? new Collection(blocklistRecordsGroupedByChannelId.get(stream.channel.id))
: new Collection()
if (stream.channel) {
const blocklistRecords = new Collection(
data.blocklistRecordsGroupedByChannel.get(stream.channel)
)
blocklistRecords.forEach((blocklistRecord: BlocklistRecord) => {
blocklistRecords.forEach((blocklistRecord: sdk.Models.BlocklistRecord) => {
if (blocklistRecord.reason === 'dmca') {
log.add({
type: 'error',
line: stream.getLine(),
message: `"${blocklistRecord.channelId}" is on the blocklist due to claims of copyright holders (${blocklistRecord.ref})`
message: `"${blocklistRecord.channel}" is on the blocklist due to claims of copyright holders (${blocklistRecord.ref})`
})
} else if (blocklistRecord.reason === 'nsfw') {
log.add({
type: 'error',
line: stream.getLine(),
message: `"${blocklistRecord.channelId}" is on the blocklist due to NSFW content (${blocklistRecord.ref})`
message: `"${blocklistRecord.channel}" is on the blocklist due to NSFW content (${blocklistRecord.ref})`
})
}
})
}
})
if (log.notEmpty()) {
if (log.isNotEmpty()) {
console.log(`\n${chalk.underline(filepath)}`)
log.forEach((logItem: LogItem) => {

@ -1,41 +1,23 @@
import { CategoriesTable, CountriesTable, LanguagesTable, RegionsTable } from '../../tables'
import { DataLoader, DataProcessor, Markdown } from '../../core'
import { DataProcessorData } from '../../types/dataProcessor'
import { DataLoaderData } from '../../types/dataLoader'
import { README_DIR, DATA_DIR, ROOT_DIR } from '../../constants'
import { Logger, Storage } from '@freearhey/core'
import { README_DIR, ROOT_DIR } from '../../constants'
import { Logger } from '@freearhey/core'
import { Markdown } from '../../core'
import { loadData } from '../../api'
async function main() {
const logger = new Logger()
const dataStorage = new Storage(DATA_DIR)
const processor = new DataProcessor()
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const {
subdivisionsKeyByCode,
languagesKeyByCode,
countriesKeyByCode,
categoriesKeyById,
subdivisions,
countries,
regions,
cities
}: DataProcessorData = processor.process(data)
logger.info('loading data from api...')
await loadData()
logger.info('creating category table...')
await new CategoriesTable({ categoriesKeyById }).make()
await new CategoriesTable().create()
logger.info('creating language table...')
await new LanguagesTable({ languagesKeyByCode }).make()
await new LanguagesTable().create()
logger.info('creating countires table...')
await new CountriesTable({
countriesKeyByCode,
subdivisionsKeyByCode,
subdivisions,
countries,
cities
}).make()
await new CountriesTable().create()
logger.info('creating region table...')
await new RegionsTable({ regions }).make()
await new RegionsTable().create()
logger.info('updating playlists.md...')
const playlists = new Markdown({

@ -1,10 +1,10 @@
import { DataLoader, DataProcessor, IssueLoader, PlaylistParser } from '../../core'
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import { DataProcessorData } from '../../types/dataProcessor'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { DataLoaderData } from '../../types/dataLoader'
import { Logger, Collection, Dictionary } from '@freearhey/core'
import { IssueLoader, PlaylistParser } from '../../core'
import { Storage } from '@freearhey/storage-js'
import { isURI, truncate } from '../../utils'
import { STREAMS_DIR } from '../../constants'
import { Issue, Stream } from '../../models'
import { isURI } from '../../utils'
import { data, loadData } from '../../api'
async function main() {
const logger = new Logger()
@ -15,29 +15,17 @@ async function main() {
const issues = await issueLoader.load()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const dataLoader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await dataLoader.load()
const {
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId,
blocklistRecordsGroupedByChannelId
}: DataProcessorData = processor.process(data)
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
storage: streamsStorage
})
const files = await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url)
const streamsGroupedByChannelId = streams.groupBy((stream: Stream) => stream.channelId)
const streamsGroupedByChannel = streams.groupBy((stream: Stream) => stream.channel)
const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
logger.info('checking streams:remove requests...')
@ -95,8 +83,8 @@ async function main() {
if (!channelId) result.status = 'missing_id'
else if (!streamUrl) result.status = 'missing_link'
else if (!isURI(streamUrl)) result.status = 'invalid_link'
else if (blocklistRecordsGroupedByChannelId.has(channelId)) result.status = 'blocked'
else if (channelsKeyById.missing(channelId)) result.status = 'wrong_id'
else if (data.blocklistRecordsGroupedByChannel.has(channelId)) result.status = 'blocked'
else if (data.channelsKeyById.missing(channelId)) result.status = 'wrong_id'
else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist'
else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate'
else result.status = 'pending'
@ -125,7 +113,7 @@ async function main() {
if (!streamUrl) result.status = 'missing_link'
else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link'
else if (channelId && channelsKeyById.missing(channelId)) result.status = 'invalid_id'
else if (channelId && data.channelsKeyById.missing(channelId)) result.status = 'invalid_id'
report.add(result)
})
@ -148,14 +136,14 @@ async function main() {
}
if (!channelId) result.status = 'missing_id'
else if (channelsKeyById.missing(channelId)) result.status = 'invalid_id'
else if (data.channelsKeyById.missing(channelId)) result.status = 'invalid_id'
else if (channelSearchRequestsBuffer.has(streamId)) result.status = 'duplicate'
else if (blocklistRecordsGroupedByChannelId.has(channelId)) result.status = 'blocked'
else if (data.blocklistRecordsGroupedByChannel.has(channelId)) result.status = 'blocked'
else if (streamsGroupedById.has(streamId)) result.status = 'fulfilled'
else if (!feedId && streamsGroupedByChannelId.has(channelId)) result.status = 'fulfilled'
else if (!feedId && streamsGroupedByChannel.has(channelId)) result.status = 'fulfilled'
else {
const channelData = channelsKeyById.get(channelId)
if (channelData && channelData.isClosed) result.status = 'closed'
const channelData = data.channelsKeyById.get(channelId)
if (channelData && channelData.isClosed()) result.status = 'closed'
}
channelSearchRequestsBuffer.set(streamId, true)
@ -163,16 +151,9 @@ async function main() {
report.add(result)
})
report = report.orderBy(item => item.issueNumber).filter(item => item.status !== 'pending')
report = report.sortBy(item => item.issueNumber).filter(item => item.status !== 'pending')
console.table(report.all())
}
main()
function truncate(string: string, limit: number = 100) {
if (!string) return string
if (string.length < limit) return string
return string.slice(0, limit) + '...'
}

@ -1,16 +0,0 @@
import axios, { AxiosInstance, AxiosResponse, AxiosRequestConfig } from 'axios'
export class ApiClient {
instance: AxiosInstance
constructor() {
this.instance = axios.create({
baseURL: 'https://iptv-org.github.io/api',
responseType: 'stream'
})
}
get(url: string, options: AxiosRequestConfig): Promise<AxiosResponse> {
return this.instance.get(url, options)
}
}

@ -1,5 +1,5 @@
import { Table } from 'console-table-printer'
import { ComplexOptions } from 'console-table-printer/dist/src/models/external-table'
import { Table } from 'console-table-printer'
export class CliTable {
table: Table

@ -1,113 +0,0 @@
import { ApiClient } from './apiClient'
import { Storage } from '@freearhey/core'
import cliProgress, { MultiBar } from 'cli-progress'
import type { DataLoaderProps, DataLoaderData } from '../types/dataLoader'
const formatBytes = (bytes: number) => {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]
}
export class DataLoader {
client: ApiClient
storage: Storage
progressBar: MultiBar
constructor(props: DataLoaderProps) {
this.client = new ApiClient()
this.storage = props.storage
this.progressBar = new cliProgress.MultiBar({
stopOnComplete: true,
hideCursor: true,
forceRedraw: true,
barsize: 36,
format(options, params, payload) {
const filename = payload.filename.padEnd(18, ' ')
const barsize = options.barsize || 40
const percent = (params.progress * 100).toFixed(2)
const speed = payload.speed ? formatBytes(payload.speed) + '/s' : 'N/A'
const total = formatBytes(params.total)
const completeSize = Math.round(params.progress * barsize)
const incompleteSize = barsize - completeSize
const bar =
options.barCompleteString && options.barIncompleteString
? options.barCompleteString.substr(0, completeSize) +
options.barGlue +
options.barIncompleteString.substr(0, incompleteSize)
: '-'.repeat(barsize)
return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
}
})
}
async load(): Promise<DataLoaderData> {
const [
countries,
regions,
subdivisions,
languages,
categories,
blocklist,
channels,
feeds,
logos,
timezones,
guides,
streams,
cities
] = await Promise.all([
this.storage.json('countries.json'),
this.storage.json('regions.json'),
this.storage.json('subdivisions.json'),
this.storage.json('languages.json'),
this.storage.json('categories.json'),
this.storage.json('blocklist.json'),
this.storage.json('channels.json'),
this.storage.json('feeds.json'),
this.storage.json('logos.json'),
this.storage.json('timezones.json'),
this.storage.json('guides.json'),
this.storage.json('streams.json'),
this.storage.json('cities.json')
])
return {
countries,
regions,
subdivisions,
languages,
categories,
blocklist,
channels,
feeds,
logos,
timezones,
guides,
streams,
cities
}
}
async download(filename: string) {
if (!this.storage || !this.progressBar) return
const stream = await this.storage.createStream(filename)
const progressBar = this.progressBar.create(0, 0, { filename })
this.client
.get(filename, {
responseType: 'stream',
onDownloadProgress({ total, loaded, rate }) {
if (total) progressBar.setTotal(total)
progressBar.update(loaded, { speed: rate })
}
})
.then(response => {
response.data.pipe(stream)
})
}
}

@ -1,165 +0,0 @@
import { DataProcessorData } from '../types/dataProcessor'
import { DataLoaderData } from '../types/dataLoader'
import { Collection } from '@freearhey/core'
import {
BlocklistRecord,
Subdivision,
Category,
Language,
Timezone,
Channel,
Country,
Region,
Stream,
Guide,
City,
Feed,
Logo
} from '../models'
export class DataProcessor {
process(data: DataLoaderData): DataProcessorData {
let regions = new Collection(data.regions).map(data => new Region(data))
let regionsKeyByCode = regions.keyBy((region: Region) => region.code)
const categories = new Collection(data.categories).map(data => new Category(data))
const categoriesKeyById = categories.keyBy((category: Category) => category.id)
const languages = new Collection(data.languages).map(data => new Language(data))
const languagesKeyByCode = languages.keyBy((language: Language) => language.code)
let subdivisions = new Collection(data.subdivisions).map(data => new Subdivision(data))
let subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code)
let subdivisionsGroupedByCountryCode = subdivisions.groupBy(
(subdivision: Subdivision) => subdivision.countryCode
)
let countries = new Collection(data.countries).map(data => new Country(data))
let countriesKeyByCode = countries.keyBy((country: Country) => country.code)
const cities = new Collection(data.cities).map(data =>
new City(data)
.withRegions(regions)
.withCountry(countriesKeyByCode)
.withSubdivision(subdivisionsKeyByCode)
)
const citiesKeyByCode = cities.keyBy((city: City) => city.code)
const citiesGroupedByCountryCode = cities.groupBy((city: City) => city.countryCode)
const citiesGroupedBySubdivisionCode = cities.groupBy((city: City) => city.subdivisionCode)
const timezones = new Collection(data.timezones).map(data =>
new Timezone(data).withCountries(countriesKeyByCode)
)
const timezonesKeyById = timezones.keyBy((timezone: Timezone) => timezone.id)
const blocklistRecords = new Collection(data.blocklist).map(data => new BlocklistRecord(data))
const blocklistRecordsGroupedByChannelId = blocklistRecords.groupBy(
(blocklistRecord: BlocklistRecord) => blocklistRecord.channelId
)
let channels = new Collection(data.channels).map(data => new Channel(data))
let channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
let feeds = new Collection(data.feeds).map(data => new Feed(data))
let feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
let feedsGroupedById = feeds.groupBy((feed: Feed) => feed.id)
const logos = new Collection(data.logos).map(data => new Logo(data).withFeed(feedsGroupedById))
const logosGroupedByChannelId = logos.groupBy((logo: Logo) => logo.channelId)
const logosGroupedByStreamId = logos.groupBy((logo: Logo) => logo.getStreamId())
const streams = new Collection(data.streams).map(data =>
new Stream(data).withLogos(logosGroupedByStreamId)
)
const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
const guides = new Collection(data.guides).map(data => new Guide(data))
const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId())
regions = regions.map((region: Region) =>
region
.withCountries(countriesKeyByCode)
.withRegions(regions)
.withSubdivisions(subdivisions)
.withCities(cities)
)
regionsKeyByCode = regions.keyBy((region: Region) => region.code)
countries = countries.map((country: Country) =>
country
.withCities(citiesGroupedByCountryCode)
.withSubdivisions(subdivisionsGroupedByCountryCode)
.withRegions(regions)
.withLanguage(languagesKeyByCode)
)
countriesKeyByCode = countries.keyBy((country: Country) => country.code)
subdivisions = subdivisions.map((subdivision: Subdivision) =>
subdivision
.withCities(citiesGroupedBySubdivisionCode)
.withCountry(countriesKeyByCode)
.withRegions(regions)
.withParent(subdivisionsKeyByCode)
)
subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code)
subdivisionsGroupedByCountryCode = subdivisions.groupBy(
(subdivision: Subdivision) => subdivision.countryCode
)
channels = channels.map((channel: Channel) =>
channel
.withFeeds(feedsGroupedByChannelId)
.withLogos(logosGroupedByChannelId)
.withCategories(categoriesKeyById)
.withCountry(countriesKeyByCode)
.withSubdivision(subdivisionsKeyByCode)
.withCategories(categoriesKeyById)
)
channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
feeds = feeds.map((feed: Feed) =>
feed
.withChannel(channelsKeyById)
.withLanguages(languagesKeyByCode)
.withTimezones(timezonesKeyById)
.withBroadcastArea(
citiesKeyByCode,
subdivisionsKeyByCode,
countriesKeyByCode,
regionsKeyByCode
)
)
feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
feedsGroupedById = feeds.groupBy((feed: Feed) => feed.id)
return {
blocklistRecordsGroupedByChannelId,
subdivisionsGroupedByCountryCode,
feedsGroupedByChannelId,
guidesGroupedByStreamId,
logosGroupedByStreamId,
subdivisionsKeyByCode,
countriesKeyByCode,
languagesKeyByCode,
streamsGroupedById,
categoriesKeyById,
timezonesKeyById,
regionsKeyByCode,
blocklistRecords,
channelsKeyById,
citiesKeyByCode,
subdivisions,
categories,
countries,
languages,
timezones,
channels,
regions,
streams,
cities,
guides,
feeds,
logos
}
}
}

@ -1,16 +1,18 @@
type Column = {
import { Collection } from '@freearhey/core'
export type HTMLTableColumn = {
name: string
nowrap?: boolean
align?: string
}
type DataItem = string[]
export type HTMLTableItem = string[]
export class HTMLTable {
data: DataItem[]
columns: Column[]
data: Collection<HTMLTableItem>
columns: Collection<HTMLTableColumn>
constructor(data: DataItem[], columns: Column[]) {
constructor(data: Collection<HTMLTableItem>, columns: Collection<HTMLTableColumn>) {
this.data = data
this.columns = columns
}
@ -19,24 +21,26 @@ export class HTMLTable {
let output = '<table>\r\n'
output += ' <thead>\r\n <tr>'
for (const column of this.columns) {
this.columns.forEach((column: HTMLTableColumn) => {
output += `<th align="left">${column.name}</th>`
}
})
output += '</tr>\r\n </thead>\r\n'
output += ' <tbody>\r\n'
for (const item of this.data) {
this.data.forEach((item: HTMLTableItem) => {
output += ' <tr>'
let i = 0
for (const prop in item) {
const column = this.columns[i]
const column = this.columns.all()[i]
const nowrap = column.nowrap ? ' nowrap' : ''
const align = column.align ? ` align="${column.align}"` : ''
output += `<td${align}${nowrap}>${item[prop]}</td>`
i++
}
output += '</tr>\r\n'
}
})
output += ' </tbody>\r\n'
output += '</table>'

@ -1,7 +1,4 @@
export * from './apiClient'
export * from './cliTable'
export * from './dataProcessor'
export * from './dataLoader'
export * from './htmlTable'
export * from './issueData'
export * from './issueLoader'

@ -1,8 +1,8 @@
import { Dictionary } from '@freearhey/core'
export class IssueData {
_data: Dictionary
constructor(data: Dictionary) {
_data: Dictionary<string>
constructor(data: Dictionary<string>) {
this._data = data
}
@ -29,6 +29,8 @@ export class IssueData {
if (this._data.missing(key)) return undefined
return this._data.get(key) === deleteSymbol ? [] : this._data.get(key).split('\r\n')
const value = this._data.get(key)
return !value || value === deleteSymbol ? [] : value.split('\r\n')
}
}

@ -1,9 +1,9 @@
import { Collection } from '@freearhey/core'
import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
import { paginateRest } from '@octokit/plugin-paginate-rest'
import { TESTING, OWNER, REPO } from '../constants'
import { Collection } from '@freearhey/core'
import { Octokit } from '@octokit/core'
import { IssueParser } from './'
import { TESTING, OWNER, REPO } from '../constants'
const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
const octokit = new CustomOctokit()

@ -1,6 +1,6 @@
import { Dictionary } from '@freearhey/core'
import { Issue } from '../models'
import { IssueData } from './issueData'
import { Issue } from '../models'
const FIELDS = new Dictionary({
'Stream ID': 'streamId',
@ -23,7 +23,7 @@ export class IssueParser {
parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue {
const fields = typeof issue.body === 'string' ? issue.body.split('###') : []
const data = new Dictionary()
const data = new Dictionary<string>()
fields.forEach((field: string) => {
const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : []
let _label = parsed.shift()
@ -33,7 +33,7 @@ export class IssueParser {
if (!_label || !_value) return data
const id: string = FIELDS.get(_label)
const id = FIELDS.get(_label)
const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
if (!id) return

@ -1,5 +1,5 @@
import fs from 'fs'
import path from 'path'
import fs from 'fs'
type MarkdownConfig = {
build: string

@ -1,58 +1,41 @@
import { Collection, Storage, Dictionary } from '@freearhey/core'
import { Storage } from '@freearhey/storage-js'
import { Collection } from '@freearhey/core'
import parser from 'iptv-playlist-parser'
import { Stream } from '../models'
type PlaylistPareserProps = {
storage: Storage
feedsGroupedByChannelId: Dictionary
logosGroupedByStreamId: Dictionary
channelsKeyById: Dictionary
}
export class PlaylistParser {
storage: Storage
feedsGroupedByChannelId: Dictionary
logosGroupedByStreamId: Dictionary
channelsKeyById: Dictionary
constructor({
storage,
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById
}: PlaylistPareserProps) {
constructor({ storage }: PlaylistPareserProps) {
this.storage = storage
this.feedsGroupedByChannelId = feedsGroupedByChannelId
this.logosGroupedByStreamId = logosGroupedByStreamId
this.channelsKeyById = channelsKeyById
}
async parse(files: string[]): Promise<Collection> {
let streams = new Collection()
async parse(files: string[]): Promise<Collection<Stream>> {
const parsed = new Collection<Stream>()
for (const filepath of files) {
if (!this.storage.existsSync(filepath)) continue
const _streams: Collection = await this.parseFile(filepath)
streams = streams.concat(_streams)
const _parsed: Collection<Stream> = await this.parseFile(filepath)
parsed.concat(_parsed)
}
return streams
return parsed
}
async parseFile(filepath: string): Promise<Collection> {
async parseFile(filepath: string): Promise<Collection<Stream>> {
const content = await this.storage.load(filepath)
const parsed: parser.Playlist = parser.parse(content)
const streams = new Collection(parsed.items).map((data: parser.PlaylistItem) => {
const stream = new Stream()
.fromPlaylistItem(data)
.withFeed(this.feedsGroupedByChannelId)
.withChannel(this.channelsKeyById)
.withLogos(this.logosGroupedByStreamId)
.setFilepath(filepath)
const streams = new Collection<Stream>()
parsed.items.forEach((data: parser.PlaylistItem) => {
const stream = Stream.fromPlaylistItem(data)
stream.filepath = filepath
return stream
streams.add(stream)
})
return streams

@ -1,18 +1,25 @@
import { Stream } from '../models'
import { TESTING } from '../constants'
import mediaInfoFactory from 'mediainfo.js'
import axios, { AxiosInstance, AxiosProxyConfig, AxiosRequestConfig } from 'axios'
import axios, { AxiosInstance, AxiosProxyConfig, AxiosRequestConfig, AxiosResponse } from 'axios'
import { SocksProxyAgent } from 'socks-proxy-agent'
import { ProxyParser } from './proxyParser.js'
import mediaInfoFactory from 'mediainfo.js'
import { OptionValues } from 'commander'
import { SocksProxyAgent } from 'socks-proxy-agent'
import { TESTING } from '../constants'
import { Stream } from '../models'
export type TestResult = {
export type StreamTesterResult = {
status: {
ok: boolean
code: string
}
}
export type StreamTesterError = {
name: string
code?: string
cause?: Error & { code?: string }
response?: AxiosResponse
}
export type StreamTesterProps = {
options: OptionValues
}
@ -46,7 +53,7 @@ export class StreamTester {
this.options = options
}
async test(stream: Stream): Promise<TestResult> {
async test(stream: Stream): Promise<StreamTesterResult> {
if (TESTING) {
const results = (await import('../../tests/__data__/input/playlist_test/results.js')).default
@ -56,8 +63,8 @@ export class StreamTester {
const res = await this.client(stream.url, {
signal: AbortSignal.timeout(this.options.timeout),
headers: {
'User-Agent': stream.getUserAgent() || 'Mozilla/5.0',
Referer: stream.getReferrer()
'User-Agent': stream.user_agent || 'Mozilla/5.0',
Referer: stream.referrer
}
})
@ -65,8 +72,7 @@ export class StreamTester {
const buffer = await res.data
const result = await mediainfo.analyzeData(
() => buffer.byteLength,
(size: any, offset: number | undefined) =>
Buffer.from(buffer).subarray(offset, offset + size)
(size: number, offset: number) => Buffer.from(buffer).subarray(offset, offset + size)
)
if (result && result.media && result.media.track.length > 0) {
@ -84,7 +90,9 @@ export class StreamTester {
}
}
}
} catch (error: any) {
} catch (err: unknown) {
const error = err as StreamTesterError
let code = 'UNKNOWN_ERROR'
if (error.name === 'CanceledError') {
code = 'TIMEOUT'
@ -97,7 +105,7 @@ export class StreamTester {
code = `AXIOS_${error.code}`
}
} else if (error.cause) {
const cause = error.cause as Error & { code?: string }
const cause = error.cause
if (cause.code) {
code = cause.code
} else {

@ -1,17 +1,19 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Category, Playlist } from '../models'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Collection } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type CategoriesGeneratorProps = {
streams: Collection
categories: Collection
streams: Collection<Stream>
categories: Collection<sdk.Models.Category>
logFile: File
}
export class CategoriesGenerator implements Generator {
streams: Collection
categories: Collection
streams: Collection<Stream>
categories: Collection<sdk.Models.Category>
storage: Storage
logFile: File
@ -23,13 +25,17 @@ export class CategoriesGenerator implements Generator {
}
async generate() {
const streams = this.streams.orderBy([(stream: Stream) => stream.getTitle()])
const streams = this.streams.sortBy([(stream: Stream) => stream.title])
this.categories.forEach(async (category: Category) => {
this.categories.forEach(async (category: sdk.Models.Category) => {
const categoryStreams = streams
.filter((stream: Stream) => stream.hasCategory(category))
.map((stream: Stream) => {
const groupTitle = stream.getCategoryNames().join(';')
const groupTitle = stream
.getCategories()
.map(category => category.name)
.sort()
.join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
@ -43,7 +49,7 @@ export class CategoriesGenerator implements Generator {
)
})
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasCategories())
const undefinedStreams = streams.filter((stream: Stream) => stream.getCategories().isEmpty())
const playlist = new Playlist(undefinedStreams, { public: true })
const filepath = 'categories/undefined.m3u'
await this.storage.save(filepath, playlist.toString())

@ -1,17 +1,19 @@
import { City, Stream, Playlist } from '../models'
import { Collection, Storage, File } from '@freearhey/core'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type CitiesGeneratorProps = {
streams: Collection
cities: Collection
streams: Collection<Stream>
cities: Collection<sdk.Models.City>
logFile: File
}
export class CitiesGenerator implements Generator {
streams: Collection
cities: Collection
streams: Collection<Stream>
cities: Collection<sdk.Models.City>
storage: Storage
logFile: File
@ -24,20 +26,29 @@ export class CitiesGenerator implements Generator {
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
this.cities.forEach(async (city: City) => {
const cityStreams = streams.filter((stream: Stream) => stream.isBroadcastInCity(city))
const streamsGroupedByCityCode = {}
streams.forEach((stream: Stream) => {
stream.getBroadcastCities().forEach((city: sdk.Models.City) => {
if (streamsGroupedByCityCode[city.code]) {
streamsGroupedByCityCode[city.code].add(stream)
} else {
streamsGroupedByCityCode[city.code] = new Collection<Stream>([stream])
}
})
})
if (cityStreams.isEmpty()) return
for (const cityCode in streamsGroupedByCityCode) {
const cityStreams = streamsGroupedByCityCode[cityCode]
const playlist = new Playlist(cityStreams, { public: true })
const filepath = `cities/${city.code.toLowerCase()}.m3u`
const filepath = `cities/${cityCode.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'city', filepath, count: playlist.streams.count() }) + EOL
)
})
}
}
}

@ -1,17 +1,19 @@
import { Country, Stream, Playlist } from '../models'
import { Collection, Storage, File } from '@freearhey/core'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type CountriesGeneratorProps = {
streams: Collection
countries: Collection
streams: Collection<Stream>
countries: Collection<sdk.Models.Country>
logFile: File
}
export class CountriesGenerator implements Generator {
streams: Collection
countries: Collection
streams: Collection<Stream>
countries: Collection<sdk.Models.Country>
storage: Storage
logFile: File
@ -24,22 +26,30 @@ export class CountriesGenerator implements Generator {
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
this.countries.forEach(async (country: Country) => {
const countryStreams = streams.filter((stream: Stream) =>
stream.isBroadcastInCountry(country)
)
if (countryStreams.isEmpty()) return
const streamsGroupedByCountryCode = {}
streams.forEach((stream: Stream) => {
stream.getBroadcastCountries().forEach((country: sdk.Models.Country) => {
if (streamsGroupedByCountryCode[country.code]) {
streamsGroupedByCountryCode[country.code].add(stream)
} else {
streamsGroupedByCountryCode[country.code] = new Collection<Stream>([stream])
}
})
})
for (const countryCode in streamsGroupedByCountryCode) {
const countryStreams = streamsGroupedByCountryCode[countryCode]
const playlist = new Playlist(countryStreams, { public: true })
const filepath = `countries/${country.code.toLowerCase()}.m3u`
const filepath = `countries/${countryCode.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'country', filepath, count: playlist.streams.count() }) + EOL
)
})
}
const internationalStreams = streams.filter((stream: Stream) => stream.isInternational())
const internationalPlaylist = new Playlist(internationalStreams, { public: true })
@ -53,7 +63,9 @@ export class CountriesGenerator implements Generator {
}) + EOL
)
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea())
const undefinedStreams = streams.filter((stream: Stream) =>
stream.getBroadcastAreaCodes().isEmpty()
)
const undefinedPlaylist = new Playlist(undefinedStreams, { public: true })
const undefinedFilepath = 'countries/undefined.m3u'
await this.storage.save(undefinedFilepath, undefinedPlaylist.toString())

@ -5,7 +5,6 @@ export * from './indexCategoryGenerator'
export * from './indexCountryGenerator'
export * from './indexGenerator'
export * from './indexLanguageGenerator'
export * from './indexNsfwGenerator'
export * from './languagesGenerator'
export * from './rawGenerator'
export * from './regionsGenerator'

@ -1,15 +1,17 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Playlist, Category } from '../models'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type IndexCategoryGeneratorProps = {
streams: Collection
streams: Collection<Stream>
logFile: File
}
export class IndexCategoryGenerator implements Generator {
streams: Collection
streams: Collection<Stream>
storage: Storage
logFile: File
@ -20,27 +22,26 @@ export class IndexCategoryGenerator implements Generator {
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy(stream => stream.getTitle())
.filter(stream => stream.isSFW())
const streams = this.streams.sortBy(stream => stream.title).filter(stream => stream.isSFW())
let groupedStreams = new Collection()
let groupedStreams = new Collection<Stream>()
streams.forEach((stream: Stream) => {
if (!stream.hasCategories()) {
const streamCategories = stream.getCategories()
if (streamCategories.isEmpty()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
stream.getCategories().forEach((category: Category) => {
streamCategories.forEach((category: sdk.Models.Category) => {
const streamClone = stream.clone()
streamClone.groupTitle = category.name
groupedStreams.push(streamClone)
groupedStreams.add(streamClone)
})
})
groupedStreams = groupedStreams.orderBy(stream => {
groupedStreams = groupedStreams.sortBy(stream => {
if (stream.groupTitle === 'Undefined') return 'ZZ'
return stream.groupTitle
})

@ -1,15 +1,17 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Playlist, Country } from '../models'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type IndexCountryGeneratorProps = {
streams: Collection
streams: Collection<Stream>
logFile: File
}
export class IndexCountryGenerator implements Generator {
streams: Collection
streams: Collection<Stream>
storage: Storage
logFile: File
@ -20,20 +22,22 @@ export class IndexCountryGenerator implements Generator {
}
async generate(): Promise<void> {
let groupedStreams = new Collection()
let groupedStreams = new Collection<Stream>()
this.streams
.orderBy((stream: Stream) => stream.getTitle())
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
if (!stream.hasBroadcastArea()) {
const broadcastAreaCountries = stream.getBroadcastCountries()
if (stream.getBroadcastAreaCodes().isEmpty()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
stream.getBroadcastCountries().forEach((country: Country) => {
broadcastAreaCountries.forEach((country: sdk.Models.Country) => {
const streamClone = stream.clone()
streamClone.groupTitle = country.name
groupedStreams.add(streamClone)
@ -46,7 +50,7 @@ export class IndexCountryGenerator implements Generator {
}
})
groupedStreams = groupedStreams.orderBy((stream: Stream) => {
groupedStreams = groupedStreams.sortBy((stream: Stream) => {
if (stream.groupTitle === 'International') return 'ZZ'
if (stream.groupTitle === 'Undefined') return 'ZZZ'

@ -1,15 +1,16 @@
import { Collection, File, Storage } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
type IndexGeneratorProps = {
streams: Collection
streams: Collection<Stream>
logFile: File
}
export class IndexGenerator implements Generator {
streams: Collection
streams: Collection<Stream>
storage: Storage
logFile: File
@ -21,10 +22,14 @@ export class IndexGenerator implements Generator {
async generate(): Promise<void> {
const sfwStreams = this.streams
.orderBy(stream => stream.getTitle())
.sortBy(stream => stream.title)
.filter((stream: Stream) => stream.isSFW())
.map((stream: Stream) => {
const groupTitle = stream.getCategoryNames().join(';')
const groupTitle = stream
.getCategories()
.map(category => category.name)
.sort()
.join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream

@ -1,15 +1,17 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Playlist, Language } from '../models'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type IndexLanguageGeneratorProps = {
streams: Collection
streams: Collection<Stream>
logFile: File
}
export class IndexLanguageGenerator implements Generator {
streams: Collection
streams: Collection<Stream>
storage: Storage
logFile: File
@ -20,26 +22,27 @@ export class IndexLanguageGenerator implements Generator {
}
async generate(): Promise<void> {
let groupedStreams = new Collection()
let groupedStreams = new Collection<Stream>()
this.streams
.orderBy((stream: Stream) => stream.getTitle())
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
if (!stream.hasLanguages()) {
const streamLanguages = stream.getLanguages()
if (streamLanguages.isEmpty()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
stream.getLanguages().forEach((language: Language) => {
streamLanguages.forEach((language: sdk.Models.Language) => {
const streamClone = stream.clone()
streamClone.groupTitle = language.name
groupedStreams.add(streamClone)
})
})
groupedStreams = groupedStreams.orderBy((stream: Stream) => {
groupedStreams = groupedStreams.sortBy((stream: Stream) => {
if (stream.groupTitle === 'Undefined') return 'ZZ'
return stream.groupTitle
})

@ -1,32 +0,0 @@
import { Collection, File, Storage } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type IndexNsfwGeneratorProps = {
streams: Collection
logFile: File
}
export class IndexNsfwGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexNsfwGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const allStreams = this.streams.orderBy((stream: Stream) => stream.getTitle())
const playlist = new Playlist(allStreams, { public: true })
const filepath = 'index.nsfw.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}

@ -1,12 +1,14 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Playlist, Language, Stream } from '../models'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Playlist, Stream } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type LanguagesGeneratorProps = { streams: Collection; logFile: File }
type LanguagesGeneratorProps = { streams: Collection<Stream>; logFile: File }
export class LanguagesGenerator implements Generator {
streams: Collection
streams: Collection<Stream>
storage: Storage
logFile: File
@ -17,20 +19,20 @@ export class LanguagesGenerator implements Generator {
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
const streams: Collection<Stream> = this.streams
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
let languages = new Collection()
const languages = new Collection<sdk.Models.Language>()
streams.forEach((stream: Stream) => {
languages = languages.concat(stream.getLanguages())
languages.concat(stream.getLanguages())
})
languages
.filter(Boolean)
.uniqBy((language: Language) => language.code)
.orderBy((language: Language) => language.name)
.forEach(async (language: Language) => {
.uniqBy((language: sdk.Models.Language) => language.code)
.sortBy((language: sdk.Models.Language) => language.name)
.forEach(async (language: sdk.Models.Language) => {
const languageStreams = streams.filter((stream: Stream) => stream.hasLanguage(language))
if (languageStreams.isEmpty()) return
@ -43,8 +45,7 @@ export class LanguagesGenerator implements Generator {
)
})
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasLanguages())
const undefinedStreams = streams.filter((stream: Stream) => stream.getLanguages().isEmpty())
if (undefinedStreams.isEmpty()) return
const playlist = new Playlist(undefinedStreams, { public: true })

@ -1,15 +1,16 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
type RawGeneratorProps = {
streams: Collection
streams: Collection<Stream>
logFile: File
}
export class RawGenerator implements Generator {
streams: Collection
streams: Collection<Stream>
storage: Storage
logFile: File
@ -24,7 +25,11 @@ export class RawGenerator implements Generator {
for (const filename of files.keys()) {
const streams = new Collection(files.get(filename)).map((stream: Stream) => {
const groupTitle = stream.getCategoryNames().join(';')
const groupTitle = stream
.getCategories()
.map(category => category.name)
.sort()
.join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream

@ -1,17 +1,19 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Playlist, Region, Stream } from '../models'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Playlist, Stream } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type RegionsGeneratorProps = {
streams: Collection
regions: Collection
streams: Collection<Stream>
regions: Collection<sdk.Models.Region>
logFile: File
}
export class RegionsGenerator implements Generator {
streams: Collection
regions: Collection
streams: Collection<Stream>
regions: Collection<sdk.Models.Region>
storage: Storage
logFile: File
@ -24,18 +26,29 @@ export class RegionsGenerator implements Generator {
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
this.regions.forEach(async (region: Region) => {
const regionStreams = streams.filter((stream: Stream) => stream.isBroadcastInRegion(region))
const streamsGroupedByRegionCode = {}
streams.forEach((stream: Stream) => {
stream.getBroadcastRegions().forEach((region: sdk.Models.Region) => {
if (streamsGroupedByRegionCode[region.code]) {
streamsGroupedByRegionCode[region.code].add(stream)
} else {
streamsGroupedByRegionCode[region.code] = new Collection<Stream>([stream])
}
})
})
for (const regionCode in streamsGroupedByRegionCode) {
const regionStreams = streamsGroupedByRegionCode[regionCode]
const playlist = new Playlist(regionStreams, { public: true })
const filepath = `regions/${region.code.toLowerCase()}.m3u`
const filepath = `regions/${regionCode.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() }) + EOL
)
})
}
}
}

@ -1,15 +1,16 @@
import { Collection, Storage, File, type Dictionary } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { Collection, Dictionary } from '@freearhey/core'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Generator } from './generator'
type SourcesGeneratorProps = {
streams: Collection
streams: Collection<Stream>
logFile: File
}
export class SourcesGenerator implements Generator {
streams: Collection
streams: Collection<Stream>
storage: Storage
logFile: File
@ -20,14 +21,19 @@ export class SourcesGenerator implements Generator {
}
async generate() {
const files: Dictionary = this.streams.groupBy((stream: Stream) => stream.getFilename())
const files: Dictionary<Stream[]> = this.streams.groupBy((stream: Stream) =>
stream.getFilename()
)
for (const filename of files.keys()) {
if (!filename) continue
let streams = new Collection(files.get(filename))
streams = streams.map((stream: Stream) => {
const groupTitle = stream.getCategoryNames().join(';')
const streams = new Collection<Stream>(files.get(filename)).map((stream: Stream) => {
const groupTitle = stream
.getCategories()
.map(category => category.name)
.sort()
.join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream

@ -1,17 +1,19 @@
import { Subdivision, Stream, Playlist } from '../models'
import { Collection, Storage, File } from '@freearhey/core'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type SubdivisionsGeneratorProps = {
streams: Collection
subdivisions: Collection
streams: Collection<Stream>
subdivisions: Collection<sdk.Models.Subdivision>
logFile: File
}
export class SubdivisionsGenerator implements Generator {
streams: Collection
subdivisions: Collection
streams: Collection<Stream>
subdivisions: Collection<sdk.Models.Subdivision>
storage: Storage
logFile: File
@ -24,22 +26,29 @@ export class SubdivisionsGenerator implements Generator {
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
this.subdivisions.forEach(async (subdivision: Subdivision) => {
const subdivisionStreams = streams.filter((stream: Stream) =>
stream.isBroadcastInSubdivision(subdivision)
)
const streamsGroupedBySubdivisionCode = {}
streams.forEach((stream: Stream) => {
stream.getBroadcastSubdivisions().forEach((subdivision: sdk.Models.Subdivision) => {
if (streamsGroupedBySubdivisionCode[subdivision.code]) {
streamsGroupedBySubdivisionCode[subdivision.code].add(stream)
} else {
streamsGroupedBySubdivisionCode[subdivision.code] = new Collection<Stream>([stream])
}
})
})
if (subdivisionStreams.isEmpty()) return
for (const subdivisionCode in streamsGroupedBySubdivisionCode) {
const subdivisionStreams = streamsGroupedBySubdivisionCode[subdivisionCode]
const playlist = new Playlist(subdivisionStreams, { public: true })
const filepath = `subdivisions/${subdivision.code.toLowerCase()}.m3u`
const filepath = `subdivisions/${subdivisionCode.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'subdivision', filepath, count: playlist.streams.count() }) + EOL
)
})
}
}
}

@ -1,15 +0,0 @@
import type { BlocklistRecordData } from '../types/blocklistRecord'
export class BlocklistRecord {
channelId: string
reason: string
ref: string
constructor(data?: BlocklistRecordData) {
if (!data) return
this.channelId = data.channel
this.reason = data.reason
this.ref = data.ref
}
}

@ -1,108 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { City, Subdivision, Region, Country } from './'
export class BroadcastArea {
codes: Collection
citiesIncluded: Collection
subdivisionsIncluded: Collection
countriesIncluded: Collection
regionsIncluded: Collection
constructor(codes: Collection) {
this.codes = codes
}
withLocations(
citiesKeyByCode: Dictionary,
subdivisionsKeyByCode: Dictionary,
countriesKeyByCode: Dictionary,
regionsKeyByCode: Dictionary
): this {
const citiesIncluded = new Collection()
const subdivisionsIncluded = new Collection()
const countriesIncluded = new Collection()
let regionsIncluded = new Collection()
this.codes.forEach((value: string) => {
const [type, code] = value.split('/')
switch (type) {
case 'ct': {
const city: City = citiesKeyByCode.get(code)
if (!city) return
citiesIncluded.add(city)
if (city.subdivision) subdivisionsIncluded.add(city.subdivision)
if (city.subdivision && city.subdivision.parent)
subdivisionsIncluded.add(city.subdivision.parent)
if (city.country) countriesIncluded.add(city.country)
regionsIncluded = regionsIncluded.concat(city.getRegions())
break
}
case 's': {
const subdivision: Subdivision = subdivisionsKeyByCode.get(code)
if (!subdivision) return
subdivisionsIncluded.add(subdivision)
if (subdivision.country) countriesIncluded.add(subdivision.country)
regionsIncluded = regionsIncluded.concat(subdivision.getRegions())
break
}
case 'c': {
const country: Country = countriesKeyByCode.get(code)
if (!country) return
countriesIncluded.add(country)
regionsIncluded = regionsIncluded.concat(country.getRegions())
break
}
case 'r': {
const region: Region = regionsKeyByCode.get(code)
if (!region) return
regionsIncluded = regionsIncluded.concat(region.getRegions())
break
}
}
})
this.citiesIncluded = citiesIncluded.uniqBy((city: City) => city.code)
this.subdivisionsIncluded = subdivisionsIncluded.uniqBy(
(subdivision: Subdivision) => subdivision.code
)
this.countriesIncluded = countriesIncluded.uniqBy((country: Country) => country.code)
this.regionsIncluded = regionsIncluded.uniqBy((region: Region) => region.code)
return this
}
getCountries(): Collection {
return this.countriesIncluded || new Collection()
}
getSubdivisions(): Collection {
return this.subdivisionsIncluded || new Collection()
}
getCities(): Collection {
return this.citiesIncluded || new Collection()
}
getRegions(): Collection {
return this.regionsIncluded || new Collection()
}
includesCountry(country: Country): boolean {
return this.getCountries().includes((_country: Country) => _country.code === country.code)
}
includesSubdivision(subdivision: Subdivision): boolean {
return this.getSubdivisions().includes(
(_subdivision: Subdivision) => _subdivision.code === subdivision.code
)
}
includesRegion(region: Region): boolean {
return this.getRegions().includes((_region: Region) => _region.code === region.code)
}
includesCity(city: City): boolean {
return this.getCities().includes((_city: City) => _city.code === city.code)
}
}

@ -1,18 +0,0 @@
import type { CategoryData, CategorySerializedData } from '../types/category'
export class Category {
id: string
name: string
constructor(data: CategoryData) {
this.id = data.id
this.name = data.name
}
serialize(): CategorySerializedData {
return {
id: this.id,
name: this.name
}
}
}

@ -1,233 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { Category, Country, Feed, Guide, Logo, Stream, Subdivision } from './index'
import type { ChannelData, ChannelSearchableData, ChannelSerializedData } from '../types/channel'
export class Channel {
id: string
name: string
altNames: Collection
network?: string
owners: Collection
countryCode: string
country?: Country
subdivisionCode?: string
subdivision?: Subdivision
cityName?: string
categoryIds: Collection
categories: Collection = new Collection()
isNSFW: boolean
launched?: string
closed?: string
replacedBy?: string
isClosed: boolean
website?: string
feeds?: Collection
logos: Collection = new Collection()
constructor(data?: ChannelData) {
if (!data) return
this.id = data.id
this.name = data.name
this.altNames = new Collection(data.alt_names)
this.network = data.network || undefined
this.owners = new Collection(data.owners)
this.countryCode = data.country
this.subdivisionCode = data.subdivision || undefined
this.cityName = data.city || undefined
this.categoryIds = new Collection(data.categories)
this.isNSFW = data.is_nsfw
this.launched = data.launched || undefined
this.closed = data.closed || undefined
this.replacedBy = data.replaced_by || undefined
this.website = data.website || undefined
this.isClosed = !!data.closed || !!data.replaced_by
}
withSubdivision(subdivisionsKeyByCode: Dictionary): this {
if (!this.subdivisionCode) return this
this.subdivision = subdivisionsKeyByCode.get(this.subdivisionCode)
return this
}
withCountry(countriesKeyByCode: Dictionary): this {
this.country = countriesKeyByCode.get(this.countryCode)
return this
}
withCategories(categoriesKeyById: Dictionary): this {
this.categories = this.categoryIds
.map((id: string) => categoriesKeyById.get(id))
.filter(Boolean)
return this
}
withFeeds(feedsGroupedByChannelId: Dictionary): this {
this.feeds = new Collection(feedsGroupedByChannelId.get(this.id))
return this
}
withLogos(logosGroupedByChannelId: Dictionary): this {
if (this.id) this.logos = new Collection(logosGroupedByChannelId.get(this.id))
return this
}
getCountry(): Country | undefined {
return this.country
}
getSubdivision(): Subdivision | undefined {
return this.subdivision
}
getCategories(): Collection {
return this.categories || new Collection()
}
hasCategories(): boolean {
return !!this.categories && this.categories.notEmpty()
}
hasCategory(category: Category): boolean {
return (
!!this.categories &&
this.categories.includes((_category: Category) => _category.id === category.id)
)
}
getFeeds(): Collection {
if (!this.feeds) return new Collection()
return this.feeds
}
getGuides(): Collection {
let guides = new Collection()
this.getFeeds().forEach((feed: Feed) => {
guides = guides.concat(feed.getGuides())
})
return guides
}
getGuideNames(): Collection {
return this.getGuides()
.map((guide: Guide) => guide.siteName)
.uniq()
}
getStreams(): Collection {
let streams = new Collection()
this.getFeeds().forEach((feed: Feed) => {
streams = streams.concat(feed.getStreams())
})
return streams
}
getStreamTitles(): Collection {
return this.getStreams()
.map((stream: Stream) => stream.getTitle())
.uniq()
}
getFeedFullNames(): Collection {
return this.getFeeds()
.map((feed: Feed) => feed.getFullName())
.uniq()
}
isSFW(): boolean {
return this.isNSFW === false
}
getLogos(): Collection {
function feed(logo: Logo): number {
if (!logo.feed) return 1
if (logo.feed.isMain) return 1
return 0
}
function format(logo: Logo): number {
const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 }
return logo.format ? levelByFormat[logo.format] : 0
}
function size(logo: Logo): number {
return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
}
return this.logos.orderBy([feed, format, size], ['desc', 'desc', 'asc'], false)
}
getLogo(): Logo | undefined {
return this.getLogos().first()
}
hasLogo(): boolean {
return this.getLogos().notEmpty()
}
getSearchable(): ChannelSearchableData {
return {
id: this.id,
name: this.name,
altNames: this.altNames.all(),
guideNames: this.getGuideNames().all(),
streamTitles: this.getStreamTitles().all(),
feedFullNames: this.getFeedFullNames().all()
}
}
serialize(): ChannelSerializedData {
return {
id: this.id,
name: this.name,
altNames: this.altNames.all(),
network: this.network,
owners: this.owners.all(),
countryCode: this.countryCode,
country: this.country ? this.country.serialize() : undefined,
subdivisionCode: this.subdivisionCode,
subdivision: this.subdivision ? this.subdivision.serialize() : undefined,
cityName: this.cityName,
categoryIds: this.categoryIds.all(),
categories: this.categories.map((category: Category) => category.serialize()).all(),
isNSFW: this.isNSFW,
launched: this.launched,
closed: this.closed,
replacedBy: this.replacedBy,
website: this.website
}
}
deserialize(data: ChannelSerializedData): this {
this.id = data.id
this.name = data.name
this.altNames = new Collection(data.altNames)
this.network = data.network
this.owners = new Collection(data.owners)
this.countryCode = data.countryCode
this.country = data.country ? new Country().deserialize(data.country) : undefined
this.subdivisionCode = data.subdivisionCode
this.cityName = data.cityName
this.categoryIds = new Collection(data.categoryIds)
this.isNSFW = data.isNSFW
this.launched = data.launched
this.closed = data.closed
this.replacedBy = data.replacedBy
this.website = data.website
return this
}
}

@ -1,78 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { Country, Region, Subdivision } from '.'
import type { CityData, CitySerializedData } from '../types/city'
export class City {
code: string
name: string
countryCode: string
country?: Country
subdivisionCode?: string
subdivision?: Subdivision
wikidataId: string
regions?: Collection
constructor(data?: CityData) {
if (!data) return
this.code = data.code
this.name = data.name
this.countryCode = data.country
this.subdivisionCode = data.subdivision || undefined
this.wikidataId = data.wikidata_id
}
withCountry(countriesKeyByCode: Dictionary): this {
this.country = countriesKeyByCode.get(this.countryCode)
return this
}
withSubdivision(subdivisionsKeyByCode: Dictionary): this {
if (!this.subdivisionCode) return this
this.subdivision = subdivisionsKeyByCode.get(this.subdivisionCode)
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter((region: Region) =>
region.countryCodes.includes(this.countryCode)
)
return this
}
getRegions(): Collection {
if (!this.regions) return new Collection()
return this.regions
}
serialize(): CitySerializedData {
return {
code: this.code,
name: this.name,
countryCode: this.countryCode,
country: this.country ? this.country.serialize() : undefined,
subdivisionCode: this.subdivisionCode || null,
subdivision: this.subdivision ? this.subdivision.serialize() : undefined,
wikidataId: this.wikidataId
}
}
deserialize(data: CitySerializedData): this {
this.code = data.code
this.name = data.name
this.countryCode = data.countryCode
this.country = data.country ? new Country().deserialize(data.country) : undefined
this.subdivisionCode = data.subdivisionCode || undefined
this.subdivision = data.subdivision
? new Subdivision().deserialize(data.subdivision)
: undefined
this.wikidataId = data.wikidataId
return this
}
}

@ -1,95 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { Region, Language, Subdivision } from '.'
import type { CountryData, CountrySerializedData } from '../types/country'
import { SubdivisionSerializedData } from '../types/subdivision'
import { RegionSerializedData } from '../types/region'
export class Country {
code: string
name: string
flag: string
languageCode: string
language?: Language
subdivisions?: Collection
regions?: Collection
cities?: Collection
constructor(data?: CountryData) {
if (!data) return
this.code = data.code
this.name = data.name
this.flag = data.flag
this.languageCode = data.lang
}
withSubdivisions(subdivisionsGroupedByCountryCode: Dictionary): this {
this.subdivisions = new Collection(subdivisionsGroupedByCountryCode.get(this.code))
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter((region: Region) => region.includesCountryCode(this.code))
return this
}
withCities(citiesGroupedByCountryCode: Dictionary): this {
this.cities = new Collection(citiesGroupedByCountryCode.get(this.code))
return this
}
withLanguage(languagesKeyByCode: Dictionary): this {
this.language = languagesKeyByCode.get(this.languageCode)
return this
}
getLanguage(): Language | undefined {
return this.language
}
getRegions(): Collection {
return this.regions || new Collection()
}
getSubdivisions(): Collection {
return this.subdivisions || new Collection()
}
getCities(): Collection {
return this.cities || new Collection()
}
serialize(): CountrySerializedData {
return {
code: this.code,
name: this.name,
flag: this.flag,
languageCode: this.languageCode,
language: this.language ? this.language.serialize() : null,
subdivisions: this.subdivisions
? this.subdivisions.map((subdivision: Subdivision) => subdivision.serialize()).all()
: [],
regions: this.regions ? this.regions.map((region: Region) => region.serialize()).all() : []
}
}
deserialize(data: CountrySerializedData): this {
this.code = data.code
this.name = data.name
this.flag = data.flag
this.languageCode = data.languageCode
this.language = data.language ? new Language().deserialize(data.language) : undefined
this.subdivisions = new Collection(data.subdivisions).map((data: SubdivisionSerializedData) =>
new Subdivision().deserialize(data)
)
this.regions = new Collection(data.regions).map((data: RegionSerializedData) =>
new Region().deserialize(data)
)
return this
}
}

@ -1,170 +0,0 @@
import { Country, Language, Region, Channel, Subdivision, BroadcastArea, City } from './index'
import { Collection, Dictionary } from '@freearhey/core'
import type { FeedData } from '../types/feed'
export class Feed {
channelId: string
channel?: Channel
id: string
name: string
isMain: boolean
broadcastAreaCodes: Collection
broadcastArea?: BroadcastArea
languageCodes: Collection
languages?: Collection
timezoneIds: Collection
timezones?: Collection
videoFormat: string
guides?: Collection
streams?: Collection
constructor(data: FeedData) {
this.channelId = data.channel
this.id = data.id
this.name = data.name
this.isMain = data.is_main
this.broadcastAreaCodes = new Collection(data.broadcast_area)
this.languageCodes = new Collection(data.languages)
this.timezoneIds = new Collection(data.timezones)
this.videoFormat = data.video_format
}
withChannel(channelsKeyById: Dictionary): this {
this.channel = channelsKeyById.get(this.channelId)
return this
}
withStreams(streamsGroupedById: Dictionary): this {
this.streams = new Collection(streamsGroupedById.get(`${this.channelId}@${this.id}`))
if (this.isMain) {
this.streams = this.streams.concat(new Collection(streamsGroupedById.get(this.channelId)))
}
return this
}
withGuides(guidesGroupedByStreamId: Dictionary): this {
this.guides = new Collection(guidesGroupedByStreamId.get(`${this.channelId}@${this.id}`))
if (this.isMain) {
this.guides = this.guides.concat(new Collection(guidesGroupedByStreamId.get(this.channelId)))
}
return this
}
withLanguages(languagesKeyByCode: Dictionary): this {
this.languages = this.languageCodes
.map((code: string) => languagesKeyByCode.get(code))
.filter(Boolean)
return this
}
withTimezones(timezonesKeyById: Dictionary): this {
this.timezones = this.timezoneIds.map((id: string) => timezonesKeyById.get(id)).filter(Boolean)
return this
}
withBroadcastArea(
citiesKeyByCode: Dictionary,
subdivisionsKeyByCode: Dictionary,
countriesKeyByCode: Dictionary,
regionsKeyByCode: Dictionary
): this {
this.broadcastArea = new BroadcastArea(this.broadcastAreaCodes).withLocations(
citiesKeyByCode,
subdivisionsKeyByCode,
countriesKeyByCode,
regionsKeyByCode
)
return this
}
hasBroadcastArea(): boolean {
return !!this.broadcastArea
}
getBroadcastCountries(): Collection {
if (!this.broadcastArea) return new Collection()
return this.broadcastArea.getCountries()
}
getBroadcastRegions(): Collection {
if (!this.broadcastArea) return new Collection()
return this.broadcastArea.getRegions()
}
getTimezones(): Collection {
return this.timezones || new Collection()
}
getLanguages(): Collection {
return this.languages || new Collection()
}
hasLanguages(): boolean {
return !!this.languages && this.languages.notEmpty()
}
hasLanguage(language: Language): boolean {
return (
!!this.languages &&
this.languages.includes((_language: Language) => _language.code === language.code)
)
}
isBroadcastInCity(city: City): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.includesCity(city)
}
isBroadcastInSubdivision(subdivision: Subdivision): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.includesSubdivision(subdivision)
}
isBroadcastInCountry(country: Country): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.includesCountry(country)
}
isBroadcastInRegion(region: Region): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.includesRegion(region)
}
isInternational(): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.codes.join(',').includes('r/')
}
getGuides(): Collection {
if (!this.guides) return new Collection()
return this.guides
}
getStreams(): Collection {
if (!this.streams) return new Collection()
return this.streams
}
getFullName(): string {
if (!this.channel) return ''
return `${this.channel.name} ${this.name}`
}
}

@ -1,54 +0,0 @@
import type { GuideData, GuideSerializedData } from '../types/guide'
export class Guide {
channelId?: string
feedId?: string
siteDomain: string
siteId: string
siteName: string
languageCode: string
constructor(data?: GuideData) {
if (!data) return
this.channelId = data.channel
this.feedId = data.feed
this.siteDomain = data.site
this.siteId = data.site_id
this.siteName = data.site_name
this.languageCode = data.lang
}
getUUID(): string {
return this.getStreamId() + this.siteId
}
getStreamId(): string | undefined {
if (!this.channelId) return undefined
if (!this.feedId) return this.channelId
return `${this.channelId}@${this.feedId}`
}
serialize(): GuideSerializedData {
return {
channelId: this.channelId,
feedId: this.feedId,
siteDomain: this.siteDomain,
siteId: this.siteId,
siteName: this.siteName,
languageCode: this.languageCode
}
}
deserialize(data: GuideSerializedData): this {
this.channelId = data.channelId
this.feedId = data.feedId
this.siteDomain = data.siteDomain
this.siteId = data.siteId
this.siteName = data.siteName
this.languageCode = data.languageCode
return this
}
}

@ -1,16 +1,3 @@
export * from './blocklistRecord'
export * from './broadcastArea'
export * from './category'
export * from './channel'
export * from './city'
export * from './country'
export * from './feed'
export * from './guide'
export * from './issue'
export * from './language'
export * from './logo'
export * from './playlist'
export * from './region'
export * from './stream'
export * from './subdivision'
export * from './timezone'

@ -1,27 +0,0 @@
import type { LanguageData, LanguageSerializedData } from '../types/language'
export class Language {
code: string
name: string
constructor(data?: LanguageData) {
if (!data) return
this.code = data.code
this.name = data.name
}
serialize(): LanguageSerializedData {
return {
code: this.code,
name: this.name
}
}
deserialize(data: LanguageSerializedData): this {
this.code = data.code
this.name = data.name
return this
}
}

@ -1,40 +0,0 @@
import { Collection, type Dictionary } from '@freearhey/core'
import type { LogoData } from '../types/logo'
import { type Feed } from './feed'
export class Logo {
channelId: string
feedId?: string
feed: Feed
tags: Collection
width: number
height: number
format?: string
url: string
constructor(data?: LogoData) {
if (!data) return
this.channelId = data.channel
this.feedId = data.feed || undefined
this.tags = new Collection(data.tags)
this.width = data.width
this.height = data.height
this.format = data.format || undefined
this.url = data.url
}
withFeed(feedsKeyById: Dictionary): this {
if (!this.feedId) return this
this.feed = feedsKeyById.get(this.feedId)
return this
}
getStreamId(): string {
if (!this.feedId) return this.channelId
return `${this.channelId}@${this.feedId}`
}
}

@ -6,12 +6,12 @@ type PlaylistOptions = {
}
export class Playlist {
streams: Collection
streams: Collection<Stream>
options: {
public: boolean
}
constructor(streams: Collection, options?: PlaylistOptions) {
constructor(streams: Collection<Stream>, options?: PlaylistOptions) {
this.streams = streams
this.options = options || { public: false }
}

@ -1,118 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { City, Country, Subdivision } from '.'
import type { RegionData, RegionSerializedData } from '../types/region'
import { CountrySerializedData } from '../types/country'
import { SubdivisionSerializedData } from '../types/subdivision'
import { CitySerializedData } from '../types/city'
export class Region {
code: string
name: string
countryCodes: Collection
countries?: Collection
subdivisions?: Collection
cities?: Collection
regions?: Collection
constructor(data?: RegionData) {
if (!data) return
this.code = data.code
this.name = data.name
this.countryCodes = new Collection(data.countries)
}
withCountries(countriesKeyByCode: Dictionary): this {
this.countries = this.countryCodes.map((code: string) => countriesKeyByCode.get(code))
return this
}
withSubdivisions(subdivisions: Collection): this {
this.subdivisions = subdivisions.filter(
(subdivision: Subdivision) => this.countryCodes.indexOf(subdivision.countryCode) > -1
)
return this
}
withCities(cities: Collection): this {
this.cities = cities.filter((city: City) => this.countryCodes.indexOf(city.countryCode) > -1)
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter(
(region: Region) => !region.countryCodes.intersects(this.countryCodes).isEmpty()
)
return this
}
getSubdivisions(): Collection {
if (!this.subdivisions) return new Collection()
return this.subdivisions
}
getCountries(): Collection {
if (!this.countries) return new Collection()
return this.countries
}
getCities(): Collection {
if (!this.cities) return new Collection()
return this.cities
}
getRegions(): Collection {
if (!this.regions) return new Collection()
return this.regions
}
includesCountryCode(code: string): boolean {
return this.countryCodes.includes((countryCode: string) => countryCode === code)
}
isWorldwide(): boolean {
return ['INT', 'WW'].includes(this.code)
}
serialize(): RegionSerializedData {
return {
code: this.code,
name: this.name,
countryCodes: this.countryCodes.all(),
countries: this.getCountries()
.map((country: Country) => country.serialize())
.all(),
subdivisions: this.getSubdivisions()
.map((subdivision: Subdivision) => subdivision.serialize())
.all(),
cities: this.getCities()
.map((city: City) => city.serialize())
.all()
}
}
deserialize(data: RegionSerializedData): this {
this.code = data.code
this.name = data.name
this.countryCodes = new Collection(data.countryCodes)
this.countries = new Collection(data.countries).map((data: CountrySerializedData) =>
new Country().deserialize(data)
)
this.subdivisions = new Collection(data.subdivisions).map((data: SubdivisionSerializedData) =>
new Subdivision().deserialize(data)
)
this.cities = new Collection(data.cities).map((data: CitySerializedData) =>
new City().deserialize(data)
)
return this
}
}

@ -1,61 +1,21 @@
import {
Feed,
Channel,
Category,
Region,
Subdivision,
Country,
Language,
Logo,
City
} from './index'
import { URL, Collection, Dictionary } from '@freearhey/core'
import type { StreamData } from '../types/stream'
import { Collection } from '@freearhey/core'
import parser from 'iptv-playlist-parser'
import { normalizeURL } from '../utils'
import * as sdk from '@iptv-org/sdk'
import { IssueData } from '../core'
import { data } from '../api'
import path from 'node:path'
export class Stream {
title: string
url: string
id?: string
channelId?: string
channel?: Channel
feedId?: string
feed?: Feed
logos: Collection = new Collection()
export class Stream extends sdk.Models.Stream {
directives: Collection<string>
filepath?: string
line?: number
label?: string
verticalResolution?: number
isInterlaced?: boolean
referrer?: string
userAgent?: string
groupTitle: string = 'Undefined'
removed: boolean = false
directives: Collection = new Collection()
constructor(data?: StreamData) {
if (!data) return
const id =
data.channelId && data.feedId ? [data.channelId, data.feedId].join('@') : data.channelId
const { verticalResolution, isInterlaced } = parseQuality(data.quality)
this.id = id || undefined
this.channelId = data.channelId || undefined
this.feedId = data.feedId || undefined
this.title = data.title || ''
this.url = data.url
this.referrer = data.referrer || undefined
this.userAgent = data.userAgent || undefined
this.verticalResolution = verticalResolution || undefined
this.isInterlaced = isInterlaced || undefined
this.label = data.label || undefined
this.directives = new Collection(data.directives)
}
tvgId?: string
label: string | null
update(issueData: IssueData): this {
updateWithIssue(issueData: IssueData): this {
const data = {
label: issueData.getString('label'),
quality: issueData.getString('quality'),
@ -66,16 +26,20 @@ export class Stream {
}
if (data.label !== undefined) this.label = data.label
if (data.quality !== undefined) this.setQuality(data.quality)
if (data.httpUserAgent !== undefined) this.userAgent = data.httpUserAgent
if (data.quality !== undefined) this.quality = data.quality
if (data.httpUserAgent !== undefined) this.user_agent = data.httpUserAgent
if (data.httpReferrer !== undefined) this.referrer = data.httpReferrer
if (data.newStreamUrl !== undefined) this.url = data.newStreamUrl
if (data.directives !== undefined) this.directives = new Collection(data.directives)
if (data.directives !== undefined) this.setDirectives(data.directives)
return this
}
fromPlaylistItem(data: parser.PlaylistItem): this {
static fromPlaylistItem(data: parser.PlaylistItem): Stream {
function escapeRegExp(text) {
return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
}
function parseName(name: string): {
title: string
label: string
@ -84,14 +48,14 @@ export class Stream {
let title = name
const [, label] = title.match(/ \[(.*)\]$/) || [null, '']
title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '')
const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, '']
const [, quality] = title.match(/ \(([0-9]+[p|i])\)$/) || [null, '']
title = title.replace(new RegExp(` \\(${quality}\\)$`), '')
return { title, label, quality }
}
function parseDirectives(string: string) {
const directives = new Collection()
function parseDirectives(string: string): Collection<string> {
const directives = new Collection<string>()
if (!string) return directives
@ -113,293 +77,329 @@ export class Stream {
const [channelId, feedId] = data.tvg.id.split('@')
const { title, label, quality } = parseName(data.name)
const { verticalResolution, isInterlaced } = parseQuality(quality)
this.id = data.tvg.id || undefined
this.feedId = feedId || undefined
this.channelId = channelId || undefined
this.line = data.line
this.label = label || undefined
this.title = title
this.verticalResolution = verticalResolution || undefined
this.isInterlaced = isInterlaced || undefined
this.url = data.url
this.referrer = data.http.referrer || undefined
this.userAgent = data.http['user-agent'] || undefined
this.directives = parseDirectives(data.raw)
return this
}
withChannel(channelsKeyById: Dictionary): this {
if (!this.channelId) return this
const stream = new Stream({
channel: channelId || null,
feed: feedId || null,
title: title,
quality: quality || null,
url: data.url,
referrer: data.http.referrer || null,
user_agent: data.http['user-agent'] || null
})
this.channel = channelsKeyById.get(this.channelId)
stream.tvgId = data.tvg.id
stream.line = data.line
stream.label = label || null
stream.directives = parseDirectives(data.raw)
return this
return stream
}
withFeed(feedsGroupedByChannelId: Dictionary): this {
if (!this.channelId) return this
const channelFeeds = feedsGroupedByChannelId.get(this.channelId) || []
if (this.feedId) this.feed = channelFeeds.find((feed: Feed) => feed.id === this.feedId)
if (!this.feedId && !this.feed) this.feed = channelFeeds.find((feed: Feed) => feed.isMain)
return this
}
isSFW(): boolean {
const channel = this.getChannel()
withLogos(logosGroupedByStreamId: Dictionary): this {
if (this.id) this.logos = new Collection(logosGroupedByStreamId.get(this.id))
if (!channel) return true
return this
return !channel.is_nsfw
}
setId(id: string): this {
this.id = id
getUniqKey(): string {
const filepath = this.getFilepath()
const tvgId = this.getTvgId()
return this
return filepath + tvgId + this.url
}
setChannelId(channelId: string): this {
this.channelId = channelId
return this
}
getVerticalResolution(): number {
if (!this.quality) return 0
setFeedId(feedId: string | undefined): this {
this.feedId = feedId
const [, verticalResolutionString] = this.quality.match(/^(\d+)/) || ['', '0']
return this
return parseInt(verticalResolutionString)
}
setQuality(quality: string): this {
const { verticalResolution, isInterlaced } = parseQuality(quality)
getBroadcastCountries(): Collection<sdk.Models.Country> {
const countries = new Collection<sdk.Models.Country>()
this.verticalResolution = verticalResolution || undefined
this.isInterlaced = isInterlaced || undefined
const feed = this.getFeed()
if (!feed) return countries
return this
feed
.getBroadcastArea()
.getLocations()
.forEach((location: sdk.Models.BroadcastAreaLocation) => {
let country: sdk.Models.Country | undefined
switch (location.type) {
case 'country': {
country = data.countriesKeyByCode.get(location.code)
break
}
case 'subdivision': {
const subdivision = data.subdivisionsKeyByCode.get(location.code)
if (!subdivision) break
country = data.countriesKeyByCode.get(subdivision.country)
break
}
case 'city': {
const city = data.citiesKeyByCode.get(location.code)
if (!city) break
country = data.countriesKeyByCode.get(city.country)
break
}
getLine(): number {
return this.line || -1
}
getFilename(): string {
if (!this.filepath) return ''
if (country) countries.add(country)
})
return path.basename(this.filepath)
return countries.uniqBy((country: sdk.Models.Country) => country.code)
}
getBroadcastSubdivisions(): Collection<sdk.Models.Subdivision> {
const subdivisions = new Collection<sdk.Models.Subdivision>()
const feed = this.getFeed()
if (!feed) return subdivisions
feed
.getBroadcastArea()
.getLocations()
.forEach((location: sdk.Models.BroadcastAreaLocation) => {
switch (location.type) {
case 'subdivision': {
const subdivision = data.subdivisionsKeyByCode.get(location.code)
if (!subdivision) break
subdivisions.add(subdivision)
if (!subdivision.parent) break
const parentSubdivision = data.subdivisionsKeyByCode.get(subdivision.parent)
if (!parentSubdivision) break
subdivisions.add(parentSubdivision)
break
}
case 'city': {
const city = data.citiesKeyByCode.get(location.code)
if (!city || !city.subdivision) break
const subdivision = data.subdivisionsKeyByCode.get(city.subdivision)
if (!subdivision) break
subdivisions.add(subdivision)
if (!subdivision.parent) break
const parentSubdivision = data.subdivisionsKeyByCode.get(subdivision.parent)
if (!parentSubdivision) break
subdivisions.add(parentSubdivision)
break
}
}
})
setFilepath(filepath: string): this {
this.filepath = filepath
return this
return subdivisions.uniqBy((subdivision: sdk.Models.Subdivision) => subdivision.code)
}
updateFilepath(): this {
if (!this.channel) return this
getBroadcastCities(): Collection<sdk.Models.City> {
const cities = new Collection<sdk.Models.City>()
this.filepath = `${this.channel.countryCode.toLowerCase()}.m3u`
const feed = this.getFeed()
if (!feed) return cities
return this
}
feed
.getBroadcastArea()
.getLocations()
.forEach((location: sdk.Models.BroadcastAreaLocation) => {
if (location.type !== 'city') return
getChannelId(): string {
return this.channelId || ''
}
const city = data.citiesKeyByCode.get(location.code)
getFeedId(): string {
if (this.feedId) return this.feedId
if (this.feed) return this.feed.id
return ''
}
if (city) cities.add(city)
})
getFilepath(): string {
return this.filepath || ''
return cities.uniqBy((city: sdk.Models.City) => city.code)
}
getBroadcastRegions(): Collection<sdk.Models.Region> {
const regions = new Collection<sdk.Models.Region>()
const feed = this.getFeed()
if (!feed) return regions
feed
.getBroadcastArea()
.getLocations()
.forEach((location: sdk.Models.BroadcastAreaLocation) => {
switch (location.type) {
case 'region': {
const region = data.regionsKeyByCode.get(location.code)
if (!region) break
regions.add(region)
const relatedRegions = data.regions.filter((_region: sdk.Models.Region) =>
new Collection<string>(_region.countries)
.intersects(new Collection<string>(region.countries))
.isNotEmpty()
)
regions.concat(relatedRegions)
break
}
case 'country': {
const country = data.countriesKeyByCode.get(location.code)
if (!country) break
const countryRegions = data.regions.filter((_region: sdk.Models.Region) =>
new Collection<string>(_region.countries).includes(
(code: string) => code === country.code
)
)
regions.concat(countryRegions)
break
}
case 'subdivision': {
const subdivision = data.subdivisionsKeyByCode.get(location.code)
if (!subdivision) break
const subdivisionRegions = data.regions.filter((_region: sdk.Models.Region) =>
new Collection<string>(_region.countries).includes(
(code: string) => code === subdivision.country
)
)
regions.concat(subdivisionRegions)
break
}
case 'city': {
const city = data.citiesKeyByCode.get(location.code)
if (!city) break
const cityRegions = data.regions.filter((_region: sdk.Models.Region) =>
new Collection<string>(_region.countries).includes(
(code: string) => code === city.country
)
)
regions.concat(cityRegions)
break
}
getReferrer(): string {
return this.referrer || ''
}
})
getUserAgent(): string {
return this.userAgent || ''
return regions.uniqBy((region: sdk.Models.Region) => region.code)
}
getQuality(): string {
if (!this.verticalResolution) return ''
let quality = this.verticalResolution.toString()
isInternational(): boolean {
const feed = this.getFeed()
if (!feed) return false
if (this.isInterlaced) quality += 'i'
else quality += 'p'
const broadcastAreaCodes = feed.getBroadcastArea().codes
if (broadcastAreaCodes.join(';').includes('r/')) return true
if (broadcastAreaCodes.filter(code => code.includes('c/')).length > 1) return true
return quality
return false
}
hasId(): boolean {
return !!this.id
}
hasCategory(category: sdk.Models.Category): boolean {
const channel = this.getChannel()
hasQuality(): boolean {
return !!this.verticalResolution
}
if (!channel) return false
getVerticalResolution(): number {
if (!this.hasQuality()) return 0
const found = channel.categories.find((id: string) => id === category.id)
return parseInt(this.getQuality().replace(/p|i/, ''))
return !!found
}
updateTitle(): this {
if (!this.channel) return this
hasLanguage(language: sdk.Models.Language): boolean {
const found = this.getLanguages().find(
(_language: sdk.Models.Language) => _language.code === language.code
)
this.title = this.channel.name
if (this.feed && !this.feed.isMain) {
this.title += ` ${this.feed.name}`
return !!found
}
setDirectives(directives: string[]): this {
this.directives = new Collection(directives).filter((directive: string) =>
/^(#KODIPROP|#EXTVLCOPT)/.test(directive)
)
return this
}
updateId(): this {
updateTvgId(): this {
if (!this.channel) return this
if (this.feed) {
this.id = `${this.channel.id}@${this.feed.id}`
this.tvgId = `${this.channel}@${this.feed}`
} else {
this.id = this.channel.id
this.tvgId = this.channel
}
return this
}
normalizeURL() {
const url = new URL(this.url)
this.url = url.normalize().toString()
}
clone(): Stream {
return Object.assign(Object.create(Object.getPrototypeOf(this)), this)
}
hasChannel() {
return !!this.channel
}
getBroadcastRegions(): Collection {
return this.feed ? this.feed.getBroadcastRegions() : new Collection()
}
getBroadcastCountries(): Collection {
return this.feed ? this.feed.getBroadcastCountries() : new Collection()
}
hasBroadcastArea(): boolean {
return this.feed ? this.feed.hasBroadcastArea() : false
}
isSFW(): boolean {
return this.channel ? this.channel.isSFW() : true
}
hasCategories(): boolean {
return this.channel ? this.channel.hasCategories() : false
}
hasCategory(category: Category): boolean {
return this.channel ? this.channel.hasCategory(category) : false
}
updateFilepath(): this {
const channel = this.getChannel()
if (!channel) return this
getCategoryNames(): string[] {
return this.getCategories()
.map((category: Category) => category.name)
.sort()
.all()
}
this.filepath = `${channel.country.toLowerCase()}.m3u`
getCategories(): Collection {
return this.channel ? this.channel.getCategories() : new Collection()
return this
}
getLanguages(): Collection {
return this.feed ? this.feed.getLanguages() : new Collection()
}
updateTitle(): this {
const channel = this.getChannel()
hasLanguages() {
return this.feed ? this.feed.hasLanguages() : false
}
if (!channel) return this
hasLanguage(language: Language) {
return this.feed ? this.feed.hasLanguage(language) : false
}
const feed = this.getFeed()
getBroadcastAreaCodes(): Collection {
return this.feed ? this.feed.broadcastAreaCodes : new Collection()
this.title = channel.name
if (feed && !feed.is_main) {
this.title += ` ${feed.name}`
}
isBroadcastInCity(city: City): boolean {
return this.feed ? this.feed.isBroadcastInCity(city) : false
}
isBroadcastInSubdivision(subdivision: Subdivision): boolean {
return this.feed ? this.feed.isBroadcastInSubdivision(subdivision) : false
return this
}
isBroadcastInCountry(country: Country): boolean {
return this.feed ? this.feed.isBroadcastInCountry(country) : false
normalizeURL() {
this.url = normalizeURL(this.url)
}
isBroadcastInRegion(region: Region): boolean {
return this.feed ? this.feed.isBroadcastInRegion(region) : false
}
getLogos(): Collection<sdk.Models.Logo> {
const logos = super.getLogos()
isInternational(): boolean {
return this.feed ? this.feed.isInternational() : false
}
if (logos.isEmpty()) return new Collection()
getLogos(): Collection {
function format(logo: Logo): number {
function format(logo: sdk.Models.Logo): number {
const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 }
return logo.format ? levelByFormat[logo.format] : 0
}
function size(logo: Logo): number {
function size(logo: sdk.Models.Logo): number {
return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
}
return this.logos.orderBy([format, size], ['desc', 'asc'], false)
return logos.sortBy([format, size], ['desc', 'asc'], false)
}
getLogo(): Logo | undefined {
return this.getLogos().first()
getFilepath(): string {
return this.filepath || ''
}
hasLogo(): boolean {
return this.getLogos().notEmpty()
getFilename(): string {
return path.basename(this.getFilepath())
}
getLogoUrl(): string {
let logo: Logo | undefined
getLine(): number {
return this.line || -1
}
if (this.hasLogo()) logo = this.getLogo()
else logo = this?.channel?.getLogo()
getTvgId(): string {
if (this.tvgId) return this.tvgId
return logo ? logo.url : ''
return this.getId()
}
getTitle(): string {
return this.title || ''
getTvgLogo(): string {
const logo = this.getLogos().first()
return logo ? logo.url : ''
}
getFullTitle(): string {
let title = `${this.getTitle()}`
let title = `${this.title}`
if (this.getQuality()) {
title += ` (${this.getQuality()})`
if (this.quality) {
title += ` (${this.quality})`
}
if (this.label) {
@ -409,39 +409,21 @@ export class Stream {
return title
}
getLabel(): string {
return this.label || ''
}
getId(): string {
return this.id || ''
}
toJSON() {
return {
channel: this.channelId || null,
feed: this.feedId || null,
title: this.title,
url: this.url,
referrer: this.referrer || null,
user_agent: this.userAgent || null,
quality: this.getQuality() || null
}
}
toString(options: { public?: boolean } = {}) {
options = { ...{ public: false }, ...options }
toString(options: { public: boolean }) {
let output = `#EXTINF:-1 tvg-id="${this.getId()}"`
let output = `#EXTINF:-1 tvg-id="${this.getTvgId()}"`
if (options.public) {
output += ` tvg-logo="${this.getLogoUrl()}" group-title="${this.groupTitle}"`
output += ` tvg-logo="${this.getTvgLogo()}" group-title="${this.groupTitle}"`
}
if (this.referrer) {
output += ` http-referrer="${this.referrer}"`
}
if (this.userAgent) {
output += ` http-user-agent="${this.userAgent}"`
if (this.user_agent) {
output += ` http-user-agent="${this.user_agent}"`
}
output += `,${this.getFullTitle()}`
@ -454,21 +436,26 @@ export class Stream {
return output
}
}
function escapeRegExp(text) {
return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
}
toObject(): sdk.Types.StreamData {
let feedId = this.feed
if (!feedId) {
const feed = this.getFeed()
if (feed) feedId = feed.id
}
function parseQuality(quality: string | null): {
verticalResolution: number | null
isInterlaced: boolean | null
} {
if (!quality) return { verticalResolution: null, isInterlaced: null }
const [, verticalResolutionString] = quality.match(/^(\d+)/) || [null, undefined]
const isInterlaced = /i$/i.test(quality)
let verticalResolution = 0
if (verticalResolutionString) verticalResolution = parseInt(verticalResolutionString)
return { verticalResolution, isInterlaced }
return {
channel: this.channel,
feed: feedId,
title: this.title,
url: this.url,
quality: this.quality,
user_agent: this.user_agent,
referrer: this.referrer
}
}
clone(): Stream {
return Object.assign(Object.create(Object.getPrototypeOf(this)), this)
}
}

@ -1,83 +0,0 @@
import { SubdivisionData, SubdivisionSerializedData } from '../types/subdivision'
import { Dictionary, Collection } from '@freearhey/core'
import { Country, Region } from '.'
export class Subdivision {
code: string
name: string
countryCode: string
country?: Country
parentCode?: string
parent?: Subdivision
regions?: Collection
cities?: Collection
constructor(data?: SubdivisionData) {
if (!data) return
this.code = data.code
this.name = data.name
this.countryCode = data.country
this.parentCode = data.parent || undefined
}
withCountry(countriesKeyByCode: Dictionary): this {
this.country = countriesKeyByCode.get(this.countryCode)
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter((region: Region) =>
region.countryCodes.includes(this.countryCode)
)
return this
}
withCities(citiesGroupedBySubdivisionCode: Dictionary): this {
this.cities = new Collection(citiesGroupedBySubdivisionCode.get(this.code))
return this
}
withParent(subdivisionsKeyByCode: Dictionary): this {
if (!this.parentCode) return this
this.parent = subdivisionsKeyByCode.get(this.parentCode)
return this
}
getRegions(): Collection {
if (!this.regions) return new Collection()
return this.regions
}
getCities(): Collection {
if (!this.cities) return new Collection()
return this.cities
}
serialize(): SubdivisionSerializedData {
return {
code: this.code,
name: this.name,
countryCode: this.countryCode,
country: this.country ? this.country.serialize() : undefined,
parentCode: this.parentCode || null
}
}
deserialize(data: SubdivisionSerializedData): this {
this.code = data.code
this.name = data.name
this.countryCode = data.countryCode
this.country = data.country ? new Country().deserialize(data.country) : undefined
this.parentCode = data.parentCode || undefined
return this
}
}

@ -1,30 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
type TimezoneData = {
id: string
utc_offset: string
countries: string[]
}
export class Timezone {
id: string
utcOffset: string
countryCodes: Collection
countries?: Collection
constructor(data: TimezoneData) {
this.id = data.id
this.utcOffset = data.utc_offset
this.countryCodes = new Collection(data.countries)
}
withCountries(countriesKeyByCode: Dictionary): this {
this.countries = this.countryCodes.map((code: string) => countriesKeyByCode.get(code))
return this
}
getCountries(): Collection {
return this.countries || new Collection()
}
}

@ -1,55 +1,62 @@
import { Storage, Collection, File, Dictionary } from '@freearhey/core'
import { HTMLTable, LogParser, LogItem } from '../core'
import { HTMLTable, HTMLTableItem, LogParser, LogItem, HTMLTableColumn } from '../core'
import { Storage, File } from '@freearhey/storage-js'
import { LOGS_DIR, README_DIR } from '../constants'
import { Category } from '../models'
import { Collection } from '@freearhey/core'
import * as sdk from '@iptv-org/sdk'
import { Table } from './table'
type CategoriesTableProps = {
categoriesKeyById: Dictionary
}
import { data } from '../api'
export class CategoriesTable implements Table {
categoriesKeyById: Dictionary
constructor({ categoriesKeyById }: CategoriesTableProps) {
this.categoriesKeyById = categoriesKeyById
}
async make() {
async create() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
let items = new Collection()
let items = new Collection<HTMLTableItem>()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.type === 'category')
.forEach((logItem: LogItem) => {
if (logItem.filepath.includes('undefined')) {
items.add([
'ZZ',
'Undefined',
logItem.count.toString(),
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
return
}
const file = new File(logItem.filepath)
const categoryId = file.name()
const category: Category = this.categoriesKeyById.get(categoryId)
const category: sdk.Models.Category | undefined = data.categoriesKeyById.get(categoryId)
if (!category) return
items.add([
category ? category.name : 'ZZ',
category ? category.name : 'Undefined',
logItem.count,
category.name,
category.name,
logItem.count.toString(),
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
})
items = items
.orderBy(item => item[0])
.sortBy(item => item[0])
.map(item => {
item.shift()
return item
})
const table = new HTMLTable(items.all(), [
const columns = new Collection<HTMLTableColumn>([
{ name: 'Category' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', nowrap: true }
])
const table = new HTMLTable(items, columns)
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_categories.md', table.toString())
}

@ -1,39 +1,21 @@
import { Storage, Collection, Dictionary } from '@freearhey/core'
import { City, Country, Subdivision } from '../models'
import { LOGS_DIR, README_DIR } from '../constants'
import { Storage } from '@freearhey/storage-js'
import { Collection } from '@freearhey/core'
import { LogParser, LogItem } from '../core'
import * as sdk from '@iptv-org/sdk'
import { Table } from './table'
type CountriesTableProps = {
countriesKeyByCode: Dictionary
subdivisionsKeyByCode: Dictionary
countries: Collection
subdivisions: Collection
cities: Collection
import { data } from '../api'
type ListItem = {
index: string
count: number
link: string
name: string
children: Collection<ListItem>
}
export class CountriesTable implements Table {
countriesKeyByCode: Dictionary
subdivisionsKeyByCode: Dictionary
countries: Collection
subdivisions: Collection
cities: Collection
constructor({
countriesKeyByCode,
subdivisionsKeyByCode,
countries,
subdivisions,
cities
}: CountriesTableProps) {
this.countriesKeyByCode = countriesKeyByCode
this.subdivisionsKeyByCode = subdivisionsKeyByCode
this.countries = countries
this.subdivisions = subdivisions
this.cities = cities
}
async make() {
async create() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
@ -43,15 +25,16 @@ export class CountriesTable implements Table {
const logCities = parsed.filter((logItem: LogItem) => logItem.type === 'city')
let items = new Collection()
this.countries.forEach((country: Country) => {
data.countries.forEach((country: sdk.Models.Country) => {
const countryCode = country.code
const countriesLogItem = logCountries.find(
(logItem: LogItem) => logItem.filepath === `countries/${country.code.toLowerCase()}.m3u`
(logItem: LogItem) => logItem.filepath === `countries/${countryCode.toLowerCase()}.m3u`
)
const countryItem = {
const countryItem: ListItem = {
index: country.name,
count: 0,
link: `https://iptv-org.github.io/iptv/countries/${country.code.toLowerCase()}.m3u`,
link: `https://iptv-org.github.io/iptv/countries/${countryCode.toLowerCase()}.m3u`,
name: `${country.flag} ${country.name}`,
children: new Collection()
}
@ -60,38 +43,41 @@ export class CountriesTable implements Table {
countryItem.count = countriesLogItem.count
}
const countrySubdivisions = this.subdivisions.filter(
(subdivision: Subdivision) => subdivision.countryCode === country.code
const countrySubdivisions = data.subdivisions.filter(
(subdivision: sdk.Models.Subdivision) => subdivision.country === countryCode
)
const countryCities = data.cities.filter(
(city: sdk.Models.City) => city.country === countryCode
)
const countryCities = this.cities.filter((city: City) => city.countryCode === country.code)
if (countrySubdivisions.notEmpty()) {
this.subdivisions.forEach((subdivision: Subdivision) => {
if (subdivision.countryCode !== country.code) return
if (countrySubdivisions.isNotEmpty()) {
data.subdivisions.forEach((subdivision: sdk.Models.Subdivision) => {
if (subdivision.country !== countryCode) return
const subdivisionCode = subdivision.code
const subdivisionCities = countryCities.filter(
(city: City) =>
(city.subdivisionCode && city.subdivisionCode === subdivision.code) ||
city.countryCode === subdivision.countryCode
(city: sdk.Models.City) =>
(city.subdivision && city.subdivision === subdivisionCode) ||
city.country === subdivision.country
)
const subdivisionsLogItem = logSubdivisions.find(
(logItem: LogItem) =>
logItem.filepath === `subdivisions/${subdivision.code.toLowerCase()}.m3u`
logItem.filepath === `subdivisions/${subdivisionCode.toLowerCase()}.m3u`
)
const subdivisionItem = {
const subdivisionItem: ListItem = {
index: subdivision.name,
name: subdivision.name,
count: 0,
link: `https://iptv-org.github.io/iptv/subdivisions/${subdivision.code.toLowerCase()}.m3u`,
children: new Collection()
link: `https://iptv-org.github.io/iptv/subdivisions/${subdivisionCode.toLowerCase()}.m3u`,
children: new Collection<ListItem>()
}
if (subdivisionsLogItem) {
subdivisionItem.count = subdivisionsLogItem.count
}
subdivisionCities.forEach((city: City) => {
if (city.countryCode !== country.code || city.subdivisionCode !== subdivision.code)
return
subdivisionCities.forEach((city: sdk.Models.City) => {
if (city.country !== countryCode || city.subdivision !== subdivisionCode) return
const citiesLogItem = logCities.find(
(logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u`
)
@ -102,16 +88,17 @@ export class CountriesTable implements Table {
index: city.name,
name: city.name,
count: citiesLogItem.count,
link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`
link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`,
children: new Collection<ListItem>()
})
})
if (subdivisionItem.count > 0 || subdivisionItem.children.notEmpty()) {
if (subdivisionItem.count > 0 || subdivisionItem.children.isNotEmpty()) {
countryItem.children.add(subdivisionItem)
}
})
} else if (countryCities.notEmpty()) {
countryCities.forEach((city: City) => {
} else if (countryCities.isNotEmpty()) {
countryCities.forEach((city: sdk.Models.City) => {
const citiesLogItem = logCities.find(
(logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u`
)
@ -128,7 +115,7 @@ export class CountriesTable implements Table {
})
}
if (countryItem.count > 0 || countryItem.children.notEmpty()) {
if (countryItem.count > 0 || countryItem.children.isNotEmpty()) {
items.add(countryItem)
}
})
@ -138,7 +125,7 @@ export class CountriesTable implements Table {
)
if (internationalLogItem) {
items.push({
items.add({
index: 'ZZ',
name: '🌐 International',
count: internationalLogItem.count,
@ -152,7 +139,7 @@ export class CountriesTable implements Table {
)
if (undefinedLogItem) {
items.push({
items.add({
index: 'ZZZ',
name: 'Undefined',
count: undefinedLogItem.count,
@ -161,20 +148,20 @@ export class CountriesTable implements Table {
})
}
items = items.orderBy(item => item.index)
items = items.sortBy(item => item.index)
const output = items
.map(item => {
.map((item: ListItem) => {
let row = `- ${item.name} <code>${item.link}</code>`
item.children
.orderBy(item => item.index)
.forEach(item => {
.sortBy((item: ListItem) => item.index)
.forEach((item: ListItem) => {
row += `\r\n - ${item.name} <code>${item.link}</code>`
item.children
.orderBy(item => item.index)
.forEach(item => {
.sortBy((item: ListItem) => item.index)
.forEach((item: ListItem) => {
row += `\r\n - ${item.name} <code>${item.link}</code>`
})
})

@ -1,55 +1,62 @@
import { Storage, Collection, File, Dictionary } from '@freearhey/core'
import { HTMLTable, LogParser, LogItem } from '../core'
import { HTMLTable, LogParser, LogItem, HTMLTableColumn, HTMLTableItem } from '../core'
import { Storage, File } from '@freearhey/storage-js'
import { LOGS_DIR, README_DIR } from '../constants'
import { Language } from '../models'
import { Collection } from '@freearhey/core'
import * as sdk from '@iptv-org/sdk'
import { Table } from './table'
type LanguagesTableProps = {
languagesKeyByCode: Dictionary
}
import { data } from '../api'
export class LanguagesTable implements Table {
languagesKeyByCode: Dictionary
constructor({ languagesKeyByCode }: LanguagesTableProps) {
this.languagesKeyByCode = languagesKeyByCode
}
async make() {
async create() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
let data = new Collection()
let items = new Collection<HTMLTableItem>()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.type === 'language')
.forEach((logItem: LogItem) => {
if (logItem.filepath.includes('undefined')) {
items.add([
'ZZ',
'Undefined',
logItem.count.toString(),
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
return
}
const file = new File(logItem.filepath)
const languageCode = file.name()
const language: Language = this.languagesKeyByCode.get(languageCode)
const language: sdk.Models.Language | undefined = data.languagesKeyByCode.get(languageCode)
if (!language) return
data.add([
language ? language.name : 'ZZ',
language ? language.name : 'Undefined',
logItem.count,
items.add([
language.name,
language.name,
logItem.count.toString(),
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
})
data = data
.orderBy(item => item[0])
items = items
.sortBy(item => item[0])
.map(item => {
item.shift()
return item
})
const table = new HTMLTable(data.all(), [
const columns = new Collection<HTMLTableColumn>([
{ name: 'Language', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left', nowrap: true }
])
const table = new HTMLTable(items, columns)
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_languages.md', table.toString())
}

@ -1,29 +1,27 @@
import { Storage, Collection } from '@freearhey/core'
import { LogParser, LogItem } from '../core'
import { LOGS_DIR, README_DIR } from '../constants'
import { Region } from '../models'
import { Storage } from '@freearhey/storage-js'
import { LogParser, LogItem } from '../core'
import { Collection } from '@freearhey/core'
import * as sdk from '@iptv-org/sdk'
import { Table } from './table'
import { data } from '../api'
type RegionsTableProps = {
regions: Collection
type ListItem = {
name: string
count: number
link: string
}
export class RegionsTable implements Table {
regions: Collection
constructor({ regions }: RegionsTableProps) {
this.regions = regions
}
async make() {
async create() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
const parsed = parser.parse(generatorsLog)
const logRegions = parsed.filter((logItem: LogItem) => logItem.type === 'region')
let items = new Collection()
this.regions.forEach((region: Region) => {
let items = new Collection<ListItem>()
data.regions.forEach((region: sdk.Models.Region) => {
const logItem = logRegions.find(
(logItem: LogItem) => logItem.filepath === `regions/${region.code.toLowerCase()}.m3u`
)
@ -31,14 +29,13 @@ export class RegionsTable implements Table {
if (!logItem) return
items.add({
index: region.name,
name: region.name,
count: logItem.count,
link: `https://iptv-org.github.io/iptv/${logItem.filepath}`
})
})
items = items.orderBy(item => item.index)
items = items.sortBy(item => item.name)
const output = items
.map(item => {

@ -1,3 +1,3 @@
export interface Table {
make(): void
create(): void
}

@ -1,5 +0,0 @@
export type BlocklistRecordData = {
channel: string
reason: string
ref: string
}

@ -1,9 +0,0 @@
export type CategorySerializedData = {
id: string
name: string
}
export type CategoryData = {
id: string
name: string
}

@ -1,50 +0,0 @@
import { Collection } from '@freearhey/core'
import type { CountrySerializedData } from './country'
import type { SubdivisionSerializedData } from './subdivision'
import type { CategorySerializedData } from './category'
export type ChannelSerializedData = {
id: string
name: string
altNames: string[]
network?: string
owners: string[]
countryCode: string
country?: CountrySerializedData
subdivisionCode?: string
subdivision?: SubdivisionSerializedData
cityName?: string
categoryIds: string[]
categories?: CategorySerializedData[]
isNSFW: boolean
launched?: string
closed?: string
replacedBy?: string
website?: string
}
export type ChannelData = {
id: string
name: string
alt_names: string[]
network: string
owners: Collection
country: string
subdivision: string
city: string
categories: Collection
is_nsfw: boolean
launched: string
closed: string
replaced_by: string
website: string
}
export type ChannelSearchableData = {
id: string
name: string
altNames: string[]
guideNames: string[]
streamTitles: string[]
feedFullNames: string[]
}

@ -1,20 +0,0 @@
import { CountrySerializedData } from './country'
import { SubdivisionSerializedData } from './subdivision'
export type CitySerializedData = {
code: string
name: string
countryCode: string
country?: CountrySerializedData
subdivisionCode: string | null
subdivision?: SubdivisionSerializedData
wikidataId: string
}
export type CityData = {
code: string
name: string
country: string
subdivision: string | null
wikidata_id: string
}

@ -1,20 +0,0 @@
import type { LanguageSerializedData } from './language'
import type { SubdivisionSerializedData } from './subdivision'
import type { RegionSerializedData } from './region'
export type CountrySerializedData = {
code: string
name: string
flag: string
languageCode: string
language: LanguageSerializedData | null
subdivisions: SubdivisionSerializedData[]
regions: RegionSerializedData[]
}
export type CountryData = {
code: string
name: string
lang: string
flag: string
}

@ -1,21 +0,0 @@
import { Storage } from '@freearhey/core'
export type DataLoaderProps = {
storage: Storage
}
export type DataLoaderData = {
countries: object | object[]
regions: object | object[]
subdivisions: object | object[]
languages: object | object[]
categories: object | object[]
blocklist: object | object[]
channels: object | object[]
feeds: object | object[]
logos: object | object[]
timezones: object | object[]
guides: object | object[]
streams: object | object[]
cities: object | object[]
}

@ -1,31 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
export type DataProcessorData = {
blocklistRecordsGroupedByChannelId: Dictionary
subdivisionsGroupedByCountryCode: Dictionary
feedsGroupedByChannelId: Dictionary
guidesGroupedByStreamId: Dictionary
logosGroupedByStreamId: Dictionary
subdivisionsKeyByCode: Dictionary
countriesKeyByCode: Dictionary
languagesKeyByCode: Dictionary
streamsGroupedById: Dictionary
categoriesKeyById: Dictionary
timezonesKeyById: Dictionary
regionsKeyByCode: Dictionary
blocklistRecords: Collection
channelsKeyById: Dictionary
citiesKeyByCode: Dictionary
subdivisions: Collection
categories: Collection
countries: Collection
languages: Collection
timezones: Collection
channels: Collection
regions: Collection
streams: Collection
cities: Collection
guides: Collection
feeds: Collection
logos: Collection
}

@ -1,10 +0,0 @@
export type FeedData = {
channel: string
id: string
name: string
is_main: boolean
broadcast_area: string[]
languages: string[]
timezones: string[]
video_format: string
}

@ -1,17 +0,0 @@
export type GuideSerializedData = {
channelId?: string
feedId?: string
siteDomain: string
siteId: string
siteName: string
languageCode: string
}
export type GuideData = {
channel: string
feed: string
site: string
site_id: string
site_name: string
lang: string
}

@ -1,9 +0,0 @@
export type LanguageSerializedData = {
code: string
name: string
}
export type LanguageData = {
code: string
name: string
}

@ -1,9 +0,0 @@
export type LogoData = {
channel: string
feed: string | null
tags: string[]
width: number
height: number
format: string | null
url: string
}

@ -1,18 +0,0 @@
import { CitySerializedData } from './city'
import { CountrySerializedData } from './country'
import { SubdivisionSerializedData } from './subdivision'
export type RegionSerializedData = {
code: string
name: string
countryCodes: string[]
countries?: CountrySerializedData[]
subdivisions?: SubdivisionSerializedData[]
cities?: CitySerializedData[]
}
export type RegionData = {
code: string
name: string
countries: string[]
}

@ -1,11 +0,0 @@
export type StreamData = {
channelId: string | null
feedId: string | null
title: string | null
url: string
referrer: string | null
userAgent: string | null
quality: string | null
label: string | null
directives: string[]
}

@ -1,16 +0,0 @@
import { CountrySerializedData } from './country'
export type SubdivisionSerializedData = {
code: string
name: string
countryCode: string
country?: CountrySerializedData
parentCode: string | null
}
export type SubdivisionData = {
code: string
name: string
country: string
parent: string | null
}

@ -1,3 +1,5 @@
import normalizeUrl from 'normalize-url'
export function isURI(string: string): boolean {
try {
new URL(string)
@ -6,3 +8,16 @@ export function isURI(string: string): boolean {
return false
}
}
export function normalizeURL(url: string): string {
const normalized = normalizeUrl(url, { stripWWW: false })
return decodeURIComponent(normalized).replace(/\s/g, '+').toString()
}
export function truncate(string: string, limit: number = 100) {
if (!string) return string
if (string.length < limit) return string
return string.slice(0, limit - 3) + '...'
}

Loading…
Cancel
Save