aiden
/
iptv
mirror of https://github.com/iptv-org/iptv
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update scripts

Browse Source
pull/28852/head
freearhey 1 month ago
parent 5887997bb3
commit adec9dd131
73 changed files with 3215 additions and 4784 deletions
Show Stats Download Patch File Download Diff File

151
scripts/api.ts
Unescape Escape View File

@ -0,0 +1,151 @@
import { Collection, Dictionary } from '@freearhey/core'
import { DATA_DIR } from './constants'
import cliProgress from 'cli-progress'
import * as sdk from '@iptv-org/sdk'
const data = {
categoriesKeyById: new Dictionary<sdk.Models.Category>(),
countriesKeyByCode: new Dictionary<sdk.Models.Country>(),
subdivisionsKeyByCode: new Dictionary<sdk.Models.Subdivision>(),
citiesKeyByCode: new Dictionary<sdk.Models.City>(),
regionsKeyByCode: new Dictionary<sdk.Models.Region>(),
languagesKeyByCode: new Dictionary<sdk.Models.Language>(),
channelsKeyById: new Dictionary<sdk.Models.Channel>(),
feedsKeyByStreamId: new Dictionary<sdk.Models.Feed>(),
feedsGroupedByChannel: new Dictionary<sdk.Models.Feed[]>(),
blocklistRecordsGroupedByChannel: new Dictionary<sdk.Models.BlocklistRecord[]>(),
categories: new Collection<sdk.Models.Category>(),
countries: new Collection<sdk.Models.Country>(),
subdivisions: new Collection<sdk.Models.Subdivision>(),
cities: new Collection<sdk.Models.City>(),
regions: new Collection<sdk.Models.Region>()
}
let searchIndex
async function loadData() {
const dataManager = new sdk.DataManager({ dataDir: DATA_DIR })
await dataManager.loadFromDisk()
dataManager.processData()
const {
channels,
feeds,
categories,
languages,
countries,
subdivisions,
cities,
regions,
blocklist
} = dataManager.getProcessedData()
searchIndex = sdk.SearchEngine.createIndex<sdk.Models.Channel>(channels)
data.categoriesKeyById = categories.keyBy((category: sdk.Models.Category) => category.id)
data.countriesKeyByCode = countries.keyBy((country: sdk.Models.Country) => country.code)
data.subdivisionsKeyByCode = subdivisions.keyBy(
(subdivision: sdk.Models.Subdivision) => subdivision.code
)
data.citiesKeyByCode = cities.keyBy((city: sdk.Models.City) => city.code)
data.regionsKeyByCode = regions.keyBy((region: sdk.Models.Region) => region.code)
data.languagesKeyByCode = languages.keyBy((language: sdk.Models.Language) => language.code)
data.channelsKeyById = channels.keyBy((channel: sdk.Models.Channel) => channel.id)
data.feedsKeyByStreamId = feeds.keyBy((feed: sdk.Models.Feed) => feed.getStreamId())
data.feedsGroupedByChannel = feeds.groupBy((feed: sdk.Models.Feed) => feed.channel)
data.blocklistRecordsGroupedByChannel = blocklist.groupBy(
(blocklistRecord: sdk.Models.BlocklistRecord) => blocklistRecord.channel
)
data.categories = categories
data.countries = countries
data.subdivisions = subdivisions
data.cities = cities
data.regions = regions
}
async function downloadData() {
function formatBytes(bytes: number) {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]
}
const files = [
'blocklist',
'categories',
'channels',
'cities',
'countries',
'feeds',
'guides',
'languages',
'logos',
'regions',
'streams',
'subdivisions',
'timezones'
]
const multiBar = new cliProgress.MultiBar({
stopOnComplete: true,
hideCursor: true,
forceRedraw: true,
barsize: 36,
format(options, params, payload) {
const filename = payload.filename.padEnd(18, ' ')
const barsize = options.barsize || 40
const percent = (params.progress * 100).toFixed(2)
const speed = payload.speed ? formatBytes(payload.speed) + '/s' : 'N/A'
const total = formatBytes(params.total)
const completeSize = Math.round(params.progress * barsize)
const incompleteSize = barsize - completeSize
const bar =
options.barCompleteString && options.barIncompleteString
? options.barCompleteString.substr(0, completeSize) +
options.barGlue +
options.barIncompleteString.substr(0, incompleteSize)
: '-'.repeat(barsize)
return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
}
})
const dataManager = new sdk.DataManager({ dataDir: DATA_DIR })
const requests: Promise<unknown>[] = []
for (const basename of files) {
const filename = `${basename}.json`
const progressBar = multiBar.create(0, 0, { filename })
const request = dataManager.downloadFileToDisk(basename, {
onDownloadProgress({ total, loaded, rate }) {
if (total) progressBar.setTotal(total)
progressBar.update(loaded, { speed: rate })
}
})
requests.push(request)
}
await Promise.allSettled(requests).catch(console.error)
}
function searchChannels(query: string): Collection<sdk.Models.Channel> {
if (!searchIndex) return new Collection<sdk.Models.Channel>()
const results = searchIndex.search(query)
const channels = new Collection<sdk.Models.Channel>()
new Collection<sdk.Types.ChannelSearchableData>(results).forEach(
(item: sdk.Types.ChannelSearchableData) => {
const channel = data.channelsKeyById.get(item.id)
if (channel) channels.add(channel)
}
)
return channels
}
export { data, loadData, downloadData, searchChannels }

70
scripts/commands/api/generate.ts
Unescape Escape View File

@ -1,39 +1,31 @@
import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
import type { DataProcessorData } from '../../types/dataProcessor'
import { API_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
import type { DataLoaderData } from '../../types/dataLoader'
import { Logger, Storage } from '@freearhey/core'
import { Stream } from '../../models'
async function main() {
const logger = new Logger()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const dataLoader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await dataLoader.load()
const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData =
processor.process(data)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
channelsKeyById,
logosGroupedByStreamId,
feedsGroupedByChannelId
})
const files = await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
streams = streams
.orderBy((stream: Stream) => stream.getId())
.map((stream: Stream) => stream.toJSON())
logger.info(`found ${streams.count()} streams`)
logger.info('saving to .api/streams.json...')
const apiStorage = new Storage(API_DIR)
await apiStorage.save('streams.json', streams.toJSON())
}
main()
import { API_DIR, STREAMS_DIR } from '../../constants'
import { Storage } from '@freearhey/storage-js'
import { PlaylistParser } from '../../core'
import { Logger } from '@freearhey/core'
import { Stream } from '../../models'
import { loadData } from '../../api'
async function main() {
const logger = new Logger()
logger.info('loading data from api...')
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage
})
const files = await streamsStorage.list('**/*.m3u')
const parsed = await parser.parse(files)
const _streams = parsed
.sortBy((stream: Stream) => stream.getId())
.map((stream: Stream) => stream.toObject())
logger.info(`found ${_streams.count()} streams`)
logger.info('saving to .api/streams.json...')
const apiStorage = new Storage(API_DIR)
await apiStorage.save('streams.json', _streams.toJSON())
}
main()

33
scripts/commands/api/load.ts
Unescape Escape View File

@ -1,26 +1,7 @@
import { DATA_DIR } from '../../constants'
import { Storage } from '@freearhey/core'
import { DataLoader } from '../../core'
async function main() {
const storage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage })
await Promise.all([
loader.download('blocklist.json'),
loader.download('categories.json'),
loader.download('channels.json'),
loader.download('countries.json'),
loader.download('languages.json'),
loader.download('regions.json'),
loader.download('subdivisions.json'),
loader.download('feeds.json'),
loader.download('logos.json'),
loader.download('timezones.json'),
loader.download('guides.json'),
loader.download('streams.json'),
loader.download('cities.json')
])
}
main()
import { downloadData } from '../../api'
async function main() {
await downloadData()
}
main()

407
scripts/commands/playlist/edit.ts
Unescape Escape View File

@ -1,217 +1,190 @@
import { Storage, Collection, Logger, Dictionary } from '@freearhey/core'
import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
import type { ChannelSearchableData } from '../../types/channel'
import { Channel, Feed, Playlist, Stream } from '../../models'
import { DataProcessorData } from '../../types/dataProcessor'
import { DataLoaderData } from '../../types/dataLoader'
import { select, input } from '@inquirer/prompts'
import { DATA_DIR } from '../../constants'
import nodeCleanup from 'node-cleanup'
import sjs from '@freearhey/search-js'
import { Command } from 'commander'
import readline from 'readline'
type ChoiceValue = { type: string; value?: Feed | Channel }
type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }
if (process.platform === 'win32') {
readline
.createInterface({
input: process.stdin,
output: process.stdout
})
.on('SIGINT', function () {
process.emit('SIGINT')
})
}
const program = new Command()
program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(process.argv)
const filepath = program.args[0]
const logger = new Logger()
const storage = new Storage()
let parsedStreams = new Collection()
main(filepath)
nodeCleanup(() => {
save(filepath)
})
export default async function main(filepath: string) {
if (!(await storage.exists(filepath))) {
throw new Error(`File "${filepath}" does not exists`)
}
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const {
channels,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
}: DataProcessorData = processor.process(data)
logger.info('loading streams...')
const parser = new PlaylistParser({
storage,
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById
})
parsedStreams = await parser.parseFile(filepath)
const streamsWithoutId = parsedStreams.filter((stream: Stream) => !stream.id)
logger.info(
`found ${parsedStreams.count()} streams (including ${streamsWithoutId.count()} without ID)`
)
logger.info('creating search index...')
const items = channels.map((channel: Channel) => channel.getSearchable()).all()
const searchIndex = sjs.createIndex(items, {
searchable: ['name', 'altNames', 'guideNames', 'streamTitles', 'feedFullNames']
})
logger.info('starting...\n')
for (const stream of streamsWithoutId.all()) {
try {
stream.id = await selectChannel(stream, searchIndex, feedsGroupedByChannelId, channelsKeyById)
} catch (err) {
logger.info(err.message)
break
}
}
streamsWithoutId.forEach((stream: Stream) => {
if (stream.id === '-') {
stream.id = ''
}
})
}
async function selectChannel(
stream: Stream,
searchIndex,
feedsGroupedByChannelId: Dictionary,
channelsKeyById: Dictionary
): Promise<string> {
const query = escapeRegex(stream.getTitle())
const similarChannels = searchIndex
.search(query)
.map((item: ChannelSearchableData) => channelsKeyById.get(item.id))
const url = stream.url.length > 50 ? stream.url.slice(0, 50) + '...' : stream.url
const selected: ChoiceValue = await select({
message: `Select channel ID for "${stream.title}" (${url}):`,
choices: getChannelChoises(new Collection(similarChannels)),
pageSize: 10
})
switch (selected.type) {
case 'skip':
return '-'
case 'type': {
const typedChannelId = await input({ message: ' Channel ID:' })
if (!typedChannelId) return ''
const selectedFeedId = await selectFeed(typedChannelId, feedsGroupedByChannelId)
if (selectedFeedId === '-') return typedChannelId
return [typedChannelId, selectedFeedId].join('@')
}
case 'channel': {
const selectedChannel = selected.value
if (!selectedChannel) return ''
const selectedFeedId = await selectFeed(selectedChannel.id, feedsGroupedByChannelId)
if (selectedFeedId === '-') return selectedChannel.id
return [selectedChannel.id, selectedFeedId].join('@')
}
}
return ''
}
async function selectFeed(channelId: string, feedsGroupedByChannelId: Dictionary): Promise<string> {
const channelFeeds = new Collection(feedsGroupedByChannelId.get(channelId))
const choices = getFeedChoises(channelFeeds)
const selected: ChoiceValue = await select({
message: `Select feed ID for "${channelId}":`,
choices,
pageSize: 10
})
switch (selected.type) {
case 'skip':
return '-'
case 'type':
return await input({ message: ' Feed ID:', default: 'SD' })
case 'feed':
const selectedFeed = selected.value
if (!selectedFeed) return ''
return selectedFeed.id
}
return ''
}
function getChannelChoises(channels: Collection): Choice[] {
const choises: Choice[] = []
channels.forEach((channel: Channel) => {
const names = new Collection([channel.name, ...channel.altNames.all()]).uniq().join(', ')
choises.push({
value: {
type: 'channel',
value: channel
},
name: `${channel.id} (${names})`,
short: `${channel.id}`
})
})
choises.push({ name: 'Type...', value: { type: 'type' } })
choises.push({ name: 'Skip', value: { type: 'skip' } })
return choises
}
function getFeedChoises(feeds: Collection): Choice[] {
const choises: Choice[] = []
feeds.forEach((feed: Feed) => {
let name = `${feed.id} (${feed.name})`
if (feed.isMain) name += ' [main]'
choises.push({
value: {
type: 'feed',
value: feed
},
default: feed.isMain,
name,
short: feed.id
})
})
choises.push({ name: 'Type...', value: { type: 'type' } })
choises.push({ name: 'Skip', value: { type: 'skip' } })
return choises
}
function save(filepath: string) {
if (!storage.existsSync(filepath)) return
const playlist = new Playlist(parsedStreams)
storage.saveSync(filepath, playlist.toString())
logger.info(`\nFile '${filepath}' successfully saved`)
}
function escapeRegex(string: string) {
return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&')
}
import { loadData, data, searchChannels } from '../../api'
import { Collection, Logger } from '@freearhey/core'
import { select, input } from '@inquirer/prompts'
import { Playlist, Stream } from '../../models'
import { Storage } from '@freearhey/storage-js'
import { PlaylistParser } from '../../core'
import nodeCleanup from 'node-cleanup'
import * as sdk from '@iptv-org/sdk'
import { truncate } from '../../utils'
import { Command } from 'commander'
import readline from 'readline'
type ChoiceValue = { type: string; value?: sdk.Models.Feed | sdk.Models.Channel }
type Choice = { name: string; short?: string; value: ChoiceValue; default?: boolean }
if (process.platform === 'win32') {
readline
.createInterface({
input: process.stdin,
output: process.stdout
})
.on('SIGINT', function () {
process.emit('SIGINT')
})
}
const program = new Command()
program.argument('<filepath>', 'Path to *.channels.xml file to edit').parse(process.argv)
const filepath = program.args[0]
const logger = new Logger()
const storage = new Storage()
let parsedStreams = new Collection<Stream>()
main(filepath)
nodeCleanup(() => {
save(filepath)
})
export default async function main(filepath: string) {
if (!(await storage.exists(filepath))) {
throw new Error(`File "${filepath}" does not exists`)
}
logger.info('loading data from api...')
await loadData()
logger.info('loading streams...')
const parser = new PlaylistParser({
storage
})
parsedStreams = await parser.parseFile(filepath)
const streamsWithoutId = parsedStreams.filter((stream: Stream) => !stream.tvgId)
logger.info(
`found ${parsedStreams.count()} streams (including ${streamsWithoutId.count()} without ID)`
)
logger.info('starting...\n')
for (const stream of streamsWithoutId.all()) {
try {
stream.tvgId = await selectChannel(stream)
} catch (err) {
logger.info(err.message)
break
}
}
streamsWithoutId.forEach((stream: Stream) => {
if (stream.channel === '-') {
stream.channel = ''
}
})
}
async function selectChannel(stream: Stream): Promise<string> {
const query = escapeRegex(stream.title)
const similarChannels = searchChannels(query)
const url = truncate(stream.url, 50)
const selected: ChoiceValue = await select({
message: `Select channel ID for "${stream.title}" (${url}):`,
choices: getChannelChoises(similarChannels),
pageSize: 10
})
switch (selected.type) {
case 'skip':
return '-'
case 'type': {
const typedChannelId = await input({ message: ' Channel ID:' })
if (!typedChannelId) return ''
const selectedFeedId = await selectFeed(typedChannelId)
if (selectedFeedId === '-') return typedChannelId
return [typedChannelId, selectedFeedId].join('@')
}
case 'channel': {
const selectedChannel = selected.value
if (!selectedChannel) return ''
const selectedFeedId = await selectFeed(selectedChannel.id)
if (selectedFeedId === '-') return selectedChannel.id
return [selectedChannel.id, selectedFeedId].join('@')
}
}
return ''
}
async function selectFeed(channelId: string): Promise<string> {
const channelFeeds = new Collection(data.feedsGroupedByChannel.get(channelId))
const choices = getFeedChoises(channelFeeds)
const selected: ChoiceValue = await select({
message: `Select feed ID for "${channelId}":`,
choices,
pageSize: 10
})
switch (selected.type) {
case 'skip':
return '-'
case 'type':
return await input({ message: ' Feed ID:', default: 'SD' })
case 'feed':
const selectedFeed = selected.value
if (!selectedFeed) return ''
return selectedFeed.id
}
return ''
}
function getChannelChoises(channels: Collection<sdk.Models.Channel>): Choice[] {
const choises: Choice[] = []
channels.forEach((channel: sdk.Models.Channel) => {
const names = new Collection([channel.name, ...channel.alt_names]).uniq().join(', ')
choises.push({
value: {
type: 'channel',
value: channel
},
name: `${channel.id} (${names})`,
short: `${channel.id}`
})
})
choises.push({ name: 'Type...', value: { type: 'type' } })
choises.push({ name: 'Skip', value: { type: 'skip' } })
return choises
}
function getFeedChoises(feeds: Collection<sdk.Models.Feed>): Choice[] {
const choises: Choice[] = []
feeds.forEach((feed: sdk.Models.Feed) => {
let name = `${feed.id} (${feed.name})`
if (feed.is_main) name += ' [main]'
choises.push({
value: {
type: 'feed',
value: feed
},
default: feed.is_main,
name,
short: feed.id
})
})
choises.push({ name: 'Type...', value: { type: 'type' } })
choises.push({ name: 'Skip', value: { type: 'skip' } })
return choises
}
function save(filepath: string) {
if (!storage.existsSync(filepath)) return
const playlist = new Playlist(parsedStreams)
storage.saveSync(filepath, playlist.toString())
logger.info(`\nFile '${filepath}' successfully saved`)
}
function escapeRegex(string: string) {
return string.replace(/[/\-\\^$*+?.()|[\]{}]/g, '\\$&')
}

162
scripts/commands/playlist/format.ts
Unescape Escape View File

@ -1,78 +1,84 @@
import { Logger, Storage } from '@freearhey/core'
import { STREAMS_DIR, DATA_DIR } from '../../constants'
import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
import { Stream, Playlist } from '../../models'
import { program } from 'commander'
import { DataLoaderData } from '../../types/dataLoader'
import { DataProcessorData } from '../../types/dataProcessor'
import path from 'node:path'
program.argument('[filepath...]', 'Path to file to format').parse(process.argv)
async function main() {
const logger = new Logger()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData =
processor.process(data)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
})
let files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
files = files.map((filepath: string) => path.basename(filepath))
let streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
logger.info('normalizing links...')
streams = streams.map(stream => {
stream.normalizeURL()
return stream
})
logger.info('removing duplicates...')
streams = streams.uniqBy(stream => stream.url)
logger.info('removing wrong id...')
streams = streams.map((stream: Stream) => {
if (!stream.channel || channelsKeyById.missing(stream.channel.id)) {
stream.id = ''
}
return stream
})
logger.info('sorting links...')
streams = streams.orderBy(
[
(stream: Stream) => stream.title,
(stream: Stream) => stream.getVerticalResolution(),
(stream: Stream) => stream.getLabel(),
(stream: Stream) => stream.url
],
['asc', 'desc', 'asc', 'asc']
)
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of groupedStreams.keys()) {
const streams = groupedStreams.get(filepath) || []
if (!streams.length) return
const playlist = new Playlist(streams, { public: false })
await streamsStorage.save(filepath, playlist.toString())
}
}
main()
import { Collection, Logger } from '@freearhey/core'
import { Stream, Playlist } from '../../models'
import { Storage } from '@freearhey/storage-js'
import { STREAMS_DIR } from '../../constants'
import { PlaylistParser } from '../../core'
import { loadData } from '../../api'
import { program } from 'commander'
import path from 'node:path'
program.argument('[filepath...]', 'Path to file to format').parse(process.argv)
async function main() {
const logger = new Logger()
logger.info('loading data from api...')
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage
})
let files = program.args.length ? program.args : await streamsStorage.list('**/*.m3u')
files = files.map((filepath: string) => path.basename(filepath))
let streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
logger.info('normalizing links...')
streams = streams.map(stream => {
stream.normalizeURL()
return stream
})
logger.info('removing duplicates...')
streams = streams.uniqBy(stream => stream.url)
logger.info('removing wrong id...')
streams = streams.map((stream: Stream) => {
const channel = stream.getChannel()
if (channel) return stream
stream.tvgId = ''
stream.channel = ''
stream.feed = ''
return stream
})
logger.info('adding the missing feed id...')
streams = streams.map((stream: Stream) => {
const feed = stream.getFeed()
if (feed) {
stream.feed = feed.id
stream.tvgId = stream.getId()
}
return stream
})
logger.info('sorting links...')
streams = streams.sortBy(
[
(stream: Stream) => stream.title,
(stream: Stream) => stream.getVerticalResolution(),
(stream: Stream) => stream.label,
(stream: Stream) => stream.url
],
['asc', 'desc', 'asc', 'asc']
)
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of groupedStreams.keys()) {
const streams = new Collection(groupedStreams.get(filepath))
if (streams.isEmpty()) return
const playlist = new Playlist(streams, { public: false })
await streamsStorage.save(filepath, playlist.toString())
}
}
main()

246
scripts/commands/playlist/generate.ts
Unescape Escape View File

@ -1,131 +1,115 @@
import { PlaylistParser, DataProcessor, DataLoader } from '../../core'
import type { DataProcessorData } from '../../types/dataProcessor'
import { DATA_DIR, LOGS_DIR, STREAMS_DIR } from '../../constants'
import type { DataLoaderData } from '../../types/dataLoader'
import { Logger, Storage, File } from '@freearhey/core'
import { Stream } from '../../models'
import uniqueId from 'lodash.uniqueid'
import {
IndexCategoryGenerator,
IndexLanguageGenerator,
IndexCountryGenerator,
SubdivisionsGenerator,
CategoriesGenerator,
CountriesGenerator,
LanguagesGenerator,
RegionsGenerator,
SourcesGenerator,
CitiesGenerator,
IndexGenerator,
RawGenerator
} from '../../generators'
async function main() {
const logger = new Logger()
const logFile = new File('generators.log')
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const {
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById,
subdivisions,
categories,
countries,
regions,
cities
}: DataProcessorData = processor.process(data)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById
})
const files = await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
const totalStreams = streams.count()
logger.info(`found ${totalStreams} streams`)
logger.info('generating raw/...')
await new RawGenerator({ streams, logFile }).generate()
logger.info('filtering streams...')
streams = streams.uniqBy((stream: Stream) =>
stream.hasId() ? stream.getChannelId() + stream.getFeedId() : uniqueId()
)
logger.info('sorting streams...')
streams = streams.orderBy(
[
(stream: Stream) => stream.getId(),
(stream: Stream) => stream.getVerticalResolution(),
(stream: Stream) => stream.getLabel()
],
['asc', 'asc', 'desc']
)
logger.info('generating categories/...')
await new CategoriesGenerator({ categories, streams, logFile }).generate()
logger.info('generating languages/...')
await new LanguagesGenerator({ streams, logFile }).generate()
logger.info('generating countries/...')
await new CountriesGenerator({
countries,
streams,
logFile
}).generate()
logger.info('generating subdivisions/...')
await new SubdivisionsGenerator({
subdivisions,
streams,
logFile
}).generate()
logger.info('generating cities/...')
await new CitiesGenerator({
cities,
streams,
logFile
}).generate()
logger.info('generating regions/...')
await new RegionsGenerator({
streams,
regions,
logFile
}).generate()
logger.info('generating sources/...')
await new SourcesGenerator({ streams, logFile }).generate()
logger.info('generating index.m3u...')
await new IndexGenerator({ streams, logFile }).generate()
logger.info('generating index.category.m3u...')
await new IndexCategoryGenerator({ streams, logFile }).generate()
logger.info('generating index.country.m3u...')
await new IndexCountryGenerator({
streams,
logFile
}).generate()
logger.info('generating index.language.m3u...')
await new IndexLanguageGenerator({ streams, logFile }).generate()
logger.info('saving generators.log...')
const logStorage = new Storage(LOGS_DIR)
logStorage.saveFile(logFile)
}
main()
import { LOGS_DIR, STREAMS_DIR } from '../../constants'
import { Storage, File } from '@freearhey/storage-js'
import { PlaylistParser } from '../../core'
import { loadData, data } from '../../api'
import { Logger } from '@freearhey/core'
import uniqueId from 'lodash.uniqueid'
import { Stream } from '../../models'
import {
IndexCategoryGenerator,
IndexLanguageGenerator,
IndexCountryGenerator,
SubdivisionsGenerator,
CategoriesGenerator,
CountriesGenerator,
LanguagesGenerator,
RegionsGenerator,
SourcesGenerator,
CitiesGenerator,
IndexGenerator,
RawGenerator
} from '../../generators'
async function main() {
const logger = new Logger()
const logFile = new File('generators.log')
logger.info('loading data from api...')
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage
})
const files = await streamsStorage.list('**/*.m3u')
let streams = await parser.parse(files)
const totalStreams = streams.count()
logger.info(`found ${totalStreams} streams`)
logger.info('generating raw/...')
await new RawGenerator({ streams, logFile }).generate()
logger.info('filtering streams...')
streams = streams.uniqBy((stream: Stream) => stream.getId() || uniqueId())
logger.info('sorting streams...')
streams = streams.sortBy(
[
(stream: Stream) => stream.getId(),
(stream: Stream) => stream.getVerticalResolution(),
(stream: Stream) => stream.label
],
['asc', 'asc', 'desc']
)
const { categories, countries, subdivisions, cities, regions } = data
logger.info('generating categories/...')
await new CategoriesGenerator({ categories, streams, logFile }).generate()
logger.info('generating languages/...')
await new LanguagesGenerator({ streams, logFile }).generate()
logger.info('generating countries/...')
await new CountriesGenerator({
countries,
streams,
logFile
}).generate()
logger.info('generating subdivisions/...')
await new SubdivisionsGenerator({
subdivisions,
streams,
logFile
}).generate()
logger.info('generating cities/...')
await new CitiesGenerator({
cities,
streams,
logFile
}).generate()
logger.info('generating regions/...')
await new RegionsGenerator({
streams,
regions,
logFile
}).generate()
logger.info('generating sources/...')
await new SourcesGenerator({ streams, logFile }).generate()
logger.info('generating index.m3u...')
await new IndexGenerator({ streams, logFile }).generate()
logger.info('generating index.category.m3u...')
await new IndexCategoryGenerator({ streams, logFile }).generate()
logger.info('generating index.country.m3u...')
await new IndexCountryGenerator({
streams,
logFile
}).generate()
logger.info('generating index.language.m3u...')
await new IndexLanguageGenerator({ streams, logFile }).generate()
logger.info('saving generators.log...')
const logStorage = new Storage(LOGS_DIR)
logStorage.saveFile(logFile)
}
main()

359
scripts/commands/playlist/test.ts
Unescape Escape View File

@ -1,182 +1,177 @@
import { Logger, Storage, Collection } from '@freearhey/core'
import { ROOT_DIR, STREAMS_DIR, DATA_DIR } from '../../constants'
import { PlaylistParser, StreamTester, CliTable, DataProcessor, DataLoader } from '../../core'
import type { TestResult } from '../../core/streamTester'
import { Stream } from '../../models'
import { program, OptionValues } from 'commander'
import { eachLimit } from 'async-es'
import chalk from 'chalk'
import os from 'node:os'
import dns from 'node:dns'
import type { DataLoaderData } from '../../types/dataLoader'
import type { DataProcessorData } from '../../types/dataProcessor'
const LIVE_UPDATE_INTERVAL = 5000
const LIVE_UPDATE_MAX_STREAMS = 100
let errors = 0
let warnings = 0
const results: { [key: string]: string } = {}
let interval: string | number | NodeJS.Timeout | undefined
let streams = new Collection()
let isLiveUpdateEnabled = true
program
.argument('[filepath...]', 'Path to file to test')
.option(
'-p, --parallel <number>',
'Batch size of streams to test concurrently',
(value: string) => parseInt(value),
os.cpus().length
)
.option('-x, --proxy <url>', 'Use the specified proxy')
.option(
'-t, --timeout <number>',
'The number of milliseconds before the request will be aborted',
(value: string) => parseInt(value),
30000
)
.parse(process.argv)
const options: OptionValues = program.opts()
const logger = new Logger()
const tester = new StreamTester({ options })
async function main() {
if (await isOffline()) {
logger.error(chalk.red('Internet connection is required for the script to work'))
return
}
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData =
processor.process(data)
logger.info('loading streams...')
const rootStorage = new Storage(ROOT_DIR)
const parser = new PlaylistParser({
storage: rootStorage,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
})
const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`)
streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
if (streams.count() > LIVE_UPDATE_MAX_STREAMS) isLiveUpdateEnabled = false
logger.info('starting...')
if (!isLiveUpdateEnabled) {
drawTable()
interval = setInterval(() => {
drawTable()
}, LIVE_UPDATE_INTERVAL)
}
await eachLimit(
streams.all(),
options.parallel,
async (stream: Stream) => {
await runTest(stream)
if (isLiveUpdateEnabled) {
drawTable()
}
},
onFinish
)
}
main()
async function runTest(stream: Stream) {
const key = stream.filepath + stream.getId() + stream.url
results[key] = chalk.white('LOADING...')
const result: TestResult = await tester.test(stream)
let status = ''
const errorStatusCodes = ['ENOTFOUND', 'HTTP_404_NOT_FOUND']
if (result.status.ok) status = chalk.green('OK')
else if (errorStatusCodes.includes(result.status.code)) {
status = chalk.red(result.status.code)
errors++
} else {
status = chalk.yellow(result.status.code)
warnings++
}
results[key] = status
}
function drawTable() {
process.stdout.write('\u001b[3J\u001b[1J')
console.clear()
const streamsGrouped = streams.groupBy((stream: Stream) => stream.filepath)
for (const filepath of streamsGrouped.keys()) {
const streams: Stream[] = streamsGrouped.get(filepath)
const table = new CliTable({
columns: [
{ name: '', alignment: 'center', minLen: 3, maxLen: 3 },
{ name: 'tvg-id', alignment: 'left', color: 'green', minLen: 25, maxLen: 25 },
{ name: 'url', alignment: 'left', color: 'green', minLen: 100, maxLen: 100 },
{ name: 'status', alignment: 'left', minLen: 25, maxLen: 25 }
]
})
streams.forEach((stream: Stream, index: number) => {
const status = results[stream.filepath + stream.getId() + stream.url] || chalk.gray('PENDING')
const row = {
'': index,
'tvg-id': stream.getId().length > 25 ? stream.getId().slice(0, 22) + '...' : stream.getId(),
url: stream.url.length > 100 ? stream.url.slice(0, 97) + '...' : stream.url,
status
}
table.append(row)
})
process.stdout.write(`\n${chalk.underline(filepath)}\n`)
process.stdout.write(table.toString())
}
}
function onFinish(error: any) {
clearInterval(interval)
if (error) {
console.error(error)
process.exit(1)
}
drawTable()
if (errors > 0 || warnings > 0) {
console.log(
chalk.red(`\n${errors + warnings} problems (${errors} errors, ${warnings} warnings)`)
)
if (errors > 0) {
process.exit(1)
}
}
process.exit(0)
}
async function isOffline() {
return new Promise((resolve, reject) => {
dns.lookup('info.cern.ch', err => {
if (err) resolve(true)
reject(false)
})
}).catch(() => {})
}
import { PlaylistParser, StreamTester, CliTable } from '../../core'
import type { TestResult } from '../../core/streamTester'
import { ROOT_DIR, STREAMS_DIR } from '../../constants'
import { Logger, Collection } from '@freearhey/core'
import { program, OptionValues } from 'commander'
import { Storage } from '@freearhey/storage-js'
import { Stream } from '../../models'
import { loadData } from '../../api'
import { eachLimit } from 'async'
import dns from 'node:dns'
import chalk from 'chalk'
import os from 'node:os'
import { truncate } from '../../utils'
const LIVE_UPDATE_INTERVAL = 5000
const LIVE_UPDATE_MAX_STREAMS = 100
let errors = 0
let warnings = 0
const results: { [key: string]: string } = {}
let interval: string | number | NodeJS.Timeout | undefined
let streams = new Collection<Stream>()
let isLiveUpdateEnabled = true
program
.argument('[filepath...]', 'Path to file to test')
.option(
'-p, --parallel <number>',
'Batch size of streams to test concurrently',
(value: string) => parseInt(value),
os.cpus().length
)
.option('-x, --proxy <url>', 'Use the specified proxy')
.option(
'-t, --timeout <number>',
'The number of milliseconds before the request will be aborted',
(value: string) => parseInt(value),
30000
)
.parse(process.argv)
const options: OptionValues = program.opts()
const logger = new Logger()
const tester = new StreamTester({ options })
async function main() {
if (await isOffline()) {
logger.error(chalk.red('Internet connection is required for the script to work'))
return
}
logger.info('loading data from api...')
await loadData()
logger.info('loading streams...')
const rootStorage = new Storage(ROOT_DIR)
const parser = new PlaylistParser({
storage: rootStorage
})
const files = program.args.length ? program.args : await rootStorage.list(`${STREAMS_DIR}/*.m3u`)
streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
if (streams.count() > LIVE_UPDATE_MAX_STREAMS) isLiveUpdateEnabled = false
logger.info('starting...')
if (!isLiveUpdateEnabled) {
drawTable()
interval = setInterval(() => {
drawTable()
}, LIVE_UPDATE_INTERVAL)
}
eachLimit(
streams.all(),
options.parallel,
async (stream: Stream) => {
await runTest(stream)
if (isLiveUpdateEnabled) {
drawTable()
}
},
onFinish
)
}
main()
async function runTest(stream: Stream) {
const key = stream.getUniqKey()
results[key] = chalk.white('LOADING...')
const result: TestResult = await tester.test(stream)
let status = ''
const errorStatusCodes = ['ENOTFOUND', 'HTTP_404_NOT_FOUND']
if (result.status.ok) status = chalk.green('OK')
else if (errorStatusCodes.includes(result.status.code)) {
status = chalk.red(result.status.code)
errors++
} else {
status = chalk.yellow(result.status.code)
warnings++
}
results[key] = status
}
function drawTable() {
process.stdout.write('\u001b[3J\u001b[1J')
console.clear()
const streamsGrouped = streams.groupBy((stream: Stream) => stream.filepath)
for (const filepath of streamsGrouped.keys()) {
const streams: Stream[] = streamsGrouped.get(filepath) || []
const table = new CliTable({
columns: [
{ name: '', alignment: 'center', minLen: 3, maxLen: 3 },
{ name: 'tvg-id', alignment: 'left', color: 'green', minLen: 25, maxLen: 25 },
{ name: 'url', alignment: 'left', color: 'green', minLen: 100, maxLen: 100 },
{ name: 'status', alignment: 'left', minLen: 25, maxLen: 25 }
]
})
streams.forEach((stream: Stream, index: number) => {
const key = stream.getUniqKey()
const status = results[key] || chalk.gray('PENDING')
const tvgId = stream.getTvgId()
const row = {
'': index,
'tvg-id': truncate(tvgId, 25),
url: truncate(stream.url, 100),
status
}
table.append(row)
})
process.stdout.write(`\n${chalk.underline(filepath)}\n`)
process.stdout.write(table.toString())
}
}
function onFinish(error: Error) {
clearInterval(interval)
if (error) {
console.error(error)
process.exit(1)
}
drawTable()
if (errors > 0 || warnings > 0) {
console.log(
chalk.red(`\n${errors + warnings} problems (${errors} errors, ${warnings} warnings)`)
)
if (errors > 0) {
process.exit(1)
}
}
process.exit(0)
}
async function isOffline() {
return new Promise((resolve, reject) => {
dns.lookup('info.cern.ch', err => {
if (err) resolve(true)
reject(false)
})
}).catch(() => {})
}

368
scripts/commands/playlist/update.ts
Unescape Escape View File

@ -1,194 +1,174 @@
import { DataLoader, DataProcessor, IssueLoader, PlaylistParser } from '../../core'
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import type { DataProcessorData } from '../../types/dataProcessor'
import { Stream, Playlist, Channel, Issue } from '../../models'
import type { DataLoaderData } from '../../types/dataLoader'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { isURI } from '../../utils'
const processedIssues = new Collection()
async function main() {
const logger = new Logger({ level: -999 })
const issueLoader = new IssueLoader()
logger.info('loading issues...')
const issues = await issueLoader.load()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const dataLoader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await dataLoader.load()
const { channelsKeyById, feedsGroupedByChannelId, logosGroupedByStreamId }: DataProcessorData =
processor.process(data)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById
})
const files = await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
logger.info('removing streams...')
await removeStreams({ streams, issues })
logger.info('edit stream description...')
await editStreams({
streams,
issues,
channelsKeyById,
feedsGroupedByChannelId
})
logger.info('add new streams...')
await addStreams({
streams,
issues,
channelsKeyById,
feedsGroupedByChannelId
})
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of groupedStreams.keys()) {
let streams = groupedStreams.get(filepath) || []
streams = streams.filter((stream: Stream) => stream.removed === false)
const playlist = new Playlist(streams, { public: false })
await streamsStorage.save(filepath, playlist.toString())
}
const output = processedIssues.map(issue_number => `closes #${issue_number}`).join(', ')
console.log(`OUTPUT=${output}`)
}
main()
async function removeStreams({ streams, issues }: { streams: Collection; issues: Collection }) {
const requests = issues.filter(
issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved')
)
requests.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('streamUrl')) return
const streamUrls = data.getString('streamUrl') || ''
let changed = false
streamUrls
.split(/\r?\n/)
.filter(Boolean)
.forEach(link => {
const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim())
if (found) {
found.removed = true
changed = true
}
})
if (changed) processedIssues.add(issue.number)
})
}
async function editStreams({
streams,
issues,
channelsKeyById,
feedsGroupedByChannelId
}: {
streams: Collection
issues: Collection
channelsKeyById: Dictionary
feedsGroupedByChannelId: Dictionary
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved')
)
requests.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('streamUrl')) return
const stream: Stream = streams.first(
(_stream: Stream) => _stream.url === data.getString('streamUrl')
)
if (!stream) return
const streamId = data.getString('streamId') || ''
const [channelId, feedId] = streamId.split('@')
if (channelId) {
stream
.setChannelId(channelId)
.setFeedId(feedId)
.withChannel(channelsKeyById)
.withFeed(feedsGroupedByChannelId)
.updateId()
.updateTitle()
.updateFilepath()
}
stream.update(data)
processedIssues.add(issue.number)
})
}
async function addStreams({
streams,
issues,
channelsKeyById,
feedsGroupedByChannelId
}: {
streams: Collection
issues: Collection
channelsKeyById: Dictionary
feedsGroupedByChannelId: Dictionary
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:add') && issue.labels.includes('approved')
)
requests.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('streamId') || data.missing('streamUrl')) return
if (streams.includes((_stream: Stream) => _stream.url === data.getString('streamUrl'))) return
const streamUrl = data.getString('streamUrl') || ''
if (!isURI(streamUrl)) return
const streamId = data.getString('streamId') || ''
const [channelId, feedId] = streamId.split('@')
const channel: Channel = channelsKeyById.get(channelId)
if (!channel) return
const label = data.getString('label') || null
const quality = data.getString('quality') || null
const httpUserAgent = data.getString('httpUserAgent') || null
const httpReferrer = data.getString('httpReferrer') || null
const directives = data.getArray('directives') || []
const stream = new Stream({
channelId,
feedId,
title: channel.name,
url: streamUrl,
userAgent: httpUserAgent,
referrer: httpReferrer,
directives,
quality,
label
})
.withChannel(channelsKeyById)
.withFeed(feedsGroupedByChannelId)
.updateTitle()
.updateFilepath()
streams.add(stream)
processedIssues.add(issue.number)
})
}
import { IssueLoader, PlaylistParser } from '../../core'
import { Playlist, Issue, Stream } from '../../models'
import { loadData, data as apiData } from '../../api'
import { Logger, Collection } from '@freearhey/core'
import { Storage } from '@freearhey/storage-js'
import { STREAMS_DIR } from '../../constants'
import * as sdk from '@iptv-org/sdk'
import { isURI } from '../../utils'
const processedIssues = new Collection()
async function main() {
const logger = new Logger({ level: -999 })
const issueLoader = new IssueLoader()
logger.info('loading issues...')
const issues = await issueLoader.load()
logger.info('loading data from api...')
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage
})
const files = await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
logger.info('removing streams...')
await removeStreams({ streams, issues })
logger.info('edit stream description...')
await editStreams({
streams,
issues
})
logger.info('add new streams...')
await addStreams({
streams,
issues
})
logger.info('saving...')
const groupedStreams = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of groupedStreams.keys()) {
let streams = new Collection(groupedStreams.get(filepath))
streams = streams.filter((stream: Stream) => stream.removed === false)
const playlist = new Playlist(streams, { public: false })
await streamsStorage.save(filepath, playlist.toString())
}
const output = processedIssues.map(issue_number => `closes #${issue_number}`).join(', ')
console.log(`OUTPUT=${output}`)
}
main()
async function removeStreams({
streams,
issues
}: {
streams: Collection<Stream>
issues: Collection<Issue>
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:remove') && issue.labels.includes('approved')
)
requests.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('streamUrl')) return
const streamUrls = data.getString('streamUrl') || ''
let changed = false
streamUrls
.split(/\r?\n/)
.filter(Boolean)
.forEach(link => {
const found: Stream = streams.first((_stream: Stream) => _stream.url === link.trim())
if (found) {
found.removed = true
changed = true
}
})
if (changed) processedIssues.add(issue.number)
})
}
async function editStreams({
streams,
issues
}: {
streams: Collection<Stream>
issues: Collection<Issue>
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:edit') && issue.labels.includes('approved')
)
requests.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('streamUrl')) return
const stream: Stream = streams.first(
(_stream: Stream) => _stream.url === data.getString('streamUrl')
)
if (!stream) return
const streamId = data.getString('streamId') || ''
const [channelId, feedId] = streamId.split('@')
if (channelId) {
stream.channel = channelId
stream.feed = feedId
stream.updateTvgId().updateTitle().updateFilepath()
}
stream.updateWithIssue(data)
processedIssues.add(issue.number)
})
}
async function addStreams({
streams,
issues
}: {
streams: Collection<Stream>
issues: Collection<Issue>
}) {
const requests = issues.filter(
issue => issue.labels.includes('streams:add') && issue.labels.includes('approved')
)
requests.forEach((issue: Issue) => {
const data = issue.data
if (data.missing('streamId') || data.missing('streamUrl')) return
if (streams.includes((_stream: Stream) => _stream.url === data.getString('streamUrl'))) return
const streamUrl = data.getString('streamUrl') || ''
if (!isURI(streamUrl)) return
const streamId = data.getString('streamId') || ''
const [channelId, feedId] = streamId.split('@')
const channel: sdk.Models.Channel | undefined = apiData.channelsKeyById.get(channelId)
if (!channel) return
const label = data.getString('label') || ''
const quality = data.getString('quality') || null
const httpUserAgent = data.getString('httpUserAgent') || null
const httpReferrer = data.getString('httpReferrer') || null
const directives = data.getArray('directives') || []
const stream = new Stream({
channel: channelId,
feed: feedId,
title: channel.name,
url: streamUrl,
user_agent: httpUserAgent,
referrer: httpReferrer,
quality
})
stream.label = label
stream.setDirectives(directives).updateTitle().updateFilepath()
streams.add(stream)
processedIssues.add(issue.number)
})
}

249
scripts/commands/playlist/validate.ts
Unescape Escape View File

@ -1,129 +1,120 @@
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import { DataLoader, DataProcessor, PlaylistParser } from '../../core'
import { DataProcessorData } from '../../types/dataProcessor'
import { DATA_DIR, ROOT_DIR } from '../../constants'
import { DataLoaderData } from '../../types/dataLoader'
import { BlocklistRecord, Stream } from '../../models'
import { program } from 'commander'
import chalk from 'chalk'
program.argument('[filepath...]', 'Path to file to validate').parse(process.argv)
type LogItem = {
type: string
line: number
message: string
}
async function main() {
const logger = new Logger()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const {
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId,
blocklistRecordsGroupedByChannelId
}: DataProcessorData = processor.process(data)
logger.info('loading streams...')
const rootStorage = new Storage(ROOT_DIR)
const parser = new PlaylistParser({
storage: rootStorage,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
})
const files = program.args.length ? program.args : await rootStorage.list('streams/**/*.m3u')
const streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
let errors = new Collection()
let warnings = new Collection()
const streamsGroupedByFilepath = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of streamsGroupedByFilepath.keys()) {
const streams = streamsGroupedByFilepath.get(filepath)
if (!streams) continue
const log = new Collection()
const buffer = new Dictionary()
streams.forEach((stream: Stream) => {
if (stream.channelId) {
const channel = channelsKeyById.get(stream.channelId)
if (!channel) {
log.add({
type: 'warning',
line: stream.getLine(),
message: `"${stream.id}" is not in the database`
})
}
}
const duplicate = stream.url && buffer.has(stream.url)
if (duplicate) {
log.add({
type: 'warning',
line: stream.getLine(),
message: `"${stream.url}" is already on the playlist`
})
} else {
buffer.set(stream.url, true)
}
const blocklistRecords = stream.channel
? new Collection(blocklistRecordsGroupedByChannelId.get(stream.channel.id))
: new Collection()
blocklistRecords.forEach((blocklistRecord: BlocklistRecord) => {
if (blocklistRecord.reason === 'dmca') {
log.add({
type: 'error',
line: stream.getLine(),
message: `"${blocklistRecord.channelId}" is on the blocklist due to claims of copyright holders (${blocklistRecord.ref})`
})
} else if (blocklistRecord.reason === 'nsfw') {
log.add({
type: 'error',
line: stream.getLine(),
message: `"${blocklistRecord.channelId}" is on the blocklist due to NSFW content (${blocklistRecord.ref})`
})
}
})
})
if (log.notEmpty()) {
console.log(`\n${chalk.underline(filepath)}`)
log.forEach((logItem: LogItem) => {
const position = logItem.line.toString().padEnd(6, ' ')
const type = logItem.type.padEnd(9, ' ')
const status = logItem.type === 'error' ? chalk.red(type) : chalk.yellow(type)
console.log(` ${chalk.gray(position)}${status}${logItem.message}`)
})
errors = errors.concat(log.filter((logItem: LogItem) => logItem.type === 'error'))
warnings = warnings.concat(log.filter((logItem: LogItem) => logItem.type === 'warning'))
}
}
if (errors.count() || warnings.count()) {
console.log(
chalk.red(
`\n${
errors.count() + warnings.count()
} problems (${errors.count()} errors, ${warnings.count()} warnings)`
)
)
if (errors.count()) {
process.exit(1)
}
}
}
main()
import { Logger, Collection, Dictionary } from '@freearhey/core'
import { Storage } from '@freearhey/storage-js'
import { PlaylistParser } from '../../core'
import { data, loadData } from '../../api'
import { ROOT_DIR } from '../../constants'
import { Stream } from '../../models'
import * as sdk from '@iptv-org/sdk'
import { program } from 'commander'
import chalk from 'chalk'
program.argument('[filepath...]', 'Path to file to validate').parse(process.argv)
type LogItem = {
type: string
line: number
message: string
}
async function main() {
const logger = new Logger()
logger.info('loading data from api...')
await loadData()
logger.info('loading streams...')
const rootStorage = new Storage(ROOT_DIR)
const parser = new PlaylistParser({
storage: rootStorage
})
const files = program.args.length ? program.args : await rootStorage.list('streams/**/*.m3u')
const streams = await parser.parse(files)
logger.info(`found ${streams.count()} streams`)
let errors = new Collection()
let warnings = new Collection()
const streamsGroupedByFilepath = streams.groupBy((stream: Stream) => stream.getFilepath())
for (const filepath of streamsGroupedByFilepath.keys()) {
const streams = streamsGroupedByFilepath.get(filepath)
if (!streams) continue
const log = new Collection<LogItem>()
const buffer = new Dictionary<boolean>()
streams.forEach((stream: Stream) => {
if (stream.channel) {
const channel = data.channelsKeyById.get(stream.channel)
if (!channel) {
log.add({
type: 'warning',
line: stream.getLine(),
message: `"${stream.tvgId}" is not in the database`
})
}
}
const duplicate = stream.url && buffer.has(stream.url)
if (duplicate) {
log.add({
type: 'warning',
line: stream.getLine(),
message: `"${stream.url}" is already on the playlist`
})
} else {
buffer.set(stream.url, true)
}
if (stream.channel) {
const blocklistRecords = new Collection(
data.blocklistRecordsGroupedByChannel.get(stream.channel)
)
blocklistRecords.forEach((blocklistRecord: sdk.Models.BlocklistRecord) => {
if (blocklistRecord.reason === 'dmca') {
log.add({
type: 'error',
line: stream.getLine(),
message: `"${blocklistRecord.channel}" is on the blocklist due to claims of copyright holders (${blocklistRecord.ref})`
})
} else if (blocklistRecord.reason === 'nsfw') {
log.add({
type: 'error',
line: stream.getLine(),
message: `"${blocklistRecord.channel}" is on the blocklist due to NSFW content (${blocklistRecord.ref})`
})
}
})
}
})
if (log.isNotEmpty()) {
console.log(`\n${chalk.underline(filepath)}`)
log.forEach((logItem: LogItem) => {
const position = logItem.line.toString().padEnd(6, ' ')
const type = logItem.type.padEnd(9, ' ')
const status = logItem.type === 'error' ? chalk.red(type) : chalk.yellow(type)
console.log(` ${chalk.gray(position)}${status}${logItem.message}`)
})
errors = errors.concat(log.filter((logItem: LogItem) => logItem.type === 'error'))
warnings = warnings.concat(log.filter((logItem: LogItem) => logItem.type === 'warning'))
}
}
if (errors.count() || warnings.count()) {
console.log(
chalk.red(
`\n${
errors.count() + warnings.count()
} problems (${errors.count()} errors, ${warnings.count()} warnings)`
)
)
if (errors.count()) {
process.exit(1)
}
}
}
main()

78
scripts/commands/readme/update.ts
Unescape Escape View File

@ -1,48 +1,30 @@
import { CategoriesTable, CountriesTable, LanguagesTable, RegionsTable } from '../../tables'
import { DataLoader, DataProcessor, Markdown } from '../../core'
import { DataProcessorData } from '../../types/dataProcessor'
import { DataLoaderData } from '../../types/dataLoader'
import { README_DIR, DATA_DIR, ROOT_DIR } from '../../constants'
import { Logger, Storage } from '@freearhey/core'
async function main() {
const logger = new Logger()
const dataStorage = new Storage(DATA_DIR)
const processor = new DataProcessor()
const loader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await loader.load()
const {
subdivisionsKeyByCode,
languagesKeyByCode,
countriesKeyByCode,
categoriesKeyById,
subdivisions,
countries,
regions,
cities
}: DataProcessorData = processor.process(data)
logger.info('creating category table...')
await new CategoriesTable({ categoriesKeyById }).make()
logger.info('creating language table...')
await new LanguagesTable({ languagesKeyByCode }).make()
logger.info('creating countires table...')
await new CountriesTable({
countriesKeyByCode,
subdivisionsKeyByCode,
subdivisions,
countries,
cities
}).make()
logger.info('creating region table...')
await new RegionsTable({ regions }).make()
logger.info('updating playlists.md...')
const playlists = new Markdown({
build: `${ROOT_DIR}/PLAYLISTS.md`,
template: `${README_DIR}/template.md`
})
playlists.compile()
}
main()
import { CategoriesTable, CountriesTable, LanguagesTable, RegionsTable } from '../../tables'
import { README_DIR, ROOT_DIR } from '../../constants'
import { Logger } from '@freearhey/core'
import { Markdown } from '../../core'
import { loadData } from '../../api'
async function main() {
const logger = new Logger()
logger.info('loading data from api...')
await loadData()
logger.info('creating category table...')
await new CategoriesTable().create()
logger.info('creating language table...')
await new LanguagesTable().create()
logger.info('creating countires table...')
await new CountriesTable().create()
logger.info('creating region table...')
await new RegionsTable().create()
logger.info('updating playlists.md...')
const playlists = new Markdown({
build: `${ROOT_DIR}/PLAYLISTS.md`,
template: `${README_DIR}/template.md`
})
playlists.compile()
}
main()

337
scripts/commands/report/create.ts
Unescape Escape View File

@ -1,178 +1,159 @@
import { DataLoader, DataProcessor, IssueLoader, PlaylistParser } from '../../core'
import { Logger, Storage, Collection, Dictionary } from '@freearhey/core'
import { DataProcessorData } from '../../types/dataProcessor'
import { DATA_DIR, STREAMS_DIR } from '../../constants'
import { DataLoaderData } from '../../types/dataLoader'
import { Issue, Stream } from '../../models'
import { isURI } from '../../utils'
async function main() {
const logger = new Logger()
const issueLoader = new IssueLoader()
let report = new Collection()
logger.info('loading issues...')
const issues = await issueLoader.load()
logger.info('loading data from api...')
const processor = new DataProcessor()
const dataStorage = new Storage(DATA_DIR)
const dataLoader = new DataLoader({ storage: dataStorage })
const data: DataLoaderData = await dataLoader.load()
const {
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId,
blocklistRecordsGroupedByChannelId
}: DataProcessorData = processor.process(data)
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage,
channelsKeyById,
feedsGroupedByChannelId,
logosGroupedByStreamId
})
const files = await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url)
const streamsGroupedByChannelId = streams.groupBy((stream: Stream) => stream.channelId)
const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
logger.info('checking streams:remove requests...')
const removeRequests = issues.filter(issue =>
issue.labels.find((label: string) => label === 'streams:remove')
)
removeRequests.forEach((issue: Issue) => {
const streamUrls = issue.data.getArray('streamUrl') || []
if (!streamUrls.length) {
const result = {
issueNumber: issue.number,
type: 'streams:remove',
streamId: undefined,
streamUrl: undefined,
status: 'missing_link'
}
report.add(result)
} else {
for (const streamUrl of streamUrls) {
const result = {
issueNumber: issue.number,
type: 'streams:remove',
streamId: undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
}
if (streamsGroupedByUrl.missing(streamUrl)) {
result.status = 'wrong_link'
}
report.add(result)
}
}
})
logger.info('checking streams:add requests...')
const addRequests = issues.filter(issue => issue.labels.includes('streams:add'))
const addRequestsBuffer = new Dictionary()
addRequests.forEach((issue: Issue) => {
const streamId = issue.data.getString('streamId') || ''
const streamUrl = issue.data.getString('streamUrl') || ''
const [channelId] = streamId.split('@')
const result = {
issueNumber: issue.number,
type: 'streams:add',
streamId: streamId || undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
}
if (!channelId) result.status = 'missing_id'
else if (!streamUrl) result.status = 'missing_link'
else if (!isURI(streamUrl)) result.status = 'invalid_link'
else if (blocklistRecordsGroupedByChannelId.has(channelId)) result.status = 'blocked'
else if (channelsKeyById.missing(channelId)) result.status = 'wrong_id'
else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist'
else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate'
else result.status = 'pending'
addRequestsBuffer.set(streamUrl, true)
report.add(result)
})
logger.info('checking streams:edit requests...')
const editRequests = issues.filter(issue =>
issue.labels.find((label: string) => label === 'streams:edit')
)
editRequests.forEach((issue: Issue) => {
const streamId = issue.data.getString('streamId') || ''
const streamUrl = issue.data.getString('streamUrl') || ''
const [channelId] = streamId.split('@')
const result = {
issueNumber: issue.number,
type: 'streams:edit',
streamId: streamId || undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
}
if (!streamUrl) result.status = 'missing_link'
else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link'
else if (channelId && channelsKeyById.missing(channelId)) result.status = 'invalid_id'
report.add(result)
})
logger.info('checking channel search requests...')
const channelSearchRequests = issues.filter(issue =>
issue.labels.find((label: string) => label === 'channel search')
)
const channelSearchRequestsBuffer = new Dictionary()
channelSearchRequests.forEach((issue: Issue) => {
const streamId = issue.data.getString('channelId') || ''
const [channelId, feedId] = streamId.split('@')
const result = {
issueNumber: issue.number,
type: 'channel search',
streamId: streamId || undefined,
streamUrl: undefined,
status: 'pending'
}
if (!channelId) result.status = 'missing_id'
else if (channelsKeyById.missing(channelId)) result.status = 'invalid_id'
else if (channelSearchRequestsBuffer.has(streamId)) result.status = 'duplicate'
else if (blocklistRecordsGroupedByChannelId.has(channelId)) result.status = 'blocked'
else if (streamsGroupedById.has(streamId)) result.status = 'fulfilled'
else if (!feedId && streamsGroupedByChannelId.has(channelId)) result.status = 'fulfilled'
else {
const channelData = channelsKeyById.get(channelId)
if (channelData && channelData.isClosed) result.status = 'closed'
}
channelSearchRequestsBuffer.set(streamId, true)
report.add(result)
})
report = report.orderBy(item => item.issueNumber).filter(item => item.status !== 'pending')
console.table(report.all())
}
main()
function truncate(string: string, limit: number = 100) {
if (!string) return string
if (string.length < limit) return string
return string.slice(0, limit) + '...'
}
import { Logger, Collection, Dictionary } from '@freearhey/core'
import { IssueLoader, PlaylistParser } from '../../core'
import { Storage } from '@freearhey/storage-js'
import { isURI, truncate } from '../../utils'
import { STREAMS_DIR } from '../../constants'
import { Issue, Stream } from '../../models'
import { data, loadData } from '../../api'
async function main() {
const logger = new Logger()
const issueLoader = new IssueLoader()
let report = new Collection()
logger.info('loading issues...')
const issues = await issueLoader.load()
logger.info('loading data from api...')
await loadData()
logger.info('loading streams...')
const streamsStorage = new Storage(STREAMS_DIR)
const parser = new PlaylistParser({
storage: streamsStorage
})
const files = await streamsStorage.list('**/*.m3u')
const streams = await parser.parse(files)
const streamsGroupedByUrl = streams.groupBy((stream: Stream) => stream.url)
const streamsGroupedByChannel = streams.groupBy((stream: Stream) => stream.channel)
const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
logger.info('checking streams:remove requests...')
const removeRequests = issues.filter(issue =>
issue.labels.find((label: string) => label === 'streams:remove')
)
removeRequests.forEach((issue: Issue) => {
const streamUrls = issue.data.getArray('streamUrl') || []
if (!streamUrls.length) {
const result = {
issueNumber: issue.number,
type: 'streams:remove',
streamId: undefined,
streamUrl: undefined,
status: 'missing_link'
}
report.add(result)
} else {
for (const streamUrl of streamUrls) {
const result = {
issueNumber: issue.number,
type: 'streams:remove',
streamId: undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
}
if (streamsGroupedByUrl.missing(streamUrl)) {
result.status = 'wrong_link'
}
report.add(result)
}
}
})
logger.info('checking streams:add requests...')
const addRequests = issues.filter(issue => issue.labels.includes('streams:add'))
const addRequestsBuffer = new Dictionary()
addRequests.forEach((issue: Issue) => {
const streamId = issue.data.getString('streamId') || ''
const streamUrl = issue.data.getString('streamUrl') || ''
const [channelId] = streamId.split('@')
const result = {
issueNumber: issue.number,
type: 'streams:add',
streamId: streamId || undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
}
if (!channelId) result.status = 'missing_id'
else if (!streamUrl) result.status = 'missing_link'
else if (!isURI(streamUrl)) result.status = 'invalid_link'
else if (data.blocklistRecordsGroupedByChannel.has(channelId)) result.status = 'blocked'
else if (data.channelsKeyById.missing(channelId)) result.status = 'wrong_id'
else if (streamsGroupedByUrl.has(streamUrl)) result.status = 'on_playlist'
else if (addRequestsBuffer.has(streamUrl)) result.status = 'duplicate'
else result.status = 'pending'
addRequestsBuffer.set(streamUrl, true)
report.add(result)
})
logger.info('checking streams:edit requests...')
const editRequests = issues.filter(issue =>
issue.labels.find((label: string) => label === 'streams:edit')
)
editRequests.forEach((issue: Issue) => {
const streamId = issue.data.getString('streamId') || ''
const streamUrl = issue.data.getString('streamUrl') || ''
const [channelId] = streamId.split('@')
const result = {
issueNumber: issue.number,
type: 'streams:edit',
streamId: streamId || undefined,
streamUrl: truncate(streamUrl),
status: 'pending'
}
if (!streamUrl) result.status = 'missing_link'
else if (streamsGroupedByUrl.missing(streamUrl)) result.status = 'invalid_link'
else if (channelId && data.channelsKeyById.missing(channelId)) result.status = 'invalid_id'
report.add(result)
})
logger.info('checking channel search requests...')
const channelSearchRequests = issues.filter(issue =>
issue.labels.find((label: string) => label === 'channel search')
)
const channelSearchRequestsBuffer = new Dictionary()
channelSearchRequests.forEach((issue: Issue) => {
const streamId = issue.data.getString('channelId') || ''
const [channelId, feedId] = streamId.split('@')
const result = {
issueNumber: issue.number,
type: 'channel search',
streamId: streamId || undefined,
streamUrl: undefined,
status: 'pending'
}
if (!channelId) result.status = 'missing_id'
else if (data.channelsKeyById.missing(channelId)) result.status = 'invalid_id'
else if (channelSearchRequestsBuffer.has(streamId)) result.status = 'duplicate'
else if (data.blocklistRecordsGroupedByChannel.has(channelId)) result.status = 'blocked'
else if (streamsGroupedById.has(streamId)) result.status = 'fulfilled'
else if (!feedId && streamsGroupedByChannel.has(channelId)) result.status = 'fulfilled'
else {
const channelData = data.channelsKeyById.get(channelId)
if (channelData && channelData.isClosed()) result.status = 'closed'
}
channelSearchRequestsBuffer.set(streamId, true)
report.add(result)
})
report = report.sortBy(item => item.issueNumber).filter(item => item.status !== 'pending')
console.table(report.all())
}
main()

16
scripts/core/apiClient.ts
Unescape Escape View File

@ -1,16 +0,0 @@
import axios, { AxiosInstance, AxiosResponse, AxiosRequestConfig } from 'axios'
export class ApiClient {
instance: AxiosInstance
constructor() {
this.instance = axios.create({
baseURL: 'https://iptv-org.github.io/api',
responseType: 'stream'
})
}
get(url: string, options: AxiosRequestConfig): Promise<AxiosResponse> {
return this.instance.get(url, options)
}
}

44
scripts/core/cliTable.ts
Unescape Escape View File

@ -1,22 +1,22 @@
import { Table } from 'console-table-printer'
import { ComplexOptions } from 'console-table-printer/dist/src/models/external-table'
export class CliTable {
table: Table
constructor(options?: ComplexOptions | string[]) {
this.table = new Table(options)
}
append(row) {
this.table.addRow(row)
}
render() {
this.table.printTable()
}
toString() {
return this.table.render()
}
}
import { ComplexOptions } from 'console-table-printer/dist/src/models/external-table'
import { Table } from 'console-table-printer'
export class CliTable {
table: Table
constructor(options?: ComplexOptions | string[]) {
this.table = new Table(options)
}
append(row) {
this.table.addRow(row)
}
render() {
this.table.printTable()
}
toString() {
return this.table.render()
}
}

113
scripts/core/dataLoader.ts
Unescape Escape View File

@ -1,113 +0,0 @@
import { ApiClient } from './apiClient'
import { Storage } from '@freearhey/core'
import cliProgress, { MultiBar } from 'cli-progress'
import type { DataLoaderProps, DataLoaderData } from '../types/dataLoader'
const formatBytes = (bytes: number) => {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]
}
export class DataLoader {
client: ApiClient
storage: Storage
progressBar: MultiBar
constructor(props: DataLoaderProps) {
this.client = new ApiClient()
this.storage = props.storage
this.progressBar = new cliProgress.MultiBar({
stopOnComplete: true,
hideCursor: true,
forceRedraw: true,
barsize: 36,
format(options, params, payload) {
const filename = payload.filename.padEnd(18, ' ')
const barsize = options.barsize || 40
const percent = (params.progress * 100).toFixed(2)
const speed = payload.speed ? formatBytes(payload.speed) + '/s' : 'N/A'
const total = formatBytes(params.total)
const completeSize = Math.round(params.progress * barsize)
const incompleteSize = barsize - completeSize
const bar =
options.barCompleteString && options.barIncompleteString
? options.barCompleteString.substr(0, completeSize) +
options.barGlue +
options.barIncompleteString.substr(0, incompleteSize)
: '-'.repeat(barsize)
return `${filename} [${bar}] ${percent}% | ETA: ${params.eta}s | ${total} | ${speed}`
}
})
}
async load(): Promise<DataLoaderData> {
const [
countries,
regions,
subdivisions,
languages,
categories,
blocklist,
channels,
feeds,
logos,
timezones,
guides,
streams,
cities
] = await Promise.all([
this.storage.json('countries.json'),
this.storage.json('regions.json'),
this.storage.json('subdivisions.json'),
this.storage.json('languages.json'),
this.storage.json('categories.json'),
this.storage.json('blocklist.json'),
this.storage.json('channels.json'),
this.storage.json('feeds.json'),
this.storage.json('logos.json'),
this.storage.json('timezones.json'),
this.storage.json('guides.json'),
this.storage.json('streams.json'),
this.storage.json('cities.json')
])
return {
countries,
regions,
subdivisions,
languages,
categories,
blocklist,
channels,
feeds,
logos,
timezones,
guides,
streams,
cities
}
}
async download(filename: string) {
if (!this.storage || !this.progressBar) return
const stream = await this.storage.createStream(filename)
const progressBar = this.progressBar.create(0, 0, { filename })
this.client
.get(filename, {
responseType: 'stream',
onDownloadProgress({ total, loaded, rate }) {
if (total) progressBar.setTotal(total)
progressBar.update(loaded, { speed: rate })
}
})
.then(response => {
response.data.pipe(stream)
})
}
}

165
scripts/core/dataProcessor.ts
Unescape Escape View File

@ -1,165 +0,0 @@
import { DataProcessorData } from '../types/dataProcessor'
import { DataLoaderData } from '../types/dataLoader'
import { Collection } from '@freearhey/core'
import {
BlocklistRecord,
Subdivision,
Category,
Language,
Timezone,
Channel,
Country,
Region,
Stream,
Guide,
City,
Feed,
Logo
} from '../models'
export class DataProcessor {
process(data: DataLoaderData): DataProcessorData {
let regions = new Collection(data.regions).map(data => new Region(data))
let regionsKeyByCode = regions.keyBy((region: Region) => region.code)
const categories = new Collection(data.categories).map(data => new Category(data))
const categoriesKeyById = categories.keyBy((category: Category) => category.id)
const languages = new Collection(data.languages).map(data => new Language(data))
const languagesKeyByCode = languages.keyBy((language: Language) => language.code)
let subdivisions = new Collection(data.subdivisions).map(data => new Subdivision(data))
let subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code)
let subdivisionsGroupedByCountryCode = subdivisions.groupBy(
(subdivision: Subdivision) => subdivision.countryCode
)
let countries = new Collection(data.countries).map(data => new Country(data))
let countriesKeyByCode = countries.keyBy((country: Country) => country.code)
const cities = new Collection(data.cities).map(data =>
new City(data)
.withRegions(regions)
.withCountry(countriesKeyByCode)
.withSubdivision(subdivisionsKeyByCode)
)
const citiesKeyByCode = cities.keyBy((city: City) => city.code)
const citiesGroupedByCountryCode = cities.groupBy((city: City) => city.countryCode)
const citiesGroupedBySubdivisionCode = cities.groupBy((city: City) => city.subdivisionCode)
const timezones = new Collection(data.timezones).map(data =>
new Timezone(data).withCountries(countriesKeyByCode)
)
const timezonesKeyById = timezones.keyBy((timezone: Timezone) => timezone.id)
const blocklistRecords = new Collection(data.blocklist).map(data => new BlocklistRecord(data))
const blocklistRecordsGroupedByChannelId = blocklistRecords.groupBy(
(blocklistRecord: BlocklistRecord) => blocklistRecord.channelId
)
let channels = new Collection(data.channels).map(data => new Channel(data))
let channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
let feeds = new Collection(data.feeds).map(data => new Feed(data))
let feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
let feedsGroupedById = feeds.groupBy((feed: Feed) => feed.id)
const logos = new Collection(data.logos).map(data => new Logo(data).withFeed(feedsGroupedById))
const logosGroupedByChannelId = logos.groupBy((logo: Logo) => logo.channelId)
const logosGroupedByStreamId = logos.groupBy((logo: Logo) => logo.getStreamId())
const streams = new Collection(data.streams).map(data =>
new Stream(data).withLogos(logosGroupedByStreamId)
)
const streamsGroupedById = streams.groupBy((stream: Stream) => stream.getId())
const guides = new Collection(data.guides).map(data => new Guide(data))
const guidesGroupedByStreamId = guides.groupBy((guide: Guide) => guide.getStreamId())
regions = regions.map((region: Region) =>
region
.withCountries(countriesKeyByCode)
.withRegions(regions)
.withSubdivisions(subdivisions)
.withCities(cities)
)
regionsKeyByCode = regions.keyBy((region: Region) => region.code)
countries = countries.map((country: Country) =>
country
.withCities(citiesGroupedByCountryCode)
.withSubdivisions(subdivisionsGroupedByCountryCode)
.withRegions(regions)
.withLanguage(languagesKeyByCode)
)
countriesKeyByCode = countries.keyBy((country: Country) => country.code)
subdivisions = subdivisions.map((subdivision: Subdivision) =>
subdivision
.withCities(citiesGroupedBySubdivisionCode)
.withCountry(countriesKeyByCode)
.withRegions(regions)
.withParent(subdivisionsKeyByCode)
)
subdivisionsKeyByCode = subdivisions.keyBy((subdivision: Subdivision) => subdivision.code)
subdivisionsGroupedByCountryCode = subdivisions.groupBy(
(subdivision: Subdivision) => subdivision.countryCode
)
channels = channels.map((channel: Channel) =>
channel
.withFeeds(feedsGroupedByChannelId)
.withLogos(logosGroupedByChannelId)
.withCategories(categoriesKeyById)
.withCountry(countriesKeyByCode)
.withSubdivision(subdivisionsKeyByCode)
.withCategories(categoriesKeyById)
)
channelsKeyById = channels.keyBy((channel: Channel) => channel.id)
feeds = feeds.map((feed: Feed) =>
feed
.withChannel(channelsKeyById)
.withLanguages(languagesKeyByCode)
.withTimezones(timezonesKeyById)
.withBroadcastArea(
citiesKeyByCode,
subdivisionsKeyByCode,
countriesKeyByCode,
regionsKeyByCode
)
)
feedsGroupedByChannelId = feeds.groupBy((feed: Feed) => feed.channelId)
feedsGroupedById = feeds.groupBy((feed: Feed) => feed.id)
return {
blocklistRecordsGroupedByChannelId,
subdivisionsGroupedByCountryCode,
feedsGroupedByChannelId,
guidesGroupedByStreamId,
logosGroupedByStreamId,
subdivisionsKeyByCode,
countriesKeyByCode,
languagesKeyByCode,
streamsGroupedById,
categoriesKeyById,
timezonesKeyById,
regionsKeyByCode,
blocklistRecords,
channelsKeyById,
citiesKeyByCode,
subdivisions,
categories,
countries,
languages,
timezones,
channels,
regions,
streams,
cities,
guides,
feeds,
logos
}
}
}

96
scripts/core/htmlTable.ts
Unescape Escape View File

@ -1,46 +1,50 @@
type Column = {
name: string
nowrap?: boolean
align?: string
}
type DataItem = string[]
export class HTMLTable {
data: DataItem[]
columns: Column[]
constructor(data: DataItem[], columns: Column[]) {
this.data = data
this.columns = columns
}
toString() {
let output = '<table>\r\n'
output += ' <thead>\r\n <tr>'
for (const column of this.columns) {
output += `<th align="left">${column.name}</th>`
}
output += '</tr>\r\n </thead>\r\n'
output += ' <tbody>\r\n'
for (const item of this.data) {
output += ' <tr>'
let i = 0
for (const prop in item) {
const column = this.columns[i]
const nowrap = column.nowrap ? ' nowrap' : ''
const align = column.align ? ` align="${column.align}"` : ''
output += `<td${align}${nowrap}>${item[prop]}</td>`
i++
}
output += '</tr>\r\n'
}
output += ' </tbody>\r\n'
output += '</table>'
return output
}
}
import { Collection } from '@freearhey/core'
export type HTMLTableColumn = {
name: string
nowrap?: boolean
align?: string
}
export type HTMLTableItem = string[]
export class HTMLTable {
data: Collection<HTMLTableItem>
columns: Collection<HTMLTableColumn>
constructor(data: Collection<HTMLTableItem>, columns: Collection<HTMLTableColumn>) {
this.data = data
this.columns = columns
}
toString() {
let output = '<table>\r\n'
output += ' <thead>\r\n <tr>'
this.columns.forEach((column: HTMLTableColumn) => {
output += `<th align="left">${column.name}</th>`
})
output += '</tr>\r\n </thead>\r\n'
output += ' <tbody>\r\n'
this.data.forEach((item: HTMLTableItem) => {
output += ' <tr>'
let i = 0
for (const prop in item) {
const column = this.columns.all()[i]
const nowrap = column.nowrap ? ' nowrap' : ''
const align = column.align ? ` align="${column.align}"` : ''
output += `<td${align}${nowrap}>${item[prop]}</td>`
i++
}
output += '</tr>\r\n'
})
output += ' </tbody>\r\n'
output += '</table>'
return output
}
}

25
scripts/core/index.ts
Unescape Escape View File

@ -1,14 +1,11 @@
export * from './apiClient'
export * from './cliTable'
export * from './dataProcessor'
export * from './dataLoader'
export * from './htmlTable'
export * from './issueData'
export * from './issueLoader'
export * from './issueParser'
export * from './logParser'
export * from './markdown'
export * from './numberParser'
export * from './playlistParser'
export * from './proxyParser'
export * from './streamTester'
export * from './cliTable'
export * from './htmlTable'
export * from './issueData'
export * from './issueLoader'
export * from './issueParser'
export * from './logParser'
export * from './markdown'
export * from './numberParser'
export * from './playlistParser'
export * from './proxyParser'
export * from './streamTester'

70
scripts/core/issueData.ts
Unescape Escape View File

@ -1,34 +1,36 @@
import { Dictionary } from '@freearhey/core'
export class IssueData {
_data: Dictionary
constructor(data: Dictionary) {
this._data = data
}
has(key: string): boolean {
return this._data.has(key)
}
missing(key: string): boolean {
return this._data.missing(key) || this._data.get(key) === undefined
}
getBoolean(key: string): boolean {
return Boolean(this._data.get(key))
}
getString(key: string): string | undefined {
const deleteSymbol = '~'
return this._data.get(key) === deleteSymbol ? '' : this._data.get(key)
}
getArray(key: string): string[] | undefined {
const deleteSymbol = '~'
if (this._data.missing(key)) return undefined
return this._data.get(key) === deleteSymbol ? [] : this._data.get(key).split('\r\n')
}
}
import { Dictionary } from '@freearhey/core'
export class IssueData {
_data: Dictionary<string>
constructor(data: Dictionary<string>) {
this._data = data
}
has(key: string): boolean {
return this._data.has(key)
}
missing(key: string): boolean {
return this._data.missing(key) || this._data.get(key) === undefined
}
getBoolean(key: string): boolean {
return Boolean(this._data.get(key))
}
getString(key: string): string | undefined {
const deleteSymbol = '~'
return this._data.get(key) === deleteSymbol ? '' : this._data.get(key)
}
getArray(key: string): string[] | undefined {
const deleteSymbol = '~'
if (this._data.missing(key)) return undefined
const value = this._data.get(key)
return !value || value === deleteSymbol ? [] : value.split('\r\n')
}
}

74
scripts/core/issueLoader.ts
Unescape Escape View File

@ -1,37 +1,37 @@
import { Collection } from '@freearhey/core'
import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
import { paginateRest } from '@octokit/plugin-paginate-rest'
import { Octokit } from '@octokit/core'
import { IssueParser } from './'
import { TESTING, OWNER, REPO } from '../constants'
const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
const octokit = new CustomOctokit()
export class IssueLoader {
async load(props?: { labels: string | string[] }) {
let labels = ''
if (props && props.labels) {
labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels
}
let issues: object[] = []
if (TESTING) {
issues = (await import('../../tests/__data__/input/issues.js')).default
} else {
issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
owner: OWNER,
repo: REPO,
per_page: 100,
labels,
status: 'open',
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
})
}
const parser = new IssueParser()
return new Collection(issues).map(parser.parse)
}
}
import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'
import { paginateRest } from '@octokit/plugin-paginate-rest'
import { TESTING, OWNER, REPO } from '../constants'
import { Collection } from '@freearhey/core'
import { Octokit } from '@octokit/core'
import { IssueParser } from './'
const CustomOctokit = Octokit.plugin(paginateRest, restEndpointMethods)
const octokit = new CustomOctokit()
export class IssueLoader {
async load(props?: { labels: string | string[] }) {
let labels = ''
if (props && props.labels) {
labels = Array.isArray(props.labels) ? props.labels.join(',') : props.labels
}
let issues: object[] = []
if (TESTING) {
issues = (await import('../../tests/__data__/input/issues.js')).default
} else {
issues = await octokit.paginate(octokit.rest.issues.listForRepo, {
owner: OWNER,
repo: REPO,
per_page: 100,
labels,
status: 'open',
headers: {
'X-GitHub-Api-Version': '2022-11-28'
}
})
}
const parser = new IssueParser()
return new Collection(issues).map(parser.parse)
}
}

96
scripts/core/issueParser.ts
Unescape Escape View File

@ -1,48 +1,48 @@
import { Dictionary } from '@freearhey/core'
import { Issue } from '../models'
import { IssueData } from './issueData'
const FIELDS = new Dictionary({
'Stream ID': 'streamId',
'Channel ID': 'channelId',
'Feed ID': 'feedId',
'Stream URL': 'streamUrl',
'New Stream URL': 'newStreamUrl',
Label: 'label',
Quality: 'quality',
'HTTP User-Agent': 'httpUserAgent',
'HTTP User Agent': 'httpUserAgent',
'HTTP Referrer': 'httpReferrer',
'What happened to the stream?': 'reason',
Reason: 'reason',
Notes: 'notes',
Directives: 'directives'
})
export class IssueParser {
parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue {
const fields = typeof issue.body === 'string' ? issue.body.split('###') : []
const data = new Dictionary()
fields.forEach((field: string) => {
const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : []
let _label = parsed.shift()
_label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : ''
let _value = parsed.join('\r\n')
_value = _value ? _value.trim() : ''
if (!_label || !_value) return data
const id: string = FIELDS.get(_label)
const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
if (!id) return
data.set(id, value)
})
const labels = issue.labels.map(label => label.name)
return new Issue({ number: issue.number, labels, data: new IssueData(data) })
}
}
import { Dictionary } from '@freearhey/core'
import { IssueData } from './issueData'
import { Issue } from '../models'
const FIELDS = new Dictionary({
'Stream ID': 'streamId',
'Channel ID': 'channelId',
'Feed ID': 'feedId',
'Stream URL': 'streamUrl',
'New Stream URL': 'newStreamUrl',
Label: 'label',
Quality: 'quality',
'HTTP User-Agent': 'httpUserAgent',
'HTTP User Agent': 'httpUserAgent',
'HTTP Referrer': 'httpReferrer',
'What happened to the stream?': 'reason',
Reason: 'reason',
Notes: 'notes',
Directives: 'directives'
})
export class IssueParser {
parse(issue: { number: number; body: string; labels: { name: string }[] }): Issue {
const fields = typeof issue.body === 'string' ? issue.body.split('###') : []
const data = new Dictionary<string>()
fields.forEach((field: string) => {
const parsed = typeof field === 'string' ? field.split(/\r?\n/).filter(Boolean) : []
let _label = parsed.shift()
_label = _label ? _label.replace(/ \(optional\)| \(required\)/, '').trim() : ''
let _value = parsed.join('\r\n')
_value = _value ? _value.trim() : ''
if (!_label || !_value) return data
const id = FIELDS.get(_label)
const value: string = _value === '_No response_' || _value === 'None' ? '' : _value
if (!id) return
data.set(id, value)
})
const labels = issue.labels.map(label => label.name)
return new Issue({ number: issue.number, labels, data: new IssueData(data) })
}
}

90
scripts/core/markdown.ts
Unescape Escape View File

@ -1,45 +1,45 @@
import fs from 'fs'
import path from 'path'
type MarkdownConfig = {
build: string
template: string
}
export class Markdown {
build: string
template: string
constructor(config: MarkdownConfig) {
this.build = config.build
this.template = config.template
}
compile() {
const workingDir = process.cwd()
const templatePath = path.resolve(workingDir, this.template)
const template = fs.readFileSync(templatePath, 'utf8')
const processedContent = this.processIncludes(template, workingDir)
if (this.build) {
const outputPath = path.resolve(workingDir, this.build)
fs.writeFileSync(outputPath, processedContent, 'utf8')
}
}
private processIncludes(template: string, baseDir: string): string {
const includeRegex = /#include\s+"([^"]+)"/g
return template.replace(includeRegex, (match, includePath) => {
try {
const fullPath = path.resolve(baseDir, includePath)
const includeContent = fs.readFileSync(fullPath, 'utf8')
return this.processIncludes(includeContent, baseDir)
} catch (error) {
console.warn(`Warning: Could not include file ${includePath}: ${error}`)
return match
}
})
}
}
import path from 'path'
import fs from 'fs'
type MarkdownConfig = {
build: string
template: string
}
export class Markdown {
build: string
template: string
constructor(config: MarkdownConfig) {
this.build = config.build
this.template = config.template
}
compile() {
const workingDir = process.cwd()
const templatePath = path.resolve(workingDir, this.template)
const template = fs.readFileSync(templatePath, 'utf8')
const processedContent = this.processIncludes(template, workingDir)
if (this.build) {
const outputPath = path.resolve(workingDir, this.build)
fs.writeFileSync(outputPath, processedContent, 'utf8')
}
}
private processIncludes(template: string, baseDir: string): string {
const includeRegex = /#include\s+"([^"]+)"/g
return template.replace(includeRegex, (match, includePath) => {
try {
const fullPath = path.resolve(baseDir, includePath)
const includeContent = fs.readFileSync(fullPath, 'utf8')
return this.processIncludes(includeContent, baseDir)
} catch (error) {
console.warn(`Warning: Could not include file ${includePath}: ${error}`)
return match
}
})
}
}

103
scripts/core/playlistParser.ts
Unescape Escape View File

@ -1,60 +1,43 @@
import { Collection, Storage, Dictionary } from '@freearhey/core'
import parser from 'iptv-playlist-parser'
import { Stream } from '../models'
type PlaylistPareserProps = {
storage: Storage
feedsGroupedByChannelId: Dictionary
logosGroupedByStreamId: Dictionary
channelsKeyById: Dictionary
}
export class PlaylistParser {
storage: Storage
feedsGroupedByChannelId: Dictionary
logosGroupedByStreamId: Dictionary
channelsKeyById: Dictionary
constructor({
storage,
feedsGroupedByChannelId,
logosGroupedByStreamId,
channelsKeyById
}: PlaylistPareserProps) {
this.storage = storage
this.feedsGroupedByChannelId = feedsGroupedByChannelId
this.logosGroupedByStreamId = logosGroupedByStreamId
this.channelsKeyById = channelsKeyById
}
async parse(files: string[]): Promise<Collection> {
let streams = new Collection()
for (const filepath of files) {
if (!this.storage.existsSync(filepath)) continue
const _streams: Collection = await this.parseFile(filepath)
streams = streams.concat(_streams)
}
return streams
}
async parseFile(filepath: string): Promise<Collection> {
const content = await this.storage.load(filepath)
const parsed: parser.Playlist = parser.parse(content)
const streams = new Collection(parsed.items).map((data: parser.PlaylistItem) => {
const stream = new Stream()
.fromPlaylistItem(data)
.withFeed(this.feedsGroupedByChannelId)
.withChannel(this.channelsKeyById)
.withLogos(this.logosGroupedByStreamId)
.setFilepath(filepath)
return stream
})
return streams
}
}
import { Storage } from '@freearhey/storage-js'
import { Collection } from '@freearhey/core'
import parser from 'iptv-playlist-parser'
import { Stream } from '../models'
type PlaylistPareserProps = {
storage: Storage
}
export class PlaylistParser {
storage: Storage
constructor({ storage }: PlaylistPareserProps) {
this.storage = storage
}
async parse(files: string[]): Promise<Collection<Stream>> {
const parsed = new Collection<Stream>()
for (const filepath of files) {
if (!this.storage.existsSync(filepath)) continue
const _parsed: Collection<Stream> = await this.parseFile(filepath)
parsed.concat(_parsed)
}
return parsed
}
async parseFile(filepath: string): Promise<Collection<Stream>> {
const content = await this.storage.load(filepath)
const parsed: parser.Playlist = parser.parse(content)
const streams = new Collection<Stream>()
parsed.items.forEach((data: parser.PlaylistItem) => {
const stream = Stream.fromPlaylistItem(data)
stream.filepath = filepath
streams.add(stream)
})
return streams
}
}

242
scripts/core/streamTester.ts
Unescape Escape View File

@ -1,117 +1,125 @@
import { Stream } from '../models'
import { TESTING } from '../constants'
import mediaInfoFactory from 'mediainfo.js'
import axios, { AxiosInstance, AxiosProxyConfig, AxiosRequestConfig } from 'axios'
import { ProxyParser } from './proxyParser.js'
import { OptionValues } from 'commander'
import { SocksProxyAgent } from 'socks-proxy-agent'
export type TestResult = {
status: {
ok: boolean
code: string
}
}
export type StreamTesterProps = {
options: OptionValues
}
export class StreamTester {
client: AxiosInstance
options: OptionValues
constructor({ options }: StreamTesterProps) {
const proxyParser = new ProxyParser()
let request: AxiosRequestConfig = {
responseType: 'arraybuffer'
}
if (options.proxy !== undefined) {
const proxy = proxyParser.parse(options.proxy) as AxiosProxyConfig
if (
proxy.protocol &&
['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol))
) {
const socksProxyAgent = new SocksProxyAgent(options.proxy)
request = { ...request, ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } }
} else {
request = { ...request, ...{ proxy } }
}
}
this.client = axios.create(request)
this.options = options
}
async test(stream: Stream): Promise<TestResult> {
if (TESTING) {
const results = (await import('../../tests/__data__/input/playlist_test/results.js')).default
return results[stream.url as keyof typeof results]
} else {
try {
const res = await this.client(stream.url, {
signal: AbortSignal.timeout(this.options.timeout),
headers: {
'User-Agent': stream.getUserAgent() || 'Mozilla/5.0',
Referer: stream.getReferrer()
}
})
const mediainfo = await mediaInfoFactory({ format: 'object' })
const buffer = await res.data
const result = await mediainfo.analyzeData(
() => buffer.byteLength,
(size: any, offset: number | undefined) =>
Buffer.from(buffer).subarray(offset, offset + size)
)
if (result && result.media && result.media.track.length > 0) {
return {
status: {
ok: true,
code: 'OK'
}
}
} else {
return {
status: {
ok: false,
code: 'NO_VIDEO'
}
}
}
} catch (error: any) {
let code = 'UNKNOWN_ERROR'
if (error.name === 'CanceledError') {
code = 'TIMEOUT'
} else if (error.name === 'AxiosError') {
if (error.response) {
const status = error.response?.status
const statusText = error.response?.statusText.toUpperCase().replace(/\s+/, '_')
code = `HTTP_${status}_${statusText}`
} else {
code = `AXIOS_${error.code}`
}
} else if (error.cause) {
const cause = error.cause as Error & { code?: string }
if (cause.code) {
code = cause.code
} else {
code = cause.name
}
}
return {
status: {
ok: false,
code
}
}
}
}
}
}
import axios, { AxiosInstance, AxiosProxyConfig, AxiosRequestConfig, AxiosResponse } from 'axios'
import { SocksProxyAgent } from 'socks-proxy-agent'
import { ProxyParser } from './proxyParser.js'
import mediaInfoFactory from 'mediainfo.js'
import { OptionValues } from 'commander'
import { TESTING } from '../constants'
import { Stream } from '../models'
export type StreamTesterResult = {
status: {
ok: boolean
code: string
}
}
export type StreamTesterError = {
name: string
code?: string
cause?: Error & { code?: string }
response?: AxiosResponse
}
export type StreamTesterProps = {
options: OptionValues
}
export class StreamTester {
client: AxiosInstance
options: OptionValues
constructor({ options }: StreamTesterProps) {
const proxyParser = new ProxyParser()
let request: AxiosRequestConfig = {
responseType: 'arraybuffer'
}
if (options.proxy !== undefined) {
const proxy = proxyParser.parse(options.proxy) as AxiosProxyConfig
if (
proxy.protocol &&
['socks', 'socks5', 'socks5h', 'socks4', 'socks4a'].includes(String(proxy.protocol))
) {
const socksProxyAgent = new SocksProxyAgent(options.proxy)
request = { ...request, ...{ httpAgent: socksProxyAgent, httpsAgent: socksProxyAgent } }
} else {
request = { ...request, ...{ proxy } }
}
}
this.client = axios.create(request)
this.options = options
}
async test(stream: Stream): Promise<StreamTesterResult> {
if (TESTING) {
const results = (await import('../../tests/__data__/input/playlist_test/results.js')).default
return results[stream.url as keyof typeof results]
} else {
try {
const res = await this.client(stream.url, {
signal: AbortSignal.timeout(this.options.timeout),
headers: {
'User-Agent': stream.user_agent || 'Mozilla/5.0',
Referer: stream.referrer
}
})
const mediainfo = await mediaInfoFactory({ format: 'object' })
const buffer = await res.data
const result = await mediainfo.analyzeData(
() => buffer.byteLength,
(size: number, offset: number) => Buffer.from(buffer).subarray(offset, offset + size)
)
if (result && result.media && result.media.track.length > 0) {
return {
status: {
ok: true,
code: 'OK'
}
}
} else {
return {
status: {
ok: false,
code: 'NO_VIDEO'
}
}
}
} catch (err: unknown) {
const error = err as StreamTesterError
let code = 'UNKNOWN_ERROR'
if (error.name === 'CanceledError') {
code = 'TIMEOUT'
} else if (error.name === 'AxiosError') {
if (error.response) {
const status = error.response?.status
const statusText = error.response?.statusText.toUpperCase().replace(/\s+/, '_')
code = `HTTP_${status}_${statusText}`
} else {
code = `AXIOS_${error.code}`
}
} else if (error.cause) {
const cause = error.cause
if (cause.code) {
code = cause.code
} else {
code = cause.name
}
}
return {
status: {
ok: false,
code
}
}
}
}
}
}

114
scripts/generators/categoriesGenerator.ts
Unescape Escape View File

@ -1,54 +1,60 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Category, Playlist } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type CategoriesGeneratorProps = {
streams: Collection
categories: Collection
logFile: File
}
export class CategoriesGenerator implements Generator {
streams: Collection
categories: Collection
storage: Storage
logFile: File
constructor({ streams, categories, logFile }: CategoriesGeneratorProps) {
this.streams = streams.clone()
this.categories = categories
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate() {
const streams = this.streams.orderBy([(stream: Stream) => stream.getTitle()])
this.categories.forEach(async (category: Category) => {
const categoryStreams = streams
.filter((stream: Stream) => stream.hasCategory(category))
.map((stream: Stream) => {
const groupTitle = stream.getCategoryNames().join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
})
const playlist = new Playlist(categoryStreams, { public: true })
const filepath = `categories/${category.id}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + EOL
)
})
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasCategories())
const playlist = new Playlist(undefinedStreams, { public: true })
const filepath = 'categories/undefined.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + EOL
)
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Collection } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type CategoriesGeneratorProps = {
streams: Collection<Stream>
categories: Collection<sdk.Models.Category>
logFile: File
}
export class CategoriesGenerator implements Generator {
streams: Collection<Stream>
categories: Collection<sdk.Models.Category>
storage: Storage
logFile: File
constructor({ streams, categories, logFile }: CategoriesGeneratorProps) {
this.streams = streams.clone()
this.categories = categories
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate() {
const streams = this.streams.sortBy([(stream: Stream) => stream.title])
this.categories.forEach(async (category: sdk.Models.Category) => {
const categoryStreams = streams
.filter((stream: Stream) => stream.hasCategory(category))
.map((stream: Stream) => {
const groupTitle = stream
.getCategories()
.map(category => category.name)
.sort()
.join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
})
const playlist = new Playlist(categoryStreams, { public: true })
const filepath = `categories/${category.id}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + EOL
)
})
const undefinedStreams = streams.filter((stream: Stream) => stream.getCategories().isEmpty())
const playlist = new Playlist(undefinedStreams, { public: true })
const filepath = 'categories/undefined.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'category', filepath, count: playlist.streams.count() }) + EOL
)
}
}

97
scripts/generators/citiesGenerator.ts
Unescape Escape View File

@ -1,43 +1,54 @@
import { City, Stream, Playlist } from '../models'
import { Collection, Storage, File } from '@freearhey/core'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type CitiesGeneratorProps = {
streams: Collection
cities: Collection
logFile: File
}
export class CitiesGenerator implements Generator {
streams: Collection
cities: Collection
storage: Storage
logFile: File
constructor({ streams, cities, logFile }: CitiesGeneratorProps) {
this.streams = streams.clone()
this.cities = cities
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
this.cities.forEach(async (city: City) => {
const cityStreams = streams.filter((stream: Stream) => stream.isBroadcastInCity(city))
if (cityStreams.isEmpty()) return
const playlist = new Playlist(cityStreams, { public: true })
const filepath = `cities/${city.code.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'city', filepath, count: playlist.streams.count() }) + EOL
)
})
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type CitiesGeneratorProps = {
streams: Collection<Stream>
cities: Collection<sdk.Models.City>
logFile: File
}
export class CitiesGenerator implements Generator {
streams: Collection<Stream>
cities: Collection<sdk.Models.City>
storage: Storage
logFile: File
constructor({ streams, cities, logFile }: CitiesGeneratorProps) {
this.streams = streams.clone()
this.cities = cities
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
const streamsGroupedByCityCode = {}
streams.forEach((stream: Stream) => {
stream.getBroadcastCities().forEach((city: sdk.Models.City) => {
if (streamsGroupedByCityCode[city.code]) {
streamsGroupedByCityCode[city.code].add(stream)
} else {
streamsGroupedByCityCode[city.code] = new Collection<Stream>([stream])
}
})
})
for (const cityCode in streamsGroupedByCityCode) {
const cityStreams = streamsGroupedByCityCode[cityCode]
const playlist = new Playlist(cityStreams, { public: true })
const filepath = `cities/${cityCode.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'city', filepath, count: playlist.streams.count() }) + EOL
)
}
}
}

148
scripts/generators/countriesGenerator.ts
Unescape Escape View File

@ -1,68 +1,80 @@
import { Country, Stream, Playlist } from '../models'
import { Collection, Storage, File } from '@freearhey/core'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type CountriesGeneratorProps = {
streams: Collection
countries: Collection
logFile: File
}
export class CountriesGenerator implements Generator {
streams: Collection
countries: Collection
storage: Storage
logFile: File
constructor({ streams, countries, logFile }: CountriesGeneratorProps) {
this.streams = streams.clone()
this.countries = countries
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
this.countries.forEach(async (country: Country) => {
const countryStreams = streams.filter((stream: Stream) =>
stream.isBroadcastInCountry(country)
)
if (countryStreams.isEmpty()) return
const playlist = new Playlist(countryStreams, { public: true })
const filepath = `countries/${country.code.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'country', filepath, count: playlist.streams.count() }) + EOL
)
})
const internationalStreams = streams.filter((stream: Stream) => stream.isInternational())
const internationalPlaylist = new Playlist(internationalStreams, { public: true })
const internationalFilepath = 'countries/int.m3u'
await this.storage.save(internationalFilepath, internationalPlaylist.toString())
this.logFile.append(
JSON.stringify({
type: 'country',
filepath: internationalFilepath,
count: internationalPlaylist.streams.count()
}) + EOL
)
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasBroadcastArea())
const undefinedPlaylist = new Playlist(undefinedStreams, { public: true })
const undefinedFilepath = 'countries/undefined.m3u'
await this.storage.save(undefinedFilepath, undefinedPlaylist.toString())
this.logFile.append(
JSON.stringify({
type: 'country',
filepath: undefinedFilepath,
count: undefinedPlaylist.streams.count()
}) + EOL
)
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type CountriesGeneratorProps = {
streams: Collection<Stream>
countries: Collection<sdk.Models.Country>
logFile: File
}
export class CountriesGenerator implements Generator {
streams: Collection<Stream>
countries: Collection<sdk.Models.Country>
storage: Storage
logFile: File
constructor({ streams, countries, logFile }: CountriesGeneratorProps) {
this.streams = streams.clone()
this.countries = countries
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
const streamsGroupedByCountryCode = {}
streams.forEach((stream: Stream) => {
stream.getBroadcastCountries().forEach((country: sdk.Models.Country) => {
if (streamsGroupedByCountryCode[country.code]) {
streamsGroupedByCountryCode[country.code].add(stream)
} else {
streamsGroupedByCountryCode[country.code] = new Collection<Stream>([stream])
}
})
})
for (const countryCode in streamsGroupedByCountryCode) {
const countryStreams = streamsGroupedByCountryCode[countryCode]
const playlist = new Playlist(countryStreams, { public: true })
const filepath = `countries/${countryCode.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'country', filepath, count: playlist.streams.count() }) + EOL
)
}
const internationalStreams = streams.filter((stream: Stream) => stream.isInternational())
const internationalPlaylist = new Playlist(internationalStreams, { public: true })
const internationalFilepath = 'countries/int.m3u'
await this.storage.save(internationalFilepath, internationalPlaylist.toString())
this.logFile.append(
JSON.stringify({
type: 'country',
filepath: internationalFilepath,
count: internationalPlaylist.streams.count()
}) + EOL
)
const undefinedStreams = streams.filter((stream: Stream) =>
stream.getBroadcastAreaCodes().isEmpty()
)
const undefinedPlaylist = new Playlist(undefinedStreams, { public: true })
const undefinedFilepath = 'countries/undefined.m3u'
await this.storage.save(undefinedFilepath, undefinedPlaylist.toString())
this.logFile.append(
JSON.stringify({
type: 'country',
filepath: undefinedFilepath,
count: undefinedPlaylist.streams.count()
}) + EOL
)
}
}

25
scripts/generators/index.ts
Unescape Escape View File

@ -1,13 +1,12 @@
export * from './categoriesGenerator'
export * from './citiesGenerator'
export * from './countriesGenerator'
export * from './indexCategoryGenerator'
export * from './indexCountryGenerator'
export * from './indexGenerator'
export * from './indexLanguageGenerator'
export * from './indexNsfwGenerator'
export * from './languagesGenerator'
export * from './rawGenerator'
export * from './regionsGenerator'
export * from './sourcesGenerator'
export * from './subdivisionsGenerator'
export * from './categoriesGenerator'
export * from './citiesGenerator'
export * from './countriesGenerator'
export * from './indexCategoryGenerator'
export * from './indexCountryGenerator'
export * from './indexGenerator'
export * from './indexLanguageGenerator'
export * from './languagesGenerator'
export * from './rawGenerator'
export * from './regionsGenerator'
export * from './sourcesGenerator'
export * from './subdivisionsGenerator'

111
scripts/generators/indexCategoryGenerator.ts
Unescape Escape View File

@ -1,55 +1,56 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Playlist, Category } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type IndexCategoryGeneratorProps = {
streams: Collection
logFile: File
}
export class IndexCategoryGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexCategoryGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy(stream => stream.getTitle())
.filter(stream => stream.isSFW())
let groupedStreams = new Collection()
streams.forEach((stream: Stream) => {
if (!stream.hasCategories()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
stream.getCategories().forEach((category: Category) => {
const streamClone = stream.clone()
streamClone.groupTitle = category.name
groupedStreams.push(streamClone)
})
})
groupedStreams = groupedStreams.orderBy(stream => {
if (stream.groupTitle === 'Undefined') return 'ZZ'
return stream.groupTitle
})
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.category.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type IndexCategoryGeneratorProps = {
streams: Collection<Stream>
logFile: File
}
export class IndexCategoryGenerator implements Generator {
streams: Collection<Stream>
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexCategoryGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams.sortBy(stream => stream.title).filter(stream => stream.isSFW())
let groupedStreams = new Collection<Stream>()
streams.forEach((stream: Stream) => {
const streamCategories = stream.getCategories()
if (streamCategories.isEmpty()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
streamCategories.forEach((category: sdk.Models.Category) => {
const streamClone = stream.clone()
streamClone.groupTitle = category.name
groupedStreams.add(streamClone)
})
})
groupedStreams = groupedStreams.sortBy(stream => {
if (stream.groupTitle === 'Undefined') return 'ZZ'
return stream.groupTitle
})
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.category.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}

130
scripts/generators/indexCountryGenerator.ts
Unescape Escape View File

@ -1,63 +1,67 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Playlist, Country } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type IndexCountryGeneratorProps = {
streams: Collection
logFile: File
}
export class IndexCountryGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexCountryGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
let groupedStreams = new Collection()
this.streams
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
if (!stream.hasBroadcastArea()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
stream.getBroadcastCountries().forEach((country: Country) => {
const streamClone = stream.clone()
streamClone.groupTitle = country.name
groupedStreams.add(streamClone)
})
if (stream.isInternational()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'International'
groupedStreams.add(streamClone)
}
})
groupedStreams = groupedStreams.orderBy((stream: Stream) => {
if (stream.groupTitle === 'International') return 'ZZ'
if (stream.groupTitle === 'Undefined') return 'ZZZ'
return stream.groupTitle
})
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.country.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type IndexCountryGeneratorProps = {
streams: Collection<Stream>
logFile: File
}
export class IndexCountryGenerator implements Generator {
streams: Collection<Stream>
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexCountryGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
let groupedStreams = new Collection<Stream>()
this.streams
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
const broadcastAreaCountries = stream.getBroadcastCountries()
if (stream.getBroadcastAreaCodes().isEmpty()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
broadcastAreaCountries.forEach((country: sdk.Models.Country) => {
const streamClone = stream.clone()
streamClone.groupTitle = country.name
groupedStreams.add(streamClone)
})
if (stream.isInternational()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'International'
groupedStreams.add(streamClone)
}
})
groupedStreams = groupedStreams.sortBy((stream: Stream) => {
if (stream.groupTitle === 'International') return 'ZZ'
if (stream.groupTitle === 'Undefined') return 'ZZZ'
return stream.groupTitle
})
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.country.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}

85
scripts/generators/indexGenerator.ts
Unescape Escape View File

@ -1,40 +1,45 @@
import { Collection, File, Storage } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type IndexGeneratorProps = {
streams: Collection
logFile: File
}
export class IndexGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const sfwStreams = this.streams
.orderBy(stream => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
.map((stream: Stream) => {
const groupTitle = stream.getCategoryNames().join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
})
const playlist = new Playlist(sfwStreams, { public: true })
const filepath = 'index.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
type IndexGeneratorProps = {
streams: Collection<Stream>
logFile: File
}
export class IndexGenerator implements Generator {
streams: Collection<Stream>
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const sfwStreams = this.streams
.sortBy(stream => stream.title)
.filter((stream: Stream) => stream.isSFW())
.map((stream: Stream) => {
const groupTitle = stream
.getCategories()
.map(category => category.name)
.sort()
.join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
})
const playlist = new Playlist(sfwStreams, { public: true })
const filepath = 'index.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}

111
scripts/generators/indexLanguageGenerator.ts
Unescape Escape View File

@ -1,54 +1,57 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Playlist, Language } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type IndexLanguageGeneratorProps = {
streams: Collection
logFile: File
}
export class IndexLanguageGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexLanguageGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
let groupedStreams = new Collection()
this.streams
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
if (!stream.hasLanguages()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
stream.getLanguages().forEach((language: Language) => {
const streamClone = stream.clone()
streamClone.groupTitle = language.name
groupedStreams.add(streamClone)
})
})
groupedStreams = groupedStreams.orderBy((stream: Stream) => {
if (stream.groupTitle === 'Undefined') return 'ZZ'
return stream.groupTitle
})
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.language.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type IndexLanguageGeneratorProps = {
streams: Collection<Stream>
logFile: File
}
export class IndexLanguageGenerator implements Generator {
streams: Collection<Stream>
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexLanguageGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
let groupedStreams = new Collection<Stream>()
this.streams
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
.forEach((stream: Stream) => {
const streamLanguages = stream.getLanguages()
if (streamLanguages.isEmpty()) {
const streamClone = stream.clone()
streamClone.groupTitle = 'Undefined'
groupedStreams.add(streamClone)
return
}
streamLanguages.forEach((language: sdk.Models.Language) => {
const streamClone = stream.clone()
streamClone.groupTitle = language.name
groupedStreams.add(streamClone)
})
})
groupedStreams = groupedStreams.sortBy((stream: Stream) => {
if (stream.groupTitle === 'Undefined') return 'ZZ'
return stream.groupTitle
})
const playlist = new Playlist(groupedStreams, { public: true })
const filepath = 'index.language.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}

32
scripts/generators/indexNsfwGenerator.ts
Unescape Escape View File

@ -1,32 +0,0 @@
import { Collection, File, Storage } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type IndexNsfwGeneratorProps = {
streams: Collection
logFile: File
}
export class IndexNsfwGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: IndexNsfwGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const allStreams = this.streams.orderBy((stream: Stream) => stream.getTitle())
const playlist = new Playlist(allStreams, { public: true })
const filepath = 'index.nsfw.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'index', filepath, count: playlist.streams.count() }) + EOL
)
}
}

115
scripts/generators/languagesGenerator.ts
Unescape Escape View File

@ -1,57 +1,58 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Playlist, Language, Stream } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type LanguagesGeneratorProps = { streams: Collection; logFile: File }
export class LanguagesGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: LanguagesGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
let languages = new Collection()
streams.forEach((stream: Stream) => {
languages = languages.concat(stream.getLanguages())
})
languages
.filter(Boolean)
.uniqBy((language: Language) => language.code)
.orderBy((language: Language) => language.name)
.forEach(async (language: Language) => {
const languageStreams = streams.filter((stream: Stream) => stream.hasLanguage(language))
if (languageStreams.isEmpty()) return
const playlist = new Playlist(languageStreams, { public: true })
const filepath = `languages/${language.code}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + EOL
)
})
const undefinedStreams = streams.filter((stream: Stream) => !stream.hasLanguages())
if (undefinedStreams.isEmpty()) return
const playlist = new Playlist(undefinedStreams, { public: true })
const filepath = 'languages/undefined.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + EOL
)
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Playlist, Stream } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type LanguagesGeneratorProps = { streams: Collection<Stream>; logFile: File }
export class LanguagesGenerator implements Generator {
streams: Collection<Stream>
storage: Storage
logFile: File
constructor({ streams, logFile }: LanguagesGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams: Collection<Stream> = this.streams
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
const languages = new Collection<sdk.Models.Language>()
streams.forEach((stream: Stream) => {
languages.concat(stream.getLanguages())
})
languages
.filter(Boolean)
.uniqBy((language: sdk.Models.Language) => language.code)
.sortBy((language: sdk.Models.Language) => language.name)
.forEach(async (language: sdk.Models.Language) => {
const languageStreams = streams.filter((stream: Stream) => stream.hasLanguage(language))
if (languageStreams.isEmpty()) return
const playlist = new Playlist(languageStreams, { public: true })
const filepath = `languages/${language.code}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + EOL
)
})
const undefinedStreams = streams.filter((stream: Stream) => stream.getLanguages().isEmpty())
if (undefinedStreams.isEmpty()) return
const playlist = new Playlist(undefinedStreams, { public: true })
const filepath = 'languages/undefined.m3u'
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'language', filepath, count: playlist.streams.count() }) + EOL
)
}
}

85
scripts/generators/rawGenerator.ts
Unescape Escape View File

@ -1,40 +1,45 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type RawGeneratorProps = {
streams: Collection
logFile: File
}
export class RawGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: RawGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate() {
const files = this.streams.groupBy((stream: Stream) => stream.getFilename())
for (const filename of files.keys()) {
const streams = new Collection(files.get(filename)).map((stream: Stream) => {
const groupTitle = stream.getCategoryNames().join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
})
const playlist = new Playlist(streams, { public: true })
const filepath = `raw/${filename}`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'raw', filepath, count: playlist.streams.count() }) + EOL
)
}
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
type RawGeneratorProps = {
streams: Collection<Stream>
logFile: File
}
export class RawGenerator implements Generator {
streams: Collection<Stream>
storage: Storage
logFile: File
constructor({ streams, logFile }: RawGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate() {
const files = this.streams.groupBy((stream: Stream) => stream.getFilename())
for (const filename of files.keys()) {
const streams = new Collection(files.get(filename)).map((stream: Stream) => {
const groupTitle = stream
.getCategories()
.map(category => category.name)
.sort()
.join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
})
const playlist = new Playlist(streams, { public: true })
const filepath = `raw/${filename}`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'raw', filepath, count: playlist.streams.count() }) + EOL
)
}
}
}

95
scripts/generators/regionsGenerator.ts
Unescape Escape View File

@ -1,41 +1,54 @@
import { Collection, Storage, File } from '@freearhey/core'
import { Playlist, Region, Stream } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type RegionsGeneratorProps = {
streams: Collection
regions: Collection
logFile: File
}
export class RegionsGenerator implements Generator {
streams: Collection
regions: Collection
storage: Storage
logFile: File
constructor({ streams, regions, logFile }: RegionsGeneratorProps) {
this.streams = streams.clone()
this.regions = regions
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
this.regions.forEach(async (region: Region) => {
const regionStreams = streams.filter((stream: Stream) => stream.isBroadcastInRegion(region))
const playlist = new Playlist(regionStreams, { public: true })
const filepath = `regions/${region.code.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() }) + EOL
)
})
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Playlist, Stream } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type RegionsGeneratorProps = {
streams: Collection<Stream>
regions: Collection<sdk.Models.Region>
logFile: File
}
export class RegionsGenerator implements Generator {
streams: Collection<Stream>
regions: Collection<sdk.Models.Region>
storage: Storage
logFile: File
constructor({ streams, regions, logFile }: RegionsGeneratorProps) {
this.streams = streams.clone()
this.regions = regions
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
const streamsGroupedByRegionCode = {}
streams.forEach((stream: Stream) => {
stream.getBroadcastRegions().forEach((region: sdk.Models.Region) => {
if (streamsGroupedByRegionCode[region.code]) {
streamsGroupedByRegionCode[region.code].add(stream)
} else {
streamsGroupedByRegionCode[region.code] = new Collection<Stream>([stream])
}
})
})
for (const regionCode in streamsGroupedByRegionCode) {
const regionStreams = streamsGroupedByRegionCode[regionCode]
const playlist = new Playlist(regionStreams, { public: true })
const filepath = `regions/${regionCode.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'region', filepath, count: playlist.streams.count() }) + EOL
)
}
}
}

92
scripts/generators/sourcesGenerator.ts
Unescape Escape View File

@ -1,43 +1,49 @@
import { Collection, Storage, File, type Dictionary } from '@freearhey/core'
import { Stream, Playlist } from '../models'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type SourcesGeneratorProps = {
streams: Collection
logFile: File
}
export class SourcesGenerator implements Generator {
streams: Collection
storage: Storage
logFile: File
constructor({ streams, logFile }: SourcesGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate() {
const files: Dictionary = this.streams.groupBy((stream: Stream) => stream.getFilename())
for (const filename of files.keys()) {
if (!filename) continue
let streams = new Collection(files.get(filename))
streams = streams.map((stream: Stream) => {
const groupTitle = stream.getCategoryNames().join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
})
const playlist = new Playlist(streams, { public: true })
const filepath = `sources/${filename}`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'source', filepath, count: playlist.streams.count() }) + EOL
)
}
}
}
import { Collection, Dictionary } from '@freearhey/core'
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Generator } from './generator'
type SourcesGeneratorProps = {
streams: Collection<Stream>
logFile: File
}
export class SourcesGenerator implements Generator {
streams: Collection<Stream>
storage: Storage
logFile: File
constructor({ streams, logFile }: SourcesGeneratorProps) {
this.streams = streams.clone()
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate() {
const files: Dictionary<Stream[]> = this.streams.groupBy((stream: Stream) =>
stream.getFilename()
)
for (const filename of files.keys()) {
if (!filename) continue
const streams = new Collection<Stream>(files.get(filename)).map((stream: Stream) => {
const groupTitle = stream
.getCategories()
.map(category => category.name)
.sort()
.join(';')
if (groupTitle) stream.groupTitle = groupTitle
return stream
})
const playlist = new Playlist(streams, { public: true })
const filepath = `sources/${filename}`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'source', filepath, count: playlist.streams.count() }) + EOL
)
}
}
}

99
scripts/generators/subdivisionsGenerator.ts
Unescape Escape View File

@ -1,45 +1,54 @@
import { Subdivision, Stream, Playlist } from '../models'
import { Collection, Storage, File } from '@freearhey/core'
import { PUBLIC_DIR, EOL } from '../constants'
import { Generator } from './generator'
type SubdivisionsGeneratorProps = {
streams: Collection
subdivisions: Collection
logFile: File
}
export class SubdivisionsGenerator implements Generator {
streams: Collection
subdivisions: Collection
storage: Storage
logFile: File
constructor({ streams, subdivisions, logFile }: SubdivisionsGeneratorProps) {
this.streams = streams.clone()
this.subdivisions = subdivisions
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.orderBy((stream: Stream) => stream.getTitle())
.filter((stream: Stream) => stream.isSFW())
this.subdivisions.forEach(async (subdivision: Subdivision) => {
const subdivisionStreams = streams.filter((stream: Stream) =>
stream.isBroadcastInSubdivision(subdivision)
)
if (subdivisionStreams.isEmpty()) return
const playlist = new Playlist(subdivisionStreams, { public: true })
const filepath = `subdivisions/${subdivision.code.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'subdivision', filepath, count: playlist.streams.count() }) + EOL
)
})
}
}
import { Storage, File } from '@freearhey/storage-js'
import { PUBLIC_DIR, EOL } from '../constants'
import { Stream, Playlist } from '../models'
import { Collection } from '@freearhey/core'
import { Generator } from './generator'
import * as sdk from '@iptv-org/sdk'
type SubdivisionsGeneratorProps = {
streams: Collection<Stream>
subdivisions: Collection<sdk.Models.Subdivision>
logFile: File
}
export class SubdivisionsGenerator implements Generator {
streams: Collection<Stream>
subdivisions: Collection<sdk.Models.Subdivision>
storage: Storage
logFile: File
constructor({ streams, subdivisions, logFile }: SubdivisionsGeneratorProps) {
this.streams = streams.clone()
this.subdivisions = subdivisions
this.storage = new Storage(PUBLIC_DIR)
this.logFile = logFile
}
async generate(): Promise<void> {
const streams = this.streams
.sortBy((stream: Stream) => stream.title)
.filter((stream: Stream) => stream.isSFW())
const streamsGroupedBySubdivisionCode = {}
streams.forEach((stream: Stream) => {
stream.getBroadcastSubdivisions().forEach((subdivision: sdk.Models.Subdivision) => {
if (streamsGroupedBySubdivisionCode[subdivision.code]) {
streamsGroupedBySubdivisionCode[subdivision.code].add(stream)
} else {
streamsGroupedBySubdivisionCode[subdivision.code] = new Collection<Stream>([stream])
}
})
})
for (const subdivisionCode in streamsGroupedBySubdivisionCode) {
const subdivisionStreams = streamsGroupedBySubdivisionCode[subdivisionCode]
const playlist = new Playlist(subdivisionStreams, { public: true })
const filepath = `subdivisions/${subdivisionCode.toLowerCase()}.m3u`
await this.storage.save(filepath, playlist.toString())
this.logFile.append(
JSON.stringify({ type: 'subdivision', filepath, count: playlist.streams.count() }) + EOL
)
}
}
}

15
scripts/models/blocklistRecord.ts
Unescape Escape View File

@ -1,15 +0,0 @@
import type { BlocklistRecordData } from '../types/blocklistRecord'
export class BlocklistRecord {
channelId: string
reason: string
ref: string
constructor(data?: BlocklistRecordData) {
if (!data) return
this.channelId = data.channel
this.reason = data.reason
this.ref = data.ref
}
}

108
scripts/models/broadcastArea.ts
Unescape Escape View File

@ -1,108 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { City, Subdivision, Region, Country } from './'
export class BroadcastArea {
codes: Collection
citiesIncluded: Collection
subdivisionsIncluded: Collection
countriesIncluded: Collection
regionsIncluded: Collection
constructor(codes: Collection) {
this.codes = codes
}
withLocations(
citiesKeyByCode: Dictionary,
subdivisionsKeyByCode: Dictionary,
countriesKeyByCode: Dictionary,
regionsKeyByCode: Dictionary
): this {
const citiesIncluded = new Collection()
const subdivisionsIncluded = new Collection()
const countriesIncluded = new Collection()
let regionsIncluded = new Collection()
this.codes.forEach((value: string) => {
const [type, code] = value.split('/')
switch (type) {
case 'ct': {
const city: City = citiesKeyByCode.get(code)
if (!city) return
citiesIncluded.add(city)
if (city.subdivision) subdivisionsIncluded.add(city.subdivision)
if (city.subdivision && city.subdivision.parent)
subdivisionsIncluded.add(city.subdivision.parent)
if (city.country) countriesIncluded.add(city.country)
regionsIncluded = regionsIncluded.concat(city.getRegions())
break
}
case 's': {
const subdivision: Subdivision = subdivisionsKeyByCode.get(code)
if (!subdivision) return
subdivisionsIncluded.add(subdivision)
if (subdivision.country) countriesIncluded.add(subdivision.country)
regionsIncluded = regionsIncluded.concat(subdivision.getRegions())
break
}
case 'c': {
const country: Country = countriesKeyByCode.get(code)
if (!country) return
countriesIncluded.add(country)
regionsIncluded = regionsIncluded.concat(country.getRegions())
break
}
case 'r': {
const region: Region = regionsKeyByCode.get(code)
if (!region) return
regionsIncluded = regionsIncluded.concat(region.getRegions())
break
}
}
})
this.citiesIncluded = citiesIncluded.uniqBy((city: City) => city.code)
this.subdivisionsIncluded = subdivisionsIncluded.uniqBy(
(subdivision: Subdivision) => subdivision.code
)
this.countriesIncluded = countriesIncluded.uniqBy((country: Country) => country.code)
this.regionsIncluded = regionsIncluded.uniqBy((region: Region) => region.code)
return this
}
getCountries(): Collection {
return this.countriesIncluded || new Collection()
}
getSubdivisions(): Collection {
return this.subdivisionsIncluded || new Collection()
}
getCities(): Collection {
return this.citiesIncluded || new Collection()
}
getRegions(): Collection {
return this.regionsIncluded || new Collection()
}
includesCountry(country: Country): boolean {
return this.getCountries().includes((_country: Country) => _country.code === country.code)
}
includesSubdivision(subdivision: Subdivision): boolean {
return this.getSubdivisions().includes(
(_subdivision: Subdivision) => _subdivision.code === subdivision.code
)
}
includesRegion(region: Region): boolean {
return this.getRegions().includes((_region: Region) => _region.code === region.code)
}
includesCity(city: City): boolean {
return this.getCities().includes((_city: City) => _city.code === city.code)
}
}

18
scripts/models/category.ts
Unescape Escape View File

@ -1,18 +0,0 @@
import type { CategoryData, CategorySerializedData } from '../types/category'
export class Category {
id: string
name: string
constructor(data: CategoryData) {
this.id = data.id
this.name = data.name
}
serialize(): CategorySerializedData {
return {
id: this.id,
name: this.name
}
}
}

233
scripts/models/channel.ts
Unescape Escape View File

@ -1,233 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { Category, Country, Feed, Guide, Logo, Stream, Subdivision } from './index'
import type { ChannelData, ChannelSearchableData, ChannelSerializedData } from '../types/channel'
export class Channel {
id: string
name: string
altNames: Collection
network?: string
owners: Collection
countryCode: string
country?: Country
subdivisionCode?: string
subdivision?: Subdivision
cityName?: string
categoryIds: Collection
categories: Collection = new Collection()
isNSFW: boolean
launched?: string
closed?: string
replacedBy?: string
isClosed: boolean
website?: string
feeds?: Collection
logos: Collection = new Collection()
constructor(data?: ChannelData) {
if (!data) return
this.id = data.id
this.name = data.name
this.altNames = new Collection(data.alt_names)
this.network = data.network || undefined
this.owners = new Collection(data.owners)
this.countryCode = data.country
this.subdivisionCode = data.subdivision || undefined
this.cityName = data.city || undefined
this.categoryIds = new Collection(data.categories)
this.isNSFW = data.is_nsfw
this.launched = data.launched || undefined
this.closed = data.closed || undefined
this.replacedBy = data.replaced_by || undefined
this.website = data.website || undefined
this.isClosed = !!data.closed || !!data.replaced_by
}
withSubdivision(subdivisionsKeyByCode: Dictionary): this {
if (!this.subdivisionCode) return this
this.subdivision = subdivisionsKeyByCode.get(this.subdivisionCode)
return this
}
withCountry(countriesKeyByCode: Dictionary): this {
this.country = countriesKeyByCode.get(this.countryCode)
return this
}
withCategories(categoriesKeyById: Dictionary): this {
this.categories = this.categoryIds
.map((id: string) => categoriesKeyById.get(id))
.filter(Boolean)
return this
}
withFeeds(feedsGroupedByChannelId: Dictionary): this {
this.feeds = new Collection(feedsGroupedByChannelId.get(this.id))
return this
}
withLogos(logosGroupedByChannelId: Dictionary): this {
if (this.id) this.logos = new Collection(logosGroupedByChannelId.get(this.id))
return this
}
getCountry(): Country | undefined {
return this.country
}
getSubdivision(): Subdivision | undefined {
return this.subdivision
}
getCategories(): Collection {
return this.categories || new Collection()
}
hasCategories(): boolean {
return !!this.categories && this.categories.notEmpty()
}
hasCategory(category: Category): boolean {
return (
!!this.categories &&
this.categories.includes((_category: Category) => _category.id === category.id)
)
}
getFeeds(): Collection {
if (!this.feeds) return new Collection()
return this.feeds
}
getGuides(): Collection {
let guides = new Collection()
this.getFeeds().forEach((feed: Feed) => {
guides = guides.concat(feed.getGuides())
})
return guides
}
getGuideNames(): Collection {
return this.getGuides()
.map((guide: Guide) => guide.siteName)
.uniq()
}
getStreams(): Collection {
let streams = new Collection()
this.getFeeds().forEach((feed: Feed) => {
streams = streams.concat(feed.getStreams())
})
return streams
}
getStreamTitles(): Collection {
return this.getStreams()
.map((stream: Stream) => stream.getTitle())
.uniq()
}
getFeedFullNames(): Collection {
return this.getFeeds()
.map((feed: Feed) => feed.getFullName())
.uniq()
}
isSFW(): boolean {
return this.isNSFW === false
}
getLogos(): Collection {
function feed(logo: Logo): number {
if (!logo.feed) return 1
if (logo.feed.isMain) return 1
return 0
}
function format(logo: Logo): number {
const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 }
return logo.format ? levelByFormat[logo.format] : 0
}
function size(logo: Logo): number {
return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
}
return this.logos.orderBy([feed, format, size], ['desc', 'desc', 'asc'], false)
}
getLogo(): Logo | undefined {
return this.getLogos().first()
}
hasLogo(): boolean {
return this.getLogos().notEmpty()
}
getSearchable(): ChannelSearchableData {
return {
id: this.id,
name: this.name,
altNames: this.altNames.all(),
guideNames: this.getGuideNames().all(),
streamTitles: this.getStreamTitles().all(),
feedFullNames: this.getFeedFullNames().all()
}
}
serialize(): ChannelSerializedData {
return {
id: this.id,
name: this.name,
altNames: this.altNames.all(),
network: this.network,
owners: this.owners.all(),
countryCode: this.countryCode,
country: this.country ? this.country.serialize() : undefined,
subdivisionCode: this.subdivisionCode,
subdivision: this.subdivision ? this.subdivision.serialize() : undefined,
cityName: this.cityName,
categoryIds: this.categoryIds.all(),
categories: this.categories.map((category: Category) => category.serialize()).all(),
isNSFW: this.isNSFW,
launched: this.launched,
closed: this.closed,
replacedBy: this.replacedBy,
website: this.website
}
}
deserialize(data: ChannelSerializedData): this {
this.id = data.id
this.name = data.name
this.altNames = new Collection(data.altNames)
this.network = data.network
this.owners = new Collection(data.owners)
this.countryCode = data.countryCode
this.country = data.country ? new Country().deserialize(data.country) : undefined
this.subdivisionCode = data.subdivisionCode
this.cityName = data.cityName
this.categoryIds = new Collection(data.categoryIds)
this.isNSFW = data.isNSFW
this.launched = data.launched
this.closed = data.closed
this.replacedBy = data.replacedBy
this.website = data.website
return this
}
}

78
scripts/models/city.ts
Unescape Escape View File

@ -1,78 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { Country, Region, Subdivision } from '.'
import type { CityData, CitySerializedData } from '../types/city'
export class City {
code: string
name: string
countryCode: string
country?: Country
subdivisionCode?: string
subdivision?: Subdivision
wikidataId: string
regions?: Collection
constructor(data?: CityData) {
if (!data) return
this.code = data.code
this.name = data.name
this.countryCode = data.country
this.subdivisionCode = data.subdivision || undefined
this.wikidataId = data.wikidata_id
}
withCountry(countriesKeyByCode: Dictionary): this {
this.country = countriesKeyByCode.get(this.countryCode)
return this
}
withSubdivision(subdivisionsKeyByCode: Dictionary): this {
if (!this.subdivisionCode) return this
this.subdivision = subdivisionsKeyByCode.get(this.subdivisionCode)
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter((region: Region) =>
region.countryCodes.includes(this.countryCode)
)
return this
}
getRegions(): Collection {
if (!this.regions) return new Collection()
return this.regions
}
serialize(): CitySerializedData {
return {
code: this.code,
name: this.name,
countryCode: this.countryCode,
country: this.country ? this.country.serialize() : undefined,
subdivisionCode: this.subdivisionCode || null,
subdivision: this.subdivision ? this.subdivision.serialize() : undefined,
wikidataId: this.wikidataId
}
}
deserialize(data: CitySerializedData): this {
this.code = data.code
this.name = data.name
this.countryCode = data.countryCode
this.country = data.country ? new Country().deserialize(data.country) : undefined
this.subdivisionCode = data.subdivisionCode || undefined
this.subdivision = data.subdivision
? new Subdivision().deserialize(data.subdivision)
: undefined
this.wikidataId = data.wikidataId
return this
}
}

95
scripts/models/country.ts
Unescape Escape View File

@ -1,95 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { Region, Language, Subdivision } from '.'
import type { CountryData, CountrySerializedData } from '../types/country'
import { SubdivisionSerializedData } from '../types/subdivision'
import { RegionSerializedData } from '../types/region'
export class Country {
code: string
name: string
flag: string
languageCode: string
language?: Language
subdivisions?: Collection
regions?: Collection
cities?: Collection
constructor(data?: CountryData) {
if (!data) return
this.code = data.code
this.name = data.name
this.flag = data.flag
this.languageCode = data.lang
}
withSubdivisions(subdivisionsGroupedByCountryCode: Dictionary): this {
this.subdivisions = new Collection(subdivisionsGroupedByCountryCode.get(this.code))
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter((region: Region) => region.includesCountryCode(this.code))
return this
}
withCities(citiesGroupedByCountryCode: Dictionary): this {
this.cities = new Collection(citiesGroupedByCountryCode.get(this.code))
return this
}
withLanguage(languagesKeyByCode: Dictionary): this {
this.language = languagesKeyByCode.get(this.languageCode)
return this
}
getLanguage(): Language | undefined {
return this.language
}
getRegions(): Collection {
return this.regions || new Collection()
}
getSubdivisions(): Collection {
return this.subdivisions || new Collection()
}
getCities(): Collection {
return this.cities || new Collection()
}
serialize(): CountrySerializedData {
return {
code: this.code,
name: this.name,
flag: this.flag,
languageCode: this.languageCode,
language: this.language ? this.language.serialize() : null,
subdivisions: this.subdivisions
? this.subdivisions.map((subdivision: Subdivision) => subdivision.serialize()).all()
: [],
regions: this.regions ? this.regions.map((region: Region) => region.serialize()).all() : []
}
}
deserialize(data: CountrySerializedData): this {
this.code = data.code
this.name = data.name
this.flag = data.flag
this.languageCode = data.languageCode
this.language = data.language ? new Language().deserialize(data.language) : undefined
this.subdivisions = new Collection(data.subdivisions).map((data: SubdivisionSerializedData) =>
new Subdivision().deserialize(data)
)
this.regions = new Collection(data.regions).map((data: RegionSerializedData) =>
new Region().deserialize(data)
)
return this
}
}

170
scripts/models/feed.ts
Unescape Escape View File

@ -1,170 +0,0 @@
import { Country, Language, Region, Channel, Subdivision, BroadcastArea, City } from './index'
import { Collection, Dictionary } from '@freearhey/core'
import type { FeedData } from '../types/feed'
export class Feed {
channelId: string
channel?: Channel
id: string
name: string
isMain: boolean
broadcastAreaCodes: Collection
broadcastArea?: BroadcastArea
languageCodes: Collection
languages?: Collection
timezoneIds: Collection
timezones?: Collection
videoFormat: string
guides?: Collection
streams?: Collection
constructor(data: FeedData) {
this.channelId = data.channel
this.id = data.id
this.name = data.name
this.isMain = data.is_main
this.broadcastAreaCodes = new Collection(data.broadcast_area)
this.languageCodes = new Collection(data.languages)
this.timezoneIds = new Collection(data.timezones)
this.videoFormat = data.video_format
}
withChannel(channelsKeyById: Dictionary): this {
this.channel = channelsKeyById.get(this.channelId)
return this
}
withStreams(streamsGroupedById: Dictionary): this {
this.streams = new Collection(streamsGroupedById.get(`${this.channelId}@${this.id}`))
if (this.isMain) {
this.streams = this.streams.concat(new Collection(streamsGroupedById.get(this.channelId)))
}
return this
}
withGuides(guidesGroupedByStreamId: Dictionary): this {
this.guides = new Collection(guidesGroupedByStreamId.get(`${this.channelId}@${this.id}`))
if (this.isMain) {
this.guides = this.guides.concat(new Collection(guidesGroupedByStreamId.get(this.channelId)))
}
return this
}
withLanguages(languagesKeyByCode: Dictionary): this {
this.languages = this.languageCodes
.map((code: string) => languagesKeyByCode.get(code))
.filter(Boolean)
return this
}
withTimezones(timezonesKeyById: Dictionary): this {
this.timezones = this.timezoneIds.map((id: string) => timezonesKeyById.get(id)).filter(Boolean)
return this
}
withBroadcastArea(
citiesKeyByCode: Dictionary,
subdivisionsKeyByCode: Dictionary,
countriesKeyByCode: Dictionary,
regionsKeyByCode: Dictionary
): this {
this.broadcastArea = new BroadcastArea(this.broadcastAreaCodes).withLocations(
citiesKeyByCode,
subdivisionsKeyByCode,
countriesKeyByCode,
regionsKeyByCode
)
return this
}
hasBroadcastArea(): boolean {
return !!this.broadcastArea
}
getBroadcastCountries(): Collection {
if (!this.broadcastArea) return new Collection()
return this.broadcastArea.getCountries()
}
getBroadcastRegions(): Collection {
if (!this.broadcastArea) return new Collection()
return this.broadcastArea.getRegions()
}
getTimezones(): Collection {
return this.timezones || new Collection()
}
getLanguages(): Collection {
return this.languages || new Collection()
}
hasLanguages(): boolean {
return !!this.languages && this.languages.notEmpty()
}
hasLanguage(language: Language): boolean {
return (
!!this.languages &&
this.languages.includes((_language: Language) => _language.code === language.code)
)
}
isBroadcastInCity(city: City): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.includesCity(city)
}
isBroadcastInSubdivision(subdivision: Subdivision): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.includesSubdivision(subdivision)
}
isBroadcastInCountry(country: Country): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.includesCountry(country)
}
isBroadcastInRegion(region: Region): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.includesRegion(region)
}
isInternational(): boolean {
if (!this.broadcastArea) return false
return this.broadcastArea.codes.join(',').includes('r/')
}
getGuides(): Collection {
if (!this.guides) return new Collection()
return this.guides
}
getStreams(): Collection {
if (!this.streams) return new Collection()
return this.streams
}
getFullName(): string {
if (!this.channel) return ''
return `${this.channel.name} ${this.name}`
}
}

54
scripts/models/guide.ts
Unescape Escape View File

@ -1,54 +0,0 @@
import type { GuideData, GuideSerializedData } from '../types/guide'
export class Guide {
channelId?: string
feedId?: string
siteDomain: string
siteId: string
siteName: string
languageCode: string
constructor(data?: GuideData) {
if (!data) return
this.channelId = data.channel
this.feedId = data.feed
this.siteDomain = data.site
this.siteId = data.site_id
this.siteName = data.site_name
this.languageCode = data.lang
}
getUUID(): string {
return this.getStreamId() + this.siteId
}
getStreamId(): string | undefined {
if (!this.channelId) return undefined
if (!this.feedId) return this.channelId
return `${this.channelId}@${this.feedId}`
}
serialize(): GuideSerializedData {
return {
channelId: this.channelId,
feedId: this.feedId,
siteDomain: this.siteDomain,
siteId: this.siteId,
siteName: this.siteName,
languageCode: this.languageCode
}
}
deserialize(data: GuideSerializedData): this {
this.channelId = data.channelId
this.feedId = data.feedId
this.siteDomain = data.siteDomain
this.siteId = data.siteId
this.siteName = data.siteName
this.languageCode = data.languageCode
return this
}
}

19
scripts/models/index.ts
Unescape Escape View File

@ -1,16 +1,3 @@
export * from './blocklistRecord'
export * from './broadcastArea'
export * from './category'
export * from './channel'
export * from './city'
export * from './country'
export * from './feed'
export * from './guide'
export * from './issue'
export * from './language'
export * from './logo'
export * from './playlist'
export * from './region'
export * from './stream'
export * from './subdivision'
export * from './timezone'
export * from './issue'
export * from './playlist'
export * from './stream'

27
scripts/models/language.ts
Unescape Escape View File

@ -1,27 +0,0 @@
import type { LanguageData, LanguageSerializedData } from '../types/language'
export class Language {
code: string
name: string
constructor(data?: LanguageData) {
if (!data) return
this.code = data.code
this.name = data.name
}
serialize(): LanguageSerializedData {
return {
code: this.code,
name: this.name
}
}
deserialize(data: LanguageSerializedData): this {
this.code = data.code
this.name = data.name
return this
}
}

40
scripts/models/logo.ts
Unescape Escape View File

@ -1,40 +0,0 @@
import { Collection, type Dictionary } from '@freearhey/core'
import type { LogoData } from '../types/logo'
import { type Feed } from './feed'
export class Logo {
channelId: string
feedId?: string
feed: Feed
tags: Collection
width: number
height: number
format?: string
url: string
constructor(data?: LogoData) {
if (!data) return
this.channelId = data.channel
this.feedId = data.feed || undefined
this.tags = new Collection(data.tags)
this.width = data.width
this.height = data.height
this.format = data.format || undefined
this.url = data.url
}
withFeed(feedsKeyById: Dictionary): this {
if (!this.feedId) return this
this.feed = feedsKeyById.get(this.feedId)
return this
}
getStreamId(): string {
if (!this.feedId) return this.channelId
return `${this.channelId}@${this.feedId}`
}
}

56
scripts/models/playlist.ts
Unescape Escape View File

@ -1,28 +1,28 @@
import { Collection } from '@freearhey/core'
import { Stream } from '../models'
type PlaylistOptions = {
public: boolean
}
export class Playlist {
streams: Collection
options: {
public: boolean
}
constructor(streams: Collection, options?: PlaylistOptions) {
this.streams = streams
this.options = options || { public: false }
}
toString() {
let output = '#EXTM3U\r\n'
this.streams.forEach((stream: Stream) => {
output += stream.toString(this.options) + '\r\n'
})
return output
}
}
import { Collection } from '@freearhey/core'
import { Stream } from '../models'
type PlaylistOptions = {
public: boolean
}
export class Playlist {
streams: Collection<Stream>
options: {
public: boolean
}
constructor(streams: Collection<Stream>, options?: PlaylistOptions) {
this.streams = streams
this.options = options || { public: false }
}
toString() {
let output = '#EXTM3U\r\n'
this.streams.forEach((stream: Stream) => {
output += stream.toString(this.options) + '\r\n'
})
return output
}
}

118
scripts/models/region.ts
Unescape Escape View File

@ -1,118 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
import { City, Country, Subdivision } from '.'
import type { RegionData, RegionSerializedData } from '../types/region'
import { CountrySerializedData } from '../types/country'
import { SubdivisionSerializedData } from '../types/subdivision'
import { CitySerializedData } from '../types/city'
export class Region {
code: string
name: string
countryCodes: Collection
countries?: Collection
subdivisions?: Collection
cities?: Collection
regions?: Collection
constructor(data?: RegionData) {
if (!data) return
this.code = data.code
this.name = data.name
this.countryCodes = new Collection(data.countries)
}
withCountries(countriesKeyByCode: Dictionary): this {
this.countries = this.countryCodes.map((code: string) => countriesKeyByCode.get(code))
return this
}
withSubdivisions(subdivisions: Collection): this {
this.subdivisions = subdivisions.filter(
(subdivision: Subdivision) => this.countryCodes.indexOf(subdivision.countryCode) > -1
)
return this
}
withCities(cities: Collection): this {
this.cities = cities.filter((city: City) => this.countryCodes.indexOf(city.countryCode) > -1)
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter(
(region: Region) => !region.countryCodes.intersects(this.countryCodes).isEmpty()
)
return this
}
getSubdivisions(): Collection {
if (!this.subdivisions) return new Collection()
return this.subdivisions
}
getCountries(): Collection {
if (!this.countries) return new Collection()
return this.countries
}
getCities(): Collection {
if (!this.cities) return new Collection()
return this.cities
}
getRegions(): Collection {
if (!this.regions) return new Collection()
return this.regions
}
includesCountryCode(code: string): boolean {
return this.countryCodes.includes((countryCode: string) => countryCode === code)
}
isWorldwide(): boolean {
return ['INT', 'WW'].includes(this.code)
}
serialize(): RegionSerializedData {
return {
code: this.code,
name: this.name,
countryCodes: this.countryCodes.all(),
countries: this.getCountries()
.map((country: Country) => country.serialize())
.all(),
subdivisions: this.getSubdivisions()
.map((subdivision: Subdivision) => subdivision.serialize())
.all(),
cities: this.getCities()
.map((city: City) => city.serialize())
.all()
}
}
deserialize(data: RegionSerializedData): this {
this.code = data.code
this.name = data.name
this.countryCodes = new Collection(data.countryCodes)
this.countries = new Collection(data.countries).map((data: CountrySerializedData) =>
new Country().deserialize(data)
)
this.subdivisions = new Collection(data.subdivisions).map((data: SubdivisionSerializedData) =>
new Subdivision().deserialize(data)
)
this.cities = new Collection(data.cities).map((data: CitySerializedData) =>
new City().deserialize(data)
)
return this
}
}

935
scripts/models/stream.ts
Unescape Escape View File

@ -1,474 +1,461 @@
import {
Feed,
Channel,
Category,
Region,
Subdivision,
Country,
Language,
Logo,
City
} from './index'
import { URL, Collection, Dictionary } from '@freearhey/core'
import type { StreamData } from '../types/stream'
import parser from 'iptv-playlist-parser'
import { IssueData } from '../core'
import path from 'node:path'
export class Stream {
title: string
url: string
id?: string
channelId?: string
channel?: Channel
feedId?: string
feed?: Feed
logos: Collection = new Collection()
filepath?: string
line?: number
label?: string
verticalResolution?: number
isInterlaced?: boolean
referrer?: string
userAgent?: string
groupTitle: string = 'Undefined'
removed: boolean = false
directives: Collection = new Collection()
constructor(data?: StreamData) {
if (!data) return
const id =
data.channelId && data.feedId ? [data.channelId, data.feedId].join('@') : data.channelId
const { verticalResolution, isInterlaced } = parseQuality(data.quality)
this.id = id || undefined
this.channelId = data.channelId || undefined
this.feedId = data.feedId || undefined
this.title = data.title || ''
this.url = data.url
this.referrer = data.referrer || undefined
this.userAgent = data.userAgent || undefined
this.verticalResolution = verticalResolution || undefined
this.isInterlaced = isInterlaced || undefined
this.label = data.label || undefined
this.directives = new Collection(data.directives)
}
update(issueData: IssueData): this {
const data = {
label: issueData.getString('label'),
quality: issueData.getString('quality'),
httpUserAgent: issueData.getString('httpUserAgent'),
httpReferrer: issueData.getString('httpReferrer'),
newStreamUrl: issueData.getString('newStreamUrl'),
directives: issueData.getArray('directives')
}
if (data.label !== undefined) this.label = data.label
if (data.quality !== undefined) this.setQuality(data.quality)
if (data.httpUserAgent !== undefined) this.userAgent = data.httpUserAgent
if (data.httpReferrer !== undefined) this.referrer = data.httpReferrer
if (data.newStreamUrl !== undefined) this.url = data.newStreamUrl
if (data.directives !== undefined) this.directives = new Collection(data.directives)
return this
}
fromPlaylistItem(data: parser.PlaylistItem): this {
function parseName(name: string): {
title: string
label: string
quality: string
} {
let title = name
const [, label] = title.match(/ \[(.*)\]$/) || [null, '']
title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '')
const [, quality] = title.match(/ \(([0-9]+p)\)$/) || [null, '']
title = title.replace(new RegExp(` \\(${quality}\\)$`), '')
return { title, label, quality }
}
function parseDirectives(string: string) {
const directives = new Collection()
if (!string) return directives
const supportedDirectives = ['#EXTVLCOPT', '#KODIPROP']
const lines = string.split('\r\n')
const regex = new RegExp(`^${supportedDirectives.join('|')}`, 'i')
lines.forEach((line: string) => {
if (regex.test(line)) {
directives.add(line.trim())
}
})
return directives
}
if (!data.name) throw new Error('"name" property is required')
if (!data.url) throw new Error('"url" property is required')
const [channelId, feedId] = data.tvg.id.split('@')
const { title, label, quality } = parseName(data.name)
const { verticalResolution, isInterlaced } = parseQuality(quality)
this.id = data.tvg.id || undefined
this.feedId = feedId || undefined
this.channelId = channelId || undefined
this.line = data.line
this.label = label || undefined
this.title = title
this.verticalResolution = verticalResolution || undefined
this.isInterlaced = isInterlaced || undefined
this.url = data.url
this.referrer = data.http.referrer || undefined
this.userAgent = data.http['user-agent'] || undefined
this.directives = parseDirectives(data.raw)
return this
}
withChannel(channelsKeyById: Dictionary): this {
if (!this.channelId) return this
this.channel = channelsKeyById.get(this.channelId)
return this
}
withFeed(feedsGroupedByChannelId: Dictionary): this {
if (!this.channelId) return this
const channelFeeds = feedsGroupedByChannelId.get(this.channelId) || []
if (this.feedId) this.feed = channelFeeds.find((feed: Feed) => feed.id === this.feedId)
if (!this.feedId && !this.feed) this.feed = channelFeeds.find((feed: Feed) => feed.isMain)
return this
}
withLogos(logosGroupedByStreamId: Dictionary): this {
if (this.id) this.logos = new Collection(logosGroupedByStreamId.get(this.id))
return this
}
setId(id: string): this {
this.id = id
return this
}
setChannelId(channelId: string): this {
this.channelId = channelId
return this
}
setFeedId(feedId: string | undefined): this {
this.feedId = feedId
return this
}
setQuality(quality: string): this {
const { verticalResolution, isInterlaced } = parseQuality(quality)
this.verticalResolution = verticalResolution || undefined
this.isInterlaced = isInterlaced || undefined
return this
}
getLine(): number {
return this.line || -1
}
getFilename(): string {
if (!this.filepath) return ''
return path.basename(this.filepath)
}
setFilepath(filepath: string): this {
this.filepath = filepath
return this
}
updateFilepath(): this {
if (!this.channel) return this
this.filepath = `${this.channel.countryCode.toLowerCase()}.m3u`
return this
}
getChannelId(): string {
return this.channelId || ''
}
getFeedId(): string {
if (this.feedId) return this.feedId
if (this.feed) return this.feed.id
return ''
}
getFilepath(): string {
return this.filepath || ''
}
getReferrer(): string {
return this.referrer || ''
}
getUserAgent(): string {
return this.userAgent || ''
}
getQuality(): string {
if (!this.verticalResolution) return ''
let quality = this.verticalResolution.toString()
if (this.isInterlaced) quality += 'i'
else quality += 'p'
return quality
}
hasId(): boolean {
return !!this.id
}
hasQuality(): boolean {
return !!this.verticalResolution
}
getVerticalResolution(): number {
if (!this.hasQuality()) return 0
return parseInt(this.getQuality().replace(/p|i/, ''))
}
updateTitle(): this {
if (!this.channel) return this
this.title = this.channel.name
if (this.feed && !this.feed.isMain) {
this.title += ` ${this.feed.name}`
}
return this
}
updateId(): this {
if (!this.channel) return this
if (this.feed) {
this.id = `${this.channel.id}@${this.feed.id}`
} else {
this.id = this.channel.id
}
return this
}
normalizeURL() {
const url = new URL(this.url)
this.url = url.normalize().toString()
}
clone(): Stream {
return Object.assign(Object.create(Object.getPrototypeOf(this)), this)
}
hasChannel() {
return !!this.channel
}
getBroadcastRegions(): Collection {
return this.feed ? this.feed.getBroadcastRegions() : new Collection()
}
getBroadcastCountries(): Collection {
return this.feed ? this.feed.getBroadcastCountries() : new Collection()
}
hasBroadcastArea(): boolean {
return this.feed ? this.feed.hasBroadcastArea() : false
}
isSFW(): boolean {
return this.channel ? this.channel.isSFW() : true
}
hasCategories(): boolean {
return this.channel ? this.channel.hasCategories() : false
}
hasCategory(category: Category): boolean {
return this.channel ? this.channel.hasCategory(category) : false
}
getCategoryNames(): string[] {
return this.getCategories()
.map((category: Category) => category.name)
.sort()
.all()
}
getCategories(): Collection {
return this.channel ? this.channel.getCategories() : new Collection()
}
getLanguages(): Collection {
return this.feed ? this.feed.getLanguages() : new Collection()
}
hasLanguages() {
return this.feed ? this.feed.hasLanguages() : false
}
hasLanguage(language: Language) {
return this.feed ? this.feed.hasLanguage(language) : false
}
getBroadcastAreaCodes(): Collection {
return this.feed ? this.feed.broadcastAreaCodes : new Collection()
}
isBroadcastInCity(city: City): boolean {
return this.feed ? this.feed.isBroadcastInCity(city) : false
}
isBroadcastInSubdivision(subdivision: Subdivision): boolean {
return this.feed ? this.feed.isBroadcastInSubdivision(subdivision) : false
}
isBroadcastInCountry(country: Country): boolean {
return this.feed ? this.feed.isBroadcastInCountry(country) : false
}
isBroadcastInRegion(region: Region): boolean {
return this.feed ? this.feed.isBroadcastInRegion(region) : false
}
isInternational(): boolean {
return this.feed ? this.feed.isInternational() : false
}
getLogos(): Collection {
function format(logo: Logo): number {
const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 }
return logo.format ? levelByFormat[logo.format] : 0
}
function size(logo: Logo): number {
return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
}
return this.logos.orderBy([format, size], ['desc', 'asc'], false)
}
getLogo(): Logo | undefined {
return this.getLogos().first()
}
hasLogo(): boolean {
return this.getLogos().notEmpty()
}
getLogoUrl(): string {
let logo: Logo | undefined
if (this.hasLogo()) logo = this.getLogo()
else logo = this?.channel?.getLogo()
return logo ? logo.url : ''
}
getTitle(): string {
return this.title || ''
}
getFullTitle(): string {
let title = `${this.getTitle()}`
if (this.getQuality()) {
title += ` (${this.getQuality()})`
}
if (this.label) {
title += ` [${this.label}]`
}
return title
}
getLabel(): string {
return this.label || ''
}
getId(): string {
return this.id || ''
}
toJSON() {
return {
channel: this.channelId || null,
feed: this.feedId || null,
title: this.title,
url: this.url,
referrer: this.referrer || null,
user_agent: this.userAgent || null,
quality: this.getQuality() || null
}
}
toString(options: { public: boolean }) {
let output = `#EXTINF:-1 tvg-id="${this.getId()}"`
if (options.public) {
output += ` tvg-logo="${this.getLogoUrl()}" group-title="${this.groupTitle}"`
}
if (this.referrer) {
output += ` http-referrer="${this.referrer}"`
}
if (this.userAgent) {
output += ` http-user-agent="${this.userAgent}"`
}
output += `,${this.getFullTitle()}`
this.directives.forEach((prop: string) => {
output += `\r\n${prop}`
})
output += `\r\n${this.url}`
return output
}
}
function escapeRegExp(text) {
return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
}
function parseQuality(quality: string | null): {
verticalResolution: number | null
isInterlaced: boolean | null
} {
if (!quality) return { verticalResolution: null, isInterlaced: null }
const [, verticalResolutionString] = quality.match(/^(\d+)/) || [null, undefined]
const isInterlaced = /i$/i.test(quality)
let verticalResolution = 0
if (verticalResolutionString) verticalResolution = parseInt(verticalResolutionString)
return { verticalResolution, isInterlaced }
}
import { Collection } from '@freearhey/core'
import parser from 'iptv-playlist-parser'
import { normalizeURL } from '../utils'
import * as sdk from '@iptv-org/sdk'
import { IssueData } from '../core'
import { data } from '../api'
import path from 'node:path'
export class Stream extends sdk.Models.Stream {
directives: Collection<string>
filepath?: string
line?: number
groupTitle: string = 'Undefined'
removed: boolean = false
tvgId?: string
label: string | null
updateWithIssue(issueData: IssueData): this {
const data = {
label: issueData.getString('label'),
quality: issueData.getString('quality'),
httpUserAgent: issueData.getString('httpUserAgent'),
httpReferrer: issueData.getString('httpReferrer'),
newStreamUrl: issueData.getString('newStreamUrl'),
directives: issueData.getArray('directives')
}
if (data.label !== undefined) this.label = data.label
if (data.quality !== undefined) this.quality = data.quality
if (data.httpUserAgent !== undefined) this.user_agent = data.httpUserAgent
if (data.httpReferrer !== undefined) this.referrer = data.httpReferrer
if (data.newStreamUrl !== undefined) this.url = data.newStreamUrl
if (data.directives !== undefined) this.setDirectives(data.directives)
return this
}
static fromPlaylistItem(data: parser.PlaylistItem): Stream {
function escapeRegExp(text) {
return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&')
}
function parseName(name: string): {
title: string
label: string
quality: string
} {
let title = name
const [, label] = title.match(/ \[(.*)\]$/) || [null, '']
title = title.replace(new RegExp(` \\[${escapeRegExp(label)}\\]$`), '')
const [, quality] = title.match(/ \(([0-9]+[p|i])\)$/) || [null, '']
title = title.replace(new RegExp(` \\(${quality}\\)$`), '')
return { title, label, quality }
}
function parseDirectives(string: string): Collection<string> {
const directives = new Collection<string>()
if (!string) return directives
const supportedDirectives = ['#EXTVLCOPT', '#KODIPROP']
const lines = string.split('\r\n')
const regex = new RegExp(`^${supportedDirectives.join('|')}`, 'i')
lines.forEach((line: string) => {
if (regex.test(line)) {
directives.add(line.trim())
}
})
return directives
}
if (!data.name) throw new Error('"name" property is required')
if (!data.url) throw new Error('"url" property is required')
const [channelId, feedId] = data.tvg.id.split('@')
const { title, label, quality } = parseName(data.name)
const stream = new Stream({
channel: channelId || null,
feed: feedId || null,
title: title,
quality: quality || null,
url: data.url,
referrer: data.http.referrer || null,
user_agent: data.http['user-agent'] || null
})
stream.tvgId = data.tvg.id
stream.line = data.line
stream.label = label || null
stream.directives = parseDirectives(data.raw)
return stream
}
isSFW(): boolean {
const channel = this.getChannel()
if (!channel) return true
return !channel.is_nsfw
}
getUniqKey(): string {
const filepath = this.getFilepath()
const tvgId = this.getTvgId()
return filepath + tvgId + this.url
}
getVerticalResolution(): number {
if (!this.quality) return 0
const [, verticalResolutionString] = this.quality.match(/^(\d+)/) || ['', '0']
return parseInt(verticalResolutionString)
}
getBroadcastCountries(): Collection<sdk.Models.Country> {
const countries = new Collection<sdk.Models.Country>()
const feed = this.getFeed()
if (!feed) return countries
feed
.getBroadcastArea()
.getLocations()
.forEach((location: sdk.Models.BroadcastAreaLocation) => {
let country: sdk.Models.Country | undefined
switch (location.type) {
case 'country': {
country = data.countriesKeyByCode.get(location.code)
break
}
case 'subdivision': {
const subdivision = data.subdivisionsKeyByCode.get(location.code)
if (!subdivision) break
country = data.countriesKeyByCode.get(subdivision.country)
break
}
case 'city': {
const city = data.citiesKeyByCode.get(location.code)
if (!city) break
country = data.countriesKeyByCode.get(city.country)
break
}
}
if (country) countries.add(country)
})
return countries.uniqBy((country: sdk.Models.Country) => country.code)
}
getBroadcastSubdivisions(): Collection<sdk.Models.Subdivision> {
const subdivisions = new Collection<sdk.Models.Subdivision>()
const feed = this.getFeed()
if (!feed) return subdivisions
feed
.getBroadcastArea()
.getLocations()
.forEach((location: sdk.Models.BroadcastAreaLocation) => {
switch (location.type) {
case 'subdivision': {
const subdivision = data.subdivisionsKeyByCode.get(location.code)
if (!subdivision) break
subdivisions.add(subdivision)
if (!subdivision.parent) break
const parentSubdivision = data.subdivisionsKeyByCode.get(subdivision.parent)
if (!parentSubdivision) break
subdivisions.add(parentSubdivision)
break
}
case 'city': {
const city = data.citiesKeyByCode.get(location.code)
if (!city || !city.subdivision) break
const subdivision = data.subdivisionsKeyByCode.get(city.subdivision)
if (!subdivision) break
subdivisions.add(subdivision)
if (!subdivision.parent) break
const parentSubdivision = data.subdivisionsKeyByCode.get(subdivision.parent)
if (!parentSubdivision) break
subdivisions.add(parentSubdivision)
break
}
}
})
return subdivisions.uniqBy((subdivision: sdk.Models.Subdivision) => subdivision.code)
}
getBroadcastCities(): Collection<sdk.Models.City> {
const cities = new Collection<sdk.Models.City>()
const feed = this.getFeed()
if (!feed) return cities
feed
.getBroadcastArea()
.getLocations()
.forEach((location: sdk.Models.BroadcastAreaLocation) => {
if (location.type !== 'city') return
const city = data.citiesKeyByCode.get(location.code)
if (city) cities.add(city)
})
return cities.uniqBy((city: sdk.Models.City) => city.code)
}
getBroadcastRegions(): Collection<sdk.Models.Region> {
const regions = new Collection<sdk.Models.Region>()
const feed = this.getFeed()
if (!feed) return regions
feed
.getBroadcastArea()
.getLocations()
.forEach((location: sdk.Models.BroadcastAreaLocation) => {
switch (location.type) {
case 'region': {
const region = data.regionsKeyByCode.get(location.code)
if (!region) break
regions.add(region)
const relatedRegions = data.regions.filter((_region: sdk.Models.Region) =>
new Collection<string>(_region.countries)
.intersects(new Collection<string>(region.countries))
.isNotEmpty()
)
regions.concat(relatedRegions)
break
}
case 'country': {
const country = data.countriesKeyByCode.get(location.code)
if (!country) break
const countryRegions = data.regions.filter((_region: sdk.Models.Region) =>
new Collection<string>(_region.countries).includes(
(code: string) => code === country.code
)
)
regions.concat(countryRegions)
break
}
case 'subdivision': {
const subdivision = data.subdivisionsKeyByCode.get(location.code)
if (!subdivision) break
const subdivisionRegions = data.regions.filter((_region: sdk.Models.Region) =>
new Collection<string>(_region.countries).includes(
(code: string) => code === subdivision.country
)
)
regions.concat(subdivisionRegions)
break
}
case 'city': {
const city = data.citiesKeyByCode.get(location.code)
if (!city) break
const cityRegions = data.regions.filter((_region: sdk.Models.Region) =>
new Collection<string>(_region.countries).includes(
(code: string) => code === city.country
)
)
regions.concat(cityRegions)
break
}
}
})
return regions.uniqBy((region: sdk.Models.Region) => region.code)
}
isInternational(): boolean {
const feed = this.getFeed()
if (!feed) return false
const broadcastAreaCodes = feed.getBroadcastArea().codes
if (broadcastAreaCodes.join(';').includes('r/')) return true
if (broadcastAreaCodes.filter(code => code.includes('c/')).length > 1) return true
return false
}
hasCategory(category: sdk.Models.Category): boolean {
const channel = this.getChannel()
if (!channel) return false
const found = channel.categories.find((id: string) => id === category.id)
return !!found
}
hasLanguage(language: sdk.Models.Language): boolean {
const found = this.getLanguages().find(
(_language: sdk.Models.Language) => _language.code === language.code
)
return !!found
}
setDirectives(directives: string[]): this {
this.directives = new Collection(directives).filter((directive: string) =>
/^(#KODIPROP|#EXTVLCOPT)/.test(directive)
)
return this
}
updateTvgId(): this {
if (!this.channel) return this
if (this.feed) {
this.tvgId = `${this.channel}@${this.feed}`
} else {
this.tvgId = this.channel
}
return this
}
updateFilepath(): this {
const channel = this.getChannel()
if (!channel) return this
this.filepath = `${channel.country.toLowerCase()}.m3u`
return this
}
updateTitle(): this {
const channel = this.getChannel()
if (!channel) return this
const feed = this.getFeed()
this.title = channel.name
if (feed && !feed.is_main) {
this.title += ` ${feed.name}`
}
return this
}
normalizeURL() {
this.url = normalizeURL(this.url)
}
getLogos(): Collection<sdk.Models.Logo> {
const logos = super.getLogos()
if (logos.isEmpty()) return new Collection()
function format(logo: sdk.Models.Logo): number {
const levelByFormat = { SVG: 0, PNG: 3, APNG: 1, WebP: 1, AVIF: 1, JPEG: 2, GIF: 1 }
return logo.format ? levelByFormat[logo.format] : 0
}
function size(logo: sdk.Models.Logo): number {
return Math.abs(512 - logo.width) + Math.abs(512 - logo.height)
}
return logos.sortBy([format, size], ['desc', 'asc'], false)
}
getFilepath(): string {
return this.filepath || ''
}
getFilename(): string {
return path.basename(this.getFilepath())
}
getLine(): number {
return this.line || -1
}
getTvgId(): string {
if (this.tvgId) return this.tvgId
return this.getId()
}
getTvgLogo(): string {
const logo = this.getLogos().first()
return logo ? logo.url : ''
}
getFullTitle(): string {
let title = `${this.title}`
if (this.quality) {
title += ` (${this.quality})`
}
if (this.label) {
title += ` [${this.label}]`
}
return title
}
toString(options: { public?: boolean } = {}) {
options = { ...{ public: false }, ...options }
let output = `#EXTINF:-1 tvg-id="${this.getTvgId()}"`
if (options.public) {
output += ` tvg-logo="${this.getTvgLogo()}" group-title="${this.groupTitle}"`
}
if (this.referrer) {
output += ` http-referrer="${this.referrer}"`
}
if (this.user_agent) {
output += ` http-user-agent="${this.user_agent}"`
}
output += `,${this.getFullTitle()}`
this.directives.forEach((prop: string) => {
output += `\r\n${prop}`
})
output += `\r\n${this.url}`
return output
}
toObject(): sdk.Types.StreamData {
let feedId = this.feed
if (!feedId) {
const feed = this.getFeed()
if (feed) feedId = feed.id
}
return {
channel: this.channel,
feed: feedId,
title: this.title,
url: this.url,
quality: this.quality,
user_agent: this.user_agent,
referrer: this.referrer
}
}
clone(): Stream {
return Object.assign(Object.create(Object.getPrototypeOf(this)), this)
}
}

83
scripts/models/subdivision.ts
Unescape Escape View File

@ -1,83 +0,0 @@
import { SubdivisionData, SubdivisionSerializedData } from '../types/subdivision'
import { Dictionary, Collection } from '@freearhey/core'
import { Country, Region } from '.'
export class Subdivision {
code: string
name: string
countryCode: string
country?: Country
parentCode?: string
parent?: Subdivision
regions?: Collection
cities?: Collection
constructor(data?: SubdivisionData) {
if (!data) return
this.code = data.code
this.name = data.name
this.countryCode = data.country
this.parentCode = data.parent || undefined
}
withCountry(countriesKeyByCode: Dictionary): this {
this.country = countriesKeyByCode.get(this.countryCode)
return this
}
withRegions(regions: Collection): this {
this.regions = regions.filter((region: Region) =>
region.countryCodes.includes(this.countryCode)
)
return this
}
withCities(citiesGroupedBySubdivisionCode: Dictionary): this {
this.cities = new Collection(citiesGroupedBySubdivisionCode.get(this.code))
return this
}
withParent(subdivisionsKeyByCode: Dictionary): this {
if (!this.parentCode) return this
this.parent = subdivisionsKeyByCode.get(this.parentCode)
return this
}
getRegions(): Collection {
if (!this.regions) return new Collection()
return this.regions
}
getCities(): Collection {
if (!this.cities) return new Collection()
return this.cities
}
serialize(): SubdivisionSerializedData {
return {
code: this.code,
name: this.name,
countryCode: this.countryCode,
country: this.country ? this.country.serialize() : undefined,
parentCode: this.parentCode || null
}
}
deserialize(data: SubdivisionSerializedData): this {
this.code = data.code
this.name = data.name
this.countryCode = data.countryCode
this.country = data.country ? new Country().deserialize(data.country) : undefined
this.parentCode = data.parentCode || undefined
return this
}
}

30
scripts/models/timezone.ts
Unescape Escape View File

@ -1,30 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
type TimezoneData = {
id: string
utc_offset: string
countries: string[]
}
export class Timezone {
id: string
utcOffset: string
countryCodes: Collection
countries?: Collection
constructor(data: TimezoneData) {
this.id = data.id
this.utcOffset = data.utc_offset
this.countryCodes = new Collection(data.countries)
}
withCountries(countriesKeyByCode: Dictionary): this {
this.countries = this.countryCodes.map((code: string) => countriesKeyByCode.get(code))
return this
}
getCountries(): Collection {
return this.countries || new Collection()
}
}

119
scripts/tables/categoriesTable.ts
Unescape Escape View File

@ -1,56 +1,63 @@
import { Storage, Collection, File, Dictionary } from '@freearhey/core'
import { HTMLTable, LogParser, LogItem } from '../core'
import { LOGS_DIR, README_DIR } from '../constants'
import { Category } from '../models'
import { Table } from './table'
type CategoriesTableProps = {
categoriesKeyById: Dictionary
}
export class CategoriesTable implements Table {
categoriesKeyById: Dictionary
constructor({ categoriesKeyById }: CategoriesTableProps) {
this.categoriesKeyById = categoriesKeyById
}
async make() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
let items = new Collection()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.type === 'category')
.forEach((logItem: LogItem) => {
const file = new File(logItem.filepath)
const categoryId = file.name()
const category: Category = this.categoriesKeyById.get(categoryId)
items.add([
category ? category.name : 'ZZ',
category ? category.name : 'Undefined',
logItem.count,
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
})
items = items
.orderBy(item => item[0])
.map(item => {
item.shift()
return item
})
const table = new HTMLTable(items.all(), [
{ name: 'Category' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', nowrap: true }
])
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_categories.md', table.toString())
}
}
import { HTMLTable, HTMLTableItem, LogParser, LogItem, HTMLTableColumn } from '../core'
import { Storage, File } from '@freearhey/storage-js'
import { LOGS_DIR, README_DIR } from '../constants'
import { Collection } from '@freearhey/core'
import * as sdk from '@iptv-org/sdk'
import { Table } from './table'
import { data } from '../api'
export class CategoriesTable implements Table {
async create() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
let items = new Collection<HTMLTableItem>()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.type === 'category')
.forEach((logItem: LogItem) => {
if (logItem.filepath.includes('undefined')) {
items.add([
'ZZ',
'Undefined',
logItem.count.toString(),
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
return
}
const file = new File(logItem.filepath)
const categoryId = file.name()
const category: sdk.Models.Category | undefined = data.categoriesKeyById.get(categoryId)
if (!category) return
items.add([
category.name,
category.name,
logItem.count.toString(),
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
})
items = items
.sortBy(item => item[0])
.map(item => {
item.shift()
return item
})
const columns = new Collection<HTMLTableColumn>([
{ name: 'Category' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', nowrap: true }
])
const table = new HTMLTable(items, columns)
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_categories.md', table.toString())
}
}

365
scripts/tables/countriesTable.ts
Unescape Escape View File

@ -1,189 +1,176 @@
import { Storage, Collection, Dictionary } from '@freearhey/core'
import { City, Country, Subdivision } from '../models'
import { LOGS_DIR, README_DIR } from '../constants'
import { LogParser, LogItem } from '../core'
import { Table } from './table'
type CountriesTableProps = {
countriesKeyByCode: Dictionary
subdivisionsKeyByCode: Dictionary
countries: Collection
subdivisions: Collection
cities: Collection
}
export class CountriesTable implements Table {
countriesKeyByCode: Dictionary
subdivisionsKeyByCode: Dictionary
countries: Collection
subdivisions: Collection
cities: Collection
constructor({
countriesKeyByCode,
subdivisionsKeyByCode,
countries,
subdivisions,
cities
}: CountriesTableProps) {
this.countriesKeyByCode = countriesKeyByCode
this.subdivisionsKeyByCode = subdivisionsKeyByCode
this.countries = countries
this.subdivisions = subdivisions
this.cities = cities
}
async make() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
const parsed = parser.parse(generatorsLog)
const logCountries = parsed.filter((logItem: LogItem) => logItem.type === 'country')
const logSubdivisions = parsed.filter((logItem: LogItem) => logItem.type === 'subdivision')
const logCities = parsed.filter((logItem: LogItem) => logItem.type === 'city')
let items = new Collection()
this.countries.forEach((country: Country) => {
const countriesLogItem = logCountries.find(
(logItem: LogItem) => logItem.filepath === `countries/${country.code.toLowerCase()}.m3u`
)
const countryItem = {
index: country.name,
count: 0,
link: `https://iptv-org.github.io/iptv/countries/${country.code.toLowerCase()}.m3u`,
name: `${country.flag} ${country.name}`,
children: new Collection()
}
if (countriesLogItem) {
countryItem.count = countriesLogItem.count
}
const countrySubdivisions = this.subdivisions.filter(
(subdivision: Subdivision) => subdivision.countryCode === country.code
)
const countryCities = this.cities.filter((city: City) => city.countryCode === country.code)
if (countrySubdivisions.notEmpty()) {
this.subdivisions.forEach((subdivision: Subdivision) => {
if (subdivision.countryCode !== country.code) return
const subdivisionCities = countryCities.filter(
(city: City) =>
(city.subdivisionCode && city.subdivisionCode === subdivision.code) ||
city.countryCode === subdivision.countryCode
)
const subdivisionsLogItem = logSubdivisions.find(
(logItem: LogItem) =>
logItem.filepath === `subdivisions/${subdivision.code.toLowerCase()}.m3u`
)
const subdivisionItem = {
index: subdivision.name,
name: subdivision.name,
count: 0,
link: `https://iptv-org.github.io/iptv/subdivisions/${subdivision.code.toLowerCase()}.m3u`,
children: new Collection()
}
if (subdivisionsLogItem) {
subdivisionItem.count = subdivisionsLogItem.count
}
subdivisionCities.forEach((city: City) => {
if (city.countryCode !== country.code || city.subdivisionCode !== subdivision.code)
return
const citiesLogItem = logCities.find(
(logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u`
)
if (!citiesLogItem) return
subdivisionItem.children.add({
index: city.name,
name: city.name,
count: citiesLogItem.count,
link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`
})
})
if (subdivisionItem.count > 0 || subdivisionItem.children.notEmpty()) {
countryItem.children.add(subdivisionItem)
}
})
} else if (countryCities.notEmpty()) {
countryCities.forEach((city: City) => {
const citiesLogItem = logCities.find(
(logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u`
)
if (!citiesLogItem) return
countryItem.children.add({
index: city.name,
name: city.name,
count: citiesLogItem.count,
link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`,
children: new Collection()
})
})
}
if (countryItem.count > 0 || countryItem.children.notEmpty()) {
items.add(countryItem)
}
})
const internationalLogItem = logCountries.find(
(logItem: LogItem) => logItem.filepath === 'countries/int.m3u'
)
if (internationalLogItem) {
items.push({
index: 'ZZ',
name: '🌐 International',
count: internationalLogItem.count,
link: `https://iptv-org.github.io/iptv/${internationalLogItem.filepath}`,
children: new Collection()
})
}
const undefinedLogItem = logCountries.find(
(logItem: LogItem) => logItem.filepath === 'countries/undefined.m3u'
)
if (undefinedLogItem) {
items.push({
index: 'ZZZ',
name: 'Undefined',
count: undefinedLogItem.count,
link: `https://iptv-org.github.io/iptv/${undefinedLogItem.filepath}`,
children: new Collection()
})
}
items = items.orderBy(item => item.index)
const output = items
.map(item => {
let row = `- ${item.name} <code>${item.link}</code>`
item.children
.orderBy(item => item.index)
.forEach(item => {
row += `\r\n - ${item.name} <code>${item.link}</code>`
item.children
.orderBy(item => item.index)
.forEach(item => {
row += `\r\n - ${item.name} <code>${item.link}</code>`
})
})
return row
})
.join('\r\n')
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_countries.md', output)
}
}
import { LOGS_DIR, README_DIR } from '../constants'
import { Storage } from '@freearhey/storage-js'
import { Collection } from '@freearhey/core'
import { LogParser, LogItem } from '../core'
import * as sdk from '@iptv-org/sdk'
import { Table } from './table'
import { data } from '../api'
type ListItem = {
index: string
count: number
link: string
name: string
children: Collection<ListItem>
}
export class CountriesTable implements Table {
async create() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
const parsed = parser.parse(generatorsLog)
const logCountries = parsed.filter((logItem: LogItem) => logItem.type === 'country')
const logSubdivisions = parsed.filter((logItem: LogItem) => logItem.type === 'subdivision')
const logCities = parsed.filter((logItem: LogItem) => logItem.type === 'city')
let items = new Collection()
data.countries.forEach((country: sdk.Models.Country) => {
const countryCode = country.code
const countriesLogItem = logCountries.find(
(logItem: LogItem) => logItem.filepath === `countries/${countryCode.toLowerCase()}.m3u`
)
const countryItem: ListItem = {
index: country.name,
count: 0,
link: `https://iptv-org.github.io/iptv/countries/${countryCode.toLowerCase()}.m3u`,
name: `${country.flag} ${country.name}`,
children: new Collection()
}
if (countriesLogItem) {
countryItem.count = countriesLogItem.count
}
const countrySubdivisions = data.subdivisions.filter(
(subdivision: sdk.Models.Subdivision) => subdivision.country === countryCode
)
const countryCities = data.cities.filter(
(city: sdk.Models.City) => city.country === countryCode
)
if (countrySubdivisions.isNotEmpty()) {
data.subdivisions.forEach((subdivision: sdk.Models.Subdivision) => {
if (subdivision.country !== countryCode) return
const subdivisionCode = subdivision.code
const subdivisionCities = countryCities.filter(
(city: sdk.Models.City) =>
(city.subdivision && city.subdivision === subdivisionCode) ||
city.country === subdivision.country
)
const subdivisionsLogItem = logSubdivisions.find(
(logItem: LogItem) =>
logItem.filepath === `subdivisions/${subdivisionCode.toLowerCase()}.m3u`
)
const subdivisionItem: ListItem = {
index: subdivision.name,
name: subdivision.name,
count: 0,
link: `https://iptv-org.github.io/iptv/subdivisions/${subdivisionCode.toLowerCase()}.m3u`,
children: new Collection<ListItem>()
}
if (subdivisionsLogItem) {
subdivisionItem.count = subdivisionsLogItem.count
}
subdivisionCities.forEach((city: sdk.Models.City) => {
if (city.country !== countryCode || city.subdivision !== subdivisionCode) return
const citiesLogItem = logCities.find(
(logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u`
)
if (!citiesLogItem) return
subdivisionItem.children.add({
index: city.name,
name: city.name,
count: citiesLogItem.count,
link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`,
children: new Collection<ListItem>()
})
})
if (subdivisionItem.count > 0 || subdivisionItem.children.isNotEmpty()) {
countryItem.children.add(subdivisionItem)
}
})
} else if (countryCities.isNotEmpty()) {
countryCities.forEach((city: sdk.Models.City) => {
const citiesLogItem = logCities.find(
(logItem: LogItem) => logItem.filepath === `cities/${city.code.toLowerCase()}.m3u`
)
if (!citiesLogItem) return
countryItem.children.add({
index: city.name,
name: city.name,
count: citiesLogItem.count,
link: `https://iptv-org.github.io/iptv/${citiesLogItem.filepath}`,
children: new Collection()
})
})
}
if (countryItem.count > 0 || countryItem.children.isNotEmpty()) {
items.add(countryItem)
}
})
const internationalLogItem = logCountries.find(
(logItem: LogItem) => logItem.filepath === 'countries/int.m3u'
)
if (internationalLogItem) {
items.add({
index: 'ZZ',
name: '🌐 International',
count: internationalLogItem.count,
link: `https://iptv-org.github.io/iptv/${internationalLogItem.filepath}`,
children: new Collection()
})
}
const undefinedLogItem = logCountries.find(
(logItem: LogItem) => logItem.filepath === 'countries/undefined.m3u'
)
if (undefinedLogItem) {
items.add({
index: 'ZZZ',
name: 'Undefined',
count: undefinedLogItem.count,
link: `https://iptv-org.github.io/iptv/${undefinedLogItem.filepath}`,
children: new Collection()
})
}
items = items.sortBy(item => item.index)
const output = items
.map((item: ListItem) => {
let row = `- ${item.name} <code>${item.link}</code>`
item.children
.sortBy((item: ListItem) => item.index)
.forEach((item: ListItem) => {
row += `\r\n - ${item.name} <code>${item.link}</code>`
item.children
.sortBy((item: ListItem) => item.index)
.forEach((item: ListItem) => {
row += `\r\n - ${item.name} <code>${item.link}</code>`
})
})
return row
})
.join('\r\n')
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_countries.md', output)
}
}

119
scripts/tables/languagesTable.ts
Unescape Escape View File

@ -1,56 +1,63 @@
import { Storage, Collection, File, Dictionary } from '@freearhey/core'
import { HTMLTable, LogParser, LogItem } from '../core'
import { LOGS_DIR, README_DIR } from '../constants'
import { Language } from '../models'
import { Table } from './table'
type LanguagesTableProps = {
languagesKeyByCode: Dictionary
}
export class LanguagesTable implements Table {
languagesKeyByCode: Dictionary
constructor({ languagesKeyByCode }: LanguagesTableProps) {
this.languagesKeyByCode = languagesKeyByCode
}
async make() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
let data = new Collection()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.type === 'language')
.forEach((logItem: LogItem) => {
const file = new File(logItem.filepath)
const languageCode = file.name()
const language: Language = this.languagesKeyByCode.get(languageCode)
data.add([
language ? language.name : 'ZZ',
language ? language.name : 'Undefined',
logItem.count,
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
})
data = data
.orderBy(item => item[0])
.map(item => {
item.shift()
return item
})
const table = new HTMLTable(data.all(), [
{ name: 'Language', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left', nowrap: true }
])
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_languages.md', table.toString())
}
}
import { HTMLTable, LogParser, LogItem, HTMLTableColumn, HTMLTableItem } from '../core'
import { Storage, File } from '@freearhey/storage-js'
import { LOGS_DIR, README_DIR } from '../constants'
import { Collection } from '@freearhey/core'
import * as sdk from '@iptv-org/sdk'
import { Table } from './table'
import { data } from '../api'
export class LanguagesTable implements Table {
async create() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
let items = new Collection<HTMLTableItem>()
parser
.parse(generatorsLog)
.filter((logItem: LogItem) => logItem.type === 'language')
.forEach((logItem: LogItem) => {
if (logItem.filepath.includes('undefined')) {
items.add([
'ZZ',
'Undefined',
logItem.count.toString(),
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
return
}
const file = new File(logItem.filepath)
const languageCode = file.name()
const language: sdk.Models.Language | undefined = data.languagesKeyByCode.get(languageCode)
if (!language) return
items.add([
language.name,
language.name,
logItem.count.toString(),
`<code>https://iptv-org.github.io/iptv/${logItem.filepath}</code>`
])
})
items = items
.sortBy(item => item[0])
.map(item => {
item.shift()
return item
})
const columns = new Collection<HTMLTableColumn>([
{ name: 'Language', align: 'left' },
{ name: 'Channels', align: 'right' },
{ name: 'Playlist', align: 'left', nowrap: true }
])
const table = new HTMLTable(items, columns)
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_languages.md', table.toString())
}
}

101
scripts/tables/regionsTable.ts
Unescape Escape View File

@ -1,52 +1,49 @@
import { Storage, Collection } from '@freearhey/core'
import { LogParser, LogItem } from '../core'
import { LOGS_DIR, README_DIR } from '../constants'
import { Region } from '../models'
import { Table } from './table'
type RegionsTableProps = {
regions: Collection
}
export class RegionsTable implements Table {
regions: Collection
constructor({ regions }: RegionsTableProps) {
this.regions = regions
}
async make() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
const parsed = parser.parse(generatorsLog)
const logRegions = parsed.filter((logItem: LogItem) => logItem.type === 'region')
let items = new Collection()
this.regions.forEach((region: Region) => {
const logItem = logRegions.find(
(logItem: LogItem) => logItem.filepath === `regions/${region.code.toLowerCase()}.m3u`
)
if (!logItem) return
items.add({
index: region.name,
name: region.name,
count: logItem.count,
link: `https://iptv-org.github.io/iptv/${logItem.filepath}`
})
})
items = items.orderBy(item => item.index)
const output = items
.map(item => {
return `- ${item.name} <code>${item.link}</code>`
})
.join('\r\n')
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_regions.md', output)
}
}
import { LOGS_DIR, README_DIR } from '../constants'
import { Storage } from '@freearhey/storage-js'
import { LogParser, LogItem } from '../core'
import { Collection } from '@freearhey/core'
import * as sdk from '@iptv-org/sdk'
import { Table } from './table'
import { data } from '../api'
type ListItem = {
name: string
count: number
link: string
}
export class RegionsTable implements Table {
async create() {
const parser = new LogParser()
const logsStorage = new Storage(LOGS_DIR)
const generatorsLog = await logsStorage.load('generators.log')
const parsed = parser.parse(generatorsLog)
const logRegions = parsed.filter((logItem: LogItem) => logItem.type === 'region')
let items = new Collection<ListItem>()
data.regions.forEach((region: sdk.Models.Region) => {
const logItem = logRegions.find(
(logItem: LogItem) => logItem.filepath === `regions/${region.code.toLowerCase()}.m3u`
)
if (!logItem) return
items.add({
name: region.name,
count: logItem.count,
link: `https://iptv-org.github.io/iptv/${logItem.filepath}`
})
})
items = items.sortBy(item => item.name)
const output = items
.map(item => {
return `- ${item.name} <code>${item.link}</code>`
})
.join('\r\n')
const readmeStorage = new Storage(README_DIR)
await readmeStorage.save('_regions.md', output)
}
}

6
scripts/tables/table.ts
Unescape Escape View File

@ -1,3 +1,3 @@
export interface Table {
make(): void
}
export interface Table {
create(): void
}

5
scripts/types/blocklistRecord.d.ts vendored
Unescape Escape View File

@ -1,5 +0,0 @@
export type BlocklistRecordData = {
channel: string
reason: string
ref: string
}

9
scripts/types/category.d.ts vendored
Unescape Escape View File

@ -1,9 +0,0 @@
export type CategorySerializedData = {
id: string
name: string
}
export type CategoryData = {
id: string
name: string
}

50
scripts/types/channel.d.ts vendored
Unescape Escape View File

@ -1,50 +0,0 @@
import { Collection } from '@freearhey/core'
import type { CountrySerializedData } from './country'
import type { SubdivisionSerializedData } from './subdivision'
import type { CategorySerializedData } from './category'
export type ChannelSerializedData = {
id: string
name: string
altNames: string[]
network?: string
owners: string[]
countryCode: string
country?: CountrySerializedData
subdivisionCode?: string
subdivision?: SubdivisionSerializedData
cityName?: string
categoryIds: string[]
categories?: CategorySerializedData[]
isNSFW: boolean
launched?: string
closed?: string
replacedBy?: string
website?: string
}
export type ChannelData = {
id: string
name: string
alt_names: string[]
network: string
owners: Collection
country: string
subdivision: string
city: string
categories: Collection
is_nsfw: boolean
launched: string
closed: string
replaced_by: string
website: string
}
export type ChannelSearchableData = {
id: string
name: string
altNames: string[]
guideNames: string[]
streamTitles: string[]
feedFullNames: string[]
}

20
scripts/types/city.d.ts vendored
Unescape Escape View File

@ -1,20 +0,0 @@
import { CountrySerializedData } from './country'
import { SubdivisionSerializedData } from './subdivision'
export type CitySerializedData = {
code: string
name: string
countryCode: string
country?: CountrySerializedData
subdivisionCode: string | null
subdivision?: SubdivisionSerializedData
wikidataId: string
}
export type CityData = {
code: string
name: string
country: string
subdivision: string | null
wikidata_id: string
}

20
scripts/types/country.d.ts vendored
Unescape Escape View File

@ -1,20 +0,0 @@
import type { LanguageSerializedData } from './language'
import type { SubdivisionSerializedData } from './subdivision'
import type { RegionSerializedData } from './region'
export type CountrySerializedData = {
code: string
name: string
flag: string
languageCode: string
language: LanguageSerializedData | null
subdivisions: SubdivisionSerializedData[]
regions: RegionSerializedData[]
}
export type CountryData = {
code: string
name: string
lang: string
flag: string
}

21
scripts/types/dataLoader.d.ts vendored
Unescape Escape View File

@ -1,21 +0,0 @@
import { Storage } from '@freearhey/core'
export type DataLoaderProps = {
storage: Storage
}
export type DataLoaderData = {
countries: object | object[]
regions: object | object[]
subdivisions: object | object[]
languages: object | object[]
categories: object | object[]
blocklist: object | object[]
channels: object | object[]
feeds: object | object[]
logos: object | object[]
timezones: object | object[]
guides: object | object[]
streams: object | object[]
cities: object | object[]
}

31
scripts/types/dataProcessor.d.ts vendored
Unescape Escape View File

@ -1,31 +0,0 @@
import { Collection, Dictionary } from '@freearhey/core'
export type DataProcessorData = {
blocklistRecordsGroupedByChannelId: Dictionary
subdivisionsGroupedByCountryCode: Dictionary
feedsGroupedByChannelId: Dictionary
guidesGroupedByStreamId: Dictionary
logosGroupedByStreamId: Dictionary
subdivisionsKeyByCode: Dictionary
countriesKeyByCode: Dictionary
languagesKeyByCode: Dictionary
streamsGroupedById: Dictionary
categoriesKeyById: Dictionary
timezonesKeyById: Dictionary
regionsKeyByCode: Dictionary
blocklistRecords: Collection
channelsKeyById: Dictionary
citiesKeyByCode: Dictionary
subdivisions: Collection
categories: Collection
countries: Collection
languages: Collection
timezones: Collection
channels: Collection
regions: Collection
streams: Collection
cities: Collection
guides: Collection
feeds: Collection
logos: Collection
}

10
scripts/types/feed.d.ts vendored
Unescape Escape View File

@ -1,10 +0,0 @@
export type FeedData = {
channel: string
id: string
name: string
is_main: boolean
broadcast_area: string[]
languages: string[]
timezones: string[]
video_format: string
}

17
scripts/types/guide.d.ts vendored
Unescape Escape View File

@ -1,17 +0,0 @@
export type GuideSerializedData = {
channelId?: string
feedId?: string
siteDomain: string
siteId: string
siteName: string
languageCode: string
}
export type GuideData = {
channel: string
feed: string
site: string
site_id: string
site_name: string
lang: string
}

9
scripts/types/language.d.ts vendored
Unescape Escape View File

@ -1,9 +0,0 @@
export type LanguageSerializedData = {
code: string
name: string
}
export type LanguageData = {
code: string
name: string
}

9
scripts/types/logo.d.ts vendored
Unescape Escape View File

@ -1,9 +0,0 @@
export type LogoData = {
channel: string
feed: string | null
tags: string[]
width: number
height: number
format: string | null
url: string
}

18
scripts/types/region.d.ts vendored
Unescape Escape View File

@ -1,18 +0,0 @@
import { CitySerializedData } from './city'
import { CountrySerializedData } from './country'
import { SubdivisionSerializedData } from './subdivision'
export type RegionSerializedData = {
code: string
name: string
countryCodes: string[]
countries?: CountrySerializedData[]
subdivisions?: SubdivisionSerializedData[]
cities?: CitySerializedData[]
}
export type RegionData = {
code: string
name: string
countries: string[]
}

11
scripts/types/stream.d.ts vendored
Unescape Escape View File

@ -1,11 +0,0 @@
export type StreamData = {
channelId: string | null
feedId: string | null
title: string | null
url: string
referrer: string | null
userAgent: string | null
quality: string | null
label: string | null
directives: string[]
}

16
scripts/types/subdivision.d.ts vendored
Unescape Escape View File

@ -1,16 +0,0 @@
import { CountrySerializedData } from './country'
export type SubdivisionSerializedData = {
code: string
name: string
countryCode: string
country?: CountrySerializedData
parentCode: string | null
}
export type SubdivisionData = {
code: string
name: string
country: string
parent: string | null
}

31
scripts/utils.ts
Unescape Escape View File

@ -1,8 +1,23 @@
export function isURI(string: string): boolean {
try {
new URL(string)
return true
} catch {
return false
}
}
import normalizeUrl from 'normalize-url'
export function isURI(string: string): boolean {
try {
new URL(string)
return true
} catch {
return false
}
}
export function normalizeURL(url: string): string {
const normalized = normalizeUrl(url, { stripWWW: false })
return decodeURIComponent(normalized).replace(/\s/g, '+').toString()
}
export function truncate(string: string, limit: number = 100) {
if (!string) return string
if (string.length < limit) return string
return string.slice(0, limit - 3) + '...'
}

Write Preview
Loading…
Cancel
Save