aiden
/
ytdl-android
mirror of https://github.com/deniscerri/ytdlnis
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

try fix sabr long format list parsing

Browse Source
pull/1323/head
deniscerri 1 month ago
parent ca36ebd6c3
commit 4f89cd9ece
No known key found for this signature in database
GPG Key ID: 95C43D517D830350
5 changed files with 224 additions and 64 deletions
Show Stats Download Patch File Download Diff File

1
app/src/main/AndroidManifest.xml
Unescape Escape View File

@ -37,6 +37,7 @@
<application
android:name=".App"
android:allowBackup="false"
android:largeHeap="true"
android:configChanges="orientation|screenSize|smallestScreenSize|screenLayout|locale"
android:dataExtractionRules="@xml/data_extraction_rules"
android:enableOnBackInvokedCallback="true"

90
app/src/main/java/com/deniscerri/ytdl/core/RuntimeManager.kt
Unescape Escape View File

@ -29,6 +29,7 @@ import kotlinx.coroutines.withTimeoutOrNull
import org.apache.commons.io.FileUtils
import java.io.File
import java.io.IOException
import java.io.InputStream
import java.util.Collections
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
@ -229,13 +230,10 @@ object RuntimeManager {
class CanceledException : Exception()
fun execute(
private fun buildYTDLCommand(
request: YTDLRequest,
processId: String? = null,
redirectErrorStream: Boolean = false,
usingCacheDir: Boolean = false,
callback: ((Float, Long, String) -> Unit)? = null
) : ExecuteResponse {
usingCacheDir: Boolean
): List<String> {
assertInit()
assertNoUpdate()
@ -268,7 +266,17 @@ object RuntimeManager {
request.addOption("--progress-delta", 0.1)
val fullCommand = mutableListOf<String>(pythonLocation.executable.absolutePath, ytdlpPath!!.absolutePath) + request.buildCommand()
return mutableListOf(pythonLocation.executable.absolutePath, ytdlpPath!!.absolutePath) + request.buildCommand()
}
fun execute(
request: YTDLRequest,
processId: String? = null,
redirectErrorStream: Boolean = false,
usingCacheDir: Boolean = false,
callback: ((Float, Long, String) -> Unit)? = null
) : ExecuteResponse {
val fullCommand = buildYTDLCommand(request, usingCacheDir)
return executeImpl(fullCommand, processId, redirectErrorStream, callback = callback)
}
@ -359,6 +367,74 @@ object RuntimeManager {
}
}
private fun startProcess(
fullCommand: List<String>,
processId: String?,
executeDirectory: File?
): Process {
if (processId != null && idProcessMap.containsKey(processId)) {
throw ExecuteException("Process ID already exists")
}
val processBuilder = ProcessBuilder(fullCommand)
processBuilder.environment().putAll(getEnvironment())
if (executeDirectory != null) {
processBuilder.directory(executeDirectory)
}
return try {
processBuilder.start().also {
if (processId != null) idProcessMap[processId] = it
}
} catch (e: IOException) {
throw ExecuteException(e)
}
}
fun <T> executeStreaming(
request: YTDLRequest,
processId: String? = null,
usingCacheDir: Boolean = false,
outputHandler: (InputStream) -> T
): T {
val fullCommand = buildYTDLCommand(request, usingCacheDir)
return executeStreamingImpl(fullCommand, processId, outputHandler = outputHandler)
}
fun <T> executeStreamingImpl(
fullCommand: List<String>,
processId: String? = null,
executeDirectory: File? = null,
outputHandler: (InputStream) -> T
): T {
val process = startProcess(fullCommand, processId, executeDirectory = executeDirectory)
val errBuffer = StringBuffer()
return try {
val stdErrProcessor = StreamGobbler(errBuffer, process.errorStream)
// Consume + fully drain stdout via the caller's handler BEFORE waitFor(),
// to avoid deadlocking on a full stdout pipe while the process still runs.
val result = process.inputStream.use { outputHandler(it) }
stdErrProcessor.join()
val exitCode = process.waitFor()
val err = errBuffer.toString()
if (exitCode != 0) {
if (processId != null && !idProcessMap.containsKey(processId)) throw CanceledException()
throw ExecuteException(err)
}
result
} catch (e: InterruptedException) {
process.destroy()
throw e
} finally {
if (processId != null) idProcessMap.remove(processId)
}
}
fun getEnvironment() : Map<String, String?> {
val env = mutableMapOf<String, String?>()

45
app/src/main/java/com/deniscerri/ytdl/util/Extensions.kt
Unescape Escape View File

@ -56,6 +56,8 @@ import com.google.android.material.bottomsheet.BottomSheetBehavior
import com.google.android.material.bottomsheet.BottomSheetDialog
import com.google.android.material.chip.Chip
import com.google.android.material.tabs.TabLayout
import com.google.gson.stream.JsonReader
import com.google.gson.stream.JsonToken
import com.neoutils.highlight.core.Highlight
import com.neoutils.highlight.core.scheme.TextColorScheme
import com.neoutils.highlight.core.util.UiColor
@ -69,6 +71,7 @@ import kotlinx.coroutines.flow.combine
import kotlinx.coroutines.launch
import kotlinx.serialization.json.Json
import me.zhanghai.android.fastscroll.FastScrollerBuilder
import org.json.JSONArray
import org.json.JSONObject
import java.io.File
import java.net.HttpCookie
@ -720,6 +723,48 @@ object Extensions {
}
}
fun readJsonValue(reader: JsonReader, skipKeys: Set<String> = setOf()): Any {
return when (reader.peek()) {
JsonToken.BEGIN_ARRAY -> {
val array = JSONArray()
reader.beginArray()
while (reader.hasNext()) {
array.put(readJsonValue(reader))
}
reader.endArray()
array
}
JsonToken.BEGIN_OBJECT -> {
val obj = JSONObject()
reader.beginObject()
while (reader.hasNext()) {
val name = reader.nextName()
if (name in skipKeys) {
reader.skipValue() // never parsed, never allocated
continue
}
obj.put(name, readJsonValue(reader))
}
reader.endObject()
obj
}
JsonToken.STRING -> reader.nextString()
JsonToken.NUMBER -> {
val raw = reader.nextString()
raw.toLongOrNull() ?: raw.toDoubleOrNull() ?: raw
}
JsonToken.BOOLEAN -> reader.nextBoolean()
JsonToken.NULL -> {
reader.nextNull()
JSONObject.NULL
}
else -> {
reader.skipValue()
JSONObject.NULL
}
}
}
fun String.hasPermission(context: Context) : Boolean {
val packageInfo = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
context.packageManager.getPackageInfo(

151
app/src/main/java/com/deniscerri/ytdl/util/extractors/ytdlp/YTDLPUtil.kt
Unescape Escape View File

@ -31,19 +31,25 @@ import com.deniscerri.ytdl.util.Extensions.isSoundCloudURL
import com.deniscerri.ytdl.util.Extensions.isURL
import com.deniscerri.ytdl.util.Extensions.isYoutubeURL
import com.deniscerri.ytdl.util.Extensions.isYoutubeWatchVideosURL
import com.deniscerri.ytdl.util.Extensions.readJsonValue
import com.deniscerri.ytdl.util.Extensions.toStringDuration
import com.deniscerri.ytdl.util.FileUtil
import com.deniscerri.ytdl.util.FormatUtil
import com.google.gson.Gson
import com.google.gson.Strictness
import com.google.gson.reflect.TypeToken
import com.google.gson.stream.JsonReader
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withContext
import org.json.JSONArray
import org.json.JSONObject
import org.json.JSONTokener
import java.io.File
import java.io.InputStreamReader
import java.lang.reflect.Type
import java.nio.charset.StandardCharsets
import java.util.Locale
import java.util.StringJoiner
import java.util.UUID
@ -482,7 +488,7 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co
fun getFormats(url: String) : List<Format> {
val request = YTDLRequest(url)
request.addOption("--print", "%(formats)s")
request.addOption("--print", "%(formats)j")
request.addOption("--print", "%(duration)s")
request.applyDefaultOptionsForFetchingData(url)
if (url.isYoutubeURL()) {
@ -503,85 +509,116 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co
}
}
val res = RuntimeManager.getInstance().execute(request)
val results: Array<String?> = try {
res.out.split(System.lineSeparator()).toTypedArray()
val formats = try {
RuntimeManager.getInstance().executeStreaming(request) { stream ->
JsonReader(InputStreamReader(stream, StandardCharsets.UTF_8)).use { reader ->
reader.strictness = Strictness.LENIENT
readFormatsList(reader)
}
}
} catch (e: Exception) {
arrayOf(res.out)
arrayListOf()
}
val json = results[0]
val jsonArray = runCatching { JSONArray(json) }.getOrElse { JSONArray() }
val formats = parseYTDLFormats(jsonArray)
if (formats.isEmpty()) {
runCatching {
getInfoJsonFile(url)?.apply {
this.delete()
}
getInfoJsonFile(url)?.apply { this.delete() }
}
}
return formats
}
private fun parseYTDLFormats(formatsInJSON: JSONArray?) : ArrayList<Format> {
fun readFormatsList(reader: JsonReader): ArrayList<Format> {
val formats = arrayListOf<Format>()
val seenFormatIds = HashSet<String>()
if (formatsInJSON != null) {
for (f in formatsInJSON.length() - 1 downTo 0){
val format = formatsInJSON.getJSONObject(f)
runCatching {
if (format.get("filesize").toString() == "None") {
format.remove("filesize")
}
}
reader.beginArray()
while (reader.hasNext()) {
val obj = readFormatObject(reader) // builds one JSONObject, keep-list filtered
runCatching {
if (format.get("filesize_approx").toString() == "None") {
format.remove("filesize_approx")
}
}
val id = obj.optString("format_id").ifBlank { obj.optString("itag") }
if (id.isNotBlank() && !seenFormatIds.add(id)) {
continue // duplicate - discard immediately, never converted to Format
}
runCatching {
if(format.get("format_note").toString() == "null"){
format.remove("format_note")
}
}
val formatProper = parseOneFormat(obj) ?: continue
formats.add(formatProper)
// `obj` falls out of scope here - eligible for GC immediately,
// nothing keeps N of them alive at once
}
reader.endArray()
val formatProper = Gson().fromJson(format.toString(), Format::class.java)
if (formatProper.format_note == null) formatProper.format_note = ""
formats.reverse()
return formats
}
val resolution = format.getString("resolution")
if (format.has("format_note")){
if (!formatProper!!.format_note.contains("audio only", true)) {
formatProper.format_note = format.getString("format_note")
}else{
if (!formatProper.format_note.endsWith("audio", true)){
formatProper.format_note = format.getString("format_note").uppercase().removeSuffix("AUDIO").trim() + " AUDIO"
}
}
private fun readFormatObject(reader: JsonReader): JSONObject {
val obj = JSONObject()
reader.beginObject()
while (reader.hasNext()) {
val name = reader.nextName()
obj.put(name, readJsonValue(reader))
}
reader.endObject()
return obj
}
if (!resolution.isNullOrBlank() && resolution != "audio only") {
formatProper.format_note = "${formatProper.format_note} (${resolution})"
}
}
private fun parseOneFormat(format: JSONObject): Format? {
runCatching {
if (format.get("filesize").toString() == "None") format.remove("filesize")
}
runCatching {
if (format.get("filesize_approx").toString() == "None") format.remove("filesize_approx")
}
runCatching {
if (format.get("format_note").toString() == "null") format.remove("format_note")
}
if (formatProper.format_note.contains("storyboard", ignoreCase = true)) continue
val formatProper = Gson().fromJson(format.toString(), Format::class.java)
if (formatProper.format_note == null) formatProper.format_note = ""
formatProper.format_note = formatProper.format_note.trim()
formatProper.container = format.getString("ext")
if (formatProper.tbr == "None") formatProper.tbr = ""
if (!formatProper.tbr.isNullOrBlank()){
formatProper.tbr += "k"
val resolution = format.optString("resolution")
if (format.has("format_note")) {
if (!formatProper.format_note.contains("audio only", true)) {
formatProper.format_note = format.getString("format_note")
} else {
if (!formatProper.format_note.endsWith("audio", true)) {
formatProper.format_note = format.getString("format_note").uppercase().removeSuffix("AUDIO").trim() + " AUDIO"
}
}
if (!resolution.isNullOrBlank() && resolution != "audio only") {
formatProper.format_note = "${formatProper.format_note} (${resolution})"
}
}
if(formatProper.vcodec.isNullOrEmpty() || formatProper.vcodec == "null"){
if(formatProper.acodec.isNullOrEmpty() || formatProper.acodec == "null"){
formatProper.vcodec = format.getStringByAny("video_ext", "ext").ifEmpty { "unknown" }
}
}
if (formatProper.format_note.contains("storyboard", ignoreCase = true)) return null
formatProper.format_note = formatProper.format_note.trim()
formatProper.container = format.getString("ext")
if (formatProper.tbr == "None") formatProper.tbr = ""
if (!formatProper.tbr.isNullOrBlank()) {
formatProper.tbr += "k"
}
formats.add(formatProper)
if (formatProper.vcodec.isNullOrEmpty() || formatProper.vcodec == "null") {
if (formatProper.acodec.isNullOrEmpty() || formatProper.acodec == "null") {
formatProper.vcodec = format.getStringByAny("video_ext", "ext").ifEmpty { "unknown" }
}
}
return formatProper
}
private fun parseYTDLFormats(formatsInJSON: JSONArray?) : ArrayList<Format> {
val formats = arrayListOf<Format>()
if (formatsInJSON != null) {
for (f in formatsInJSON.length() - 1 downTo 0){
val formatRaw = formatsInJSON.getJSONObject(f)
val format = parseOneFormat(formatRaw) ?: continue
formats.add(format)
}
}
return formats

1
app/src/main/res/layout/fragment_download_log.xml
Unescape Escape View File

@ -113,6 +113,7 @@
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:contentDescription="@string/copy_log"
android:stateListAnimator="@null"
android:text="@string/copy_log"
app:icon="@drawable/ic_copy"
app:layout_anchor="@id/bottomAppBar" />

Write Preview
Loading…
Cancel
Save