diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 9b82d481..e68a41f2 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -37,6 +37,7 @@ Unit)? = null - ) : ExecuteResponse { + usingCacheDir: Boolean + ): List { assertInit() assertNoUpdate() @@ -268,7 +266,17 @@ object RuntimeManager { request.addOption("--progress-delta", 0.1) - val fullCommand = mutableListOf(pythonLocation.executable.absolutePath, ytdlpPath!!.absolutePath) + request.buildCommand() + return mutableListOf(pythonLocation.executable.absolutePath, ytdlpPath!!.absolutePath) + request.buildCommand() + } + + fun execute( + request: YTDLRequest, + processId: String? = null, + redirectErrorStream: Boolean = false, + usingCacheDir: Boolean = false, + callback: ((Float, Long, String) -> Unit)? = null + ) : ExecuteResponse { + val fullCommand = buildYTDLCommand(request, usingCacheDir) return executeImpl(fullCommand, processId, redirectErrorStream, callback = callback) } @@ -359,6 +367,74 @@ object RuntimeManager { } } + private fun startProcess( + fullCommand: List, + processId: String?, + executeDirectory: File? + ): Process { + if (processId != null && idProcessMap.containsKey(processId)) { + throw ExecuteException("Process ID already exists") + } + + val processBuilder = ProcessBuilder(fullCommand) + processBuilder.environment().putAll(getEnvironment()) + if (executeDirectory != null) { + processBuilder.directory(executeDirectory) + } + + return try { + processBuilder.start().also { + if (processId != null) idProcessMap[processId] = it + } + } catch (e: IOException) { + throw ExecuteException(e) + } + } + + fun executeStreaming( + request: YTDLRequest, + processId: String? = null, + usingCacheDir: Boolean = false, + outputHandler: (InputStream) -> T + ): T { + val fullCommand = buildYTDLCommand(request, usingCacheDir) + return executeStreamingImpl(fullCommand, processId, outputHandler = outputHandler) + } + + fun executeStreamingImpl( + fullCommand: List, + processId: String? = null, + executeDirectory: File? = null, + outputHandler: (InputStream) -> T + ): T { + val process = startProcess(fullCommand, processId, executeDirectory = executeDirectory) + val errBuffer = StringBuffer() + + return try { + val stdErrProcessor = StreamGobbler(errBuffer, process.errorStream) + + // Consume + fully drain stdout via the caller's handler BEFORE waitFor(), + // to avoid deadlocking on a full stdout pipe while the process still runs. + val result = process.inputStream.use { outputHandler(it) } + + stdErrProcessor.join() + val exitCode = process.waitFor() + val err = errBuffer.toString() + + if (exitCode != 0) { + if (processId != null && !idProcessMap.containsKey(processId)) throw CanceledException() + throw ExecuteException(err) + } + + result + } catch (e: InterruptedException) { + process.destroy() + throw e + } finally { + if (processId != null) idProcessMap.remove(processId) + } + } + fun getEnvironment() : Map { val env = mutableMapOf() diff --git a/app/src/main/java/com/deniscerri/ytdl/util/Extensions.kt b/app/src/main/java/com/deniscerri/ytdl/util/Extensions.kt index 03ddcb32..93d09dcd 100644 --- a/app/src/main/java/com/deniscerri/ytdl/util/Extensions.kt +++ b/app/src/main/java/com/deniscerri/ytdl/util/Extensions.kt @@ -56,6 +56,8 @@ import com.google.android.material.bottomsheet.BottomSheetBehavior import com.google.android.material.bottomsheet.BottomSheetDialog import com.google.android.material.chip.Chip import com.google.android.material.tabs.TabLayout +import com.google.gson.stream.JsonReader +import com.google.gson.stream.JsonToken import com.neoutils.highlight.core.Highlight import com.neoutils.highlight.core.scheme.TextColorScheme import com.neoutils.highlight.core.util.UiColor @@ -69,6 +71,7 @@ import kotlinx.coroutines.flow.combine import kotlinx.coroutines.launch import kotlinx.serialization.json.Json import me.zhanghai.android.fastscroll.FastScrollerBuilder +import org.json.JSONArray import org.json.JSONObject import java.io.File import java.net.HttpCookie @@ -720,6 +723,48 @@ object Extensions { } } + fun readJsonValue(reader: JsonReader, skipKeys: Set = setOf()): Any { + return when (reader.peek()) { + JsonToken.BEGIN_ARRAY -> { + val array = JSONArray() + reader.beginArray() + while (reader.hasNext()) { + array.put(readJsonValue(reader)) + } + reader.endArray() + array + } + JsonToken.BEGIN_OBJECT -> { + val obj = JSONObject() + reader.beginObject() + while (reader.hasNext()) { + val name = reader.nextName() + if (name in skipKeys) { + reader.skipValue() // never parsed, never allocated + continue + } + obj.put(name, readJsonValue(reader)) + } + reader.endObject() + obj + } + JsonToken.STRING -> reader.nextString() + JsonToken.NUMBER -> { + val raw = reader.nextString() + raw.toLongOrNull() ?: raw.toDoubleOrNull() ?: raw + } + JsonToken.BOOLEAN -> reader.nextBoolean() + JsonToken.NULL -> { + reader.nextNull() + JSONObject.NULL + } + else -> { + reader.skipValue() + JSONObject.NULL + } + } + } + fun String.hasPermission(context: Context) : Boolean { val packageInfo = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { context.packageManager.getPackageInfo( diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/ytdlp/YTDLPUtil.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/ytdlp/YTDLPUtil.kt index 96e8b9b2..34db5341 100644 --- a/app/src/main/java/com/deniscerri/ytdl/util/extractors/ytdlp/YTDLPUtil.kt +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/ytdlp/YTDLPUtil.kt @@ -31,19 +31,25 @@ import com.deniscerri.ytdl.util.Extensions.isSoundCloudURL import com.deniscerri.ytdl.util.Extensions.isURL import com.deniscerri.ytdl.util.Extensions.isYoutubeURL import com.deniscerri.ytdl.util.Extensions.isYoutubeWatchVideosURL +import com.deniscerri.ytdl.util.Extensions.readJsonValue import com.deniscerri.ytdl.util.Extensions.toStringDuration import com.deniscerri.ytdl.util.FileUtil import com.deniscerri.ytdl.util.FormatUtil import com.google.gson.Gson +import com.google.gson.Strictness import com.google.gson.reflect.TypeToken +import com.google.gson.stream.JsonReader import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking import kotlinx.coroutines.withContext import org.json.JSONArray import org.json.JSONObject +import org.json.JSONTokener import java.io.File +import java.io.InputStreamReader import java.lang.reflect.Type +import java.nio.charset.StandardCharsets import java.util.Locale import java.util.StringJoiner import java.util.UUID @@ -482,7 +488,7 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co fun getFormats(url: String) : List { val request = YTDLRequest(url) - request.addOption("--print", "%(formats)s") + request.addOption("--print", "%(formats)j") request.addOption("--print", "%(duration)s") request.applyDefaultOptionsForFetchingData(url) if (url.isYoutubeURL()) { @@ -503,85 +509,116 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co } } - val res = RuntimeManager.getInstance().execute(request) - val results: Array = try { - res.out.split(System.lineSeparator()).toTypedArray() + val formats = try { + RuntimeManager.getInstance().executeStreaming(request) { stream -> + JsonReader(InputStreamReader(stream, StandardCharsets.UTF_8)).use { reader -> + reader.strictness = Strictness.LENIENT + readFormatsList(reader) + } + } } catch (e: Exception) { - arrayOf(res.out) + arrayListOf() } - val json = results[0] - val jsonArray = runCatching { JSONArray(json) }.getOrElse { JSONArray() } - val formats = parseYTDLFormats(jsonArray) if (formats.isEmpty()) { runCatching { - getInfoJsonFile(url)?.apply { - this.delete() - } + getInfoJsonFile(url)?.apply { this.delete() } } } return formats } - private fun parseYTDLFormats(formatsInJSON: JSONArray?) : ArrayList { + fun readFormatsList(reader: JsonReader): ArrayList { val formats = arrayListOf() + val seenFormatIds = HashSet() - if (formatsInJSON != null) { - for (f in formatsInJSON.length() - 1 downTo 0){ - val format = formatsInJSON.getJSONObject(f) - runCatching { - if (format.get("filesize").toString() == "None") { - format.remove("filesize") - } - } + reader.beginArray() + while (reader.hasNext()) { + val obj = readFormatObject(reader) // builds one JSONObject, keep-list filtered - runCatching { - if (format.get("filesize_approx").toString() == "None") { - format.remove("filesize_approx") - } - } + val id = obj.optString("format_id").ifBlank { obj.optString("itag") } + if (id.isNotBlank() && !seenFormatIds.add(id)) { + continue // duplicate - discard immediately, never converted to Format + } - runCatching { - if(format.get("format_note").toString() == "null"){ - format.remove("format_note") - } - } + val formatProper = parseOneFormat(obj) ?: continue + formats.add(formatProper) + // `obj` falls out of scope here - eligible for GC immediately, + // nothing keeps N of them alive at once + } + reader.endArray() - val formatProper = Gson().fromJson(format.toString(), Format::class.java) - if (formatProper.format_note == null) formatProper.format_note = "" + formats.reverse() + return formats + } - val resolution = format.getString("resolution") - if (format.has("format_note")){ - if (!formatProper!!.format_note.contains("audio only", true)) { - formatProper.format_note = format.getString("format_note") - }else{ - if (!formatProper.format_note.endsWith("audio", true)){ - formatProper.format_note = format.getString("format_note").uppercase().removeSuffix("AUDIO").trim() + " AUDIO" - } - } + private fun readFormatObject(reader: JsonReader): JSONObject { + val obj = JSONObject() + reader.beginObject() + while (reader.hasNext()) { + val name = reader.nextName() + obj.put(name, readJsonValue(reader)) + } + reader.endObject() + return obj + } - if (!resolution.isNullOrBlank() && resolution != "audio only") { - formatProper.format_note = "${formatProper.format_note} (${resolution})" - } - } + private fun parseOneFormat(format: JSONObject): Format? { + runCatching { + if (format.get("filesize").toString() == "None") format.remove("filesize") + } + runCatching { + if (format.get("filesize_approx").toString() == "None") format.remove("filesize_approx") + } + runCatching { + if (format.get("format_note").toString() == "null") format.remove("format_note") + } - if (formatProper.format_note.contains("storyboard", ignoreCase = true)) continue + val formatProper = Gson().fromJson(format.toString(), Format::class.java) + if (formatProper.format_note == null) formatProper.format_note = "" - formatProper.format_note = formatProper.format_note.trim() - formatProper.container = format.getString("ext") - if (formatProper.tbr == "None") formatProper.tbr = "" - if (!formatProper.tbr.isNullOrBlank()){ - formatProper.tbr += "k" + val resolution = format.optString("resolution") + if (format.has("format_note")) { + if (!formatProper.format_note.contains("audio only", true)) { + formatProper.format_note = format.getString("format_note") + } else { + if (!formatProper.format_note.endsWith("audio", true)) { + formatProper.format_note = format.getString("format_note").uppercase().removeSuffix("AUDIO").trim() + " AUDIO" } + } + if (!resolution.isNullOrBlank() && resolution != "audio only") { + formatProper.format_note = "${formatProper.format_note} (${resolution})" + } + } - if(formatProper.vcodec.isNullOrEmpty() || formatProper.vcodec == "null"){ - if(formatProper.acodec.isNullOrEmpty() || formatProper.acodec == "null"){ - formatProper.vcodec = format.getStringByAny("video_ext", "ext").ifEmpty { "unknown" } - } - } + if (formatProper.format_note.contains("storyboard", ignoreCase = true)) return null + + formatProper.format_note = formatProper.format_note.trim() + formatProper.container = format.getString("ext") + if (formatProper.tbr == "None") formatProper.tbr = "" + if (!formatProper.tbr.isNullOrBlank()) { + formatProper.tbr += "k" + } - formats.add(formatProper) + if (formatProper.vcodec.isNullOrEmpty() || formatProper.vcodec == "null") { + if (formatProper.acodec.isNullOrEmpty() || formatProper.acodec == "null") { + formatProper.vcodec = format.getStringByAny("video_ext", "ext").ifEmpty { "unknown" } + } + } + + return formatProper + } + + + private fun parseYTDLFormats(formatsInJSON: JSONArray?) : ArrayList { + val formats = arrayListOf() + + if (formatsInJSON != null) { + for (f in formatsInJSON.length() - 1 downTo 0){ + val formatRaw = formatsInJSON.getJSONObject(f) + val format = parseOneFormat(formatRaw) ?: continue + formats.add(format) } } return formats diff --git a/app/src/main/res/layout/fragment_download_log.xml b/app/src/main/res/layout/fragment_download_log.xml index fe9db267..fc188bca 100644 --- a/app/src/main/res/layout/fragment_download_log.xml +++ b/app/src/main/res/layout/fragment_download_log.xml @@ -113,6 +113,7 @@ android:layout_width="wrap_content" android:layout_height="wrap_content" android:contentDescription="@string/copy_log" + android:stateListAnimator="@null" android:text="@string/copy_log" app:icon="@drawable/ic_copy" app:layout_anchor="@id/bottomAppBar" />