mirror of https://github.com/deniscerri/ytdlnis
1.8.3 pre
deniscerri
1 year ago
parent
e4c08c2c45
commit
3b3f79719d
@ -0,0 +1,127 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en"><head><title></title><script>
|
||||||
|
/**
|
||||||
|
* Factory method to create and load a BotGuardClient instance.
|
||||||
|
* @param options - Configuration options for the BotGuardClient.
|
||||||
|
* @returns A promise that resolves to a loaded BotGuardClient instance.
|
||||||
|
*/
|
||||||
|
function loadBotGuard(challengeData) {
|
||||||
|
this.vm = this[challengeData.globalName];
|
||||||
|
this.program = challengeData.program;
|
||||||
|
this.vmFunctions = {};
|
||||||
|
this.syncSnapshotFunction = null;
|
||||||
|
|
||||||
|
if (!this.vm)
|
||||||
|
throw new Error('[BotGuardClient]: VM not found in the global object');
|
||||||
|
|
||||||
|
if (!this.vm.a)
|
||||||
|
throw new Error('[BotGuardClient]: Could not load program');
|
||||||
|
|
||||||
|
const vmFunctionsCallback = function (
|
||||||
|
asyncSnapshotFunction,
|
||||||
|
shutdownFunction,
|
||||||
|
passEventFunction,
|
||||||
|
checkCameraFunction
|
||||||
|
) {
|
||||||
|
this.vmFunctions = {
|
||||||
|
asyncSnapshotFunction: asyncSnapshotFunction,
|
||||||
|
shutdownFunction: shutdownFunction,
|
||||||
|
passEventFunction: passEventFunction,
|
||||||
|
checkCameraFunction: checkCameraFunction
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
this.syncSnapshotFunction = this.vm.a(this.program, vmFunctionsCallback, true, this.userInteractionElement, function () {/** no-op */ }, [ [], [] ])[0]
|
||||||
|
|
||||||
|
// an asynchronous function runs in the background and it will eventually call
|
||||||
|
// `vmFunctionsCallback`, however we need to manually tell JavaScript to pass
|
||||||
|
// control to the things running in the background by interrupting this async
|
||||||
|
// function in any way, e.g. with a delay of 1ms. The loop is most probably not
|
||||||
|
// needed but is there just because.
|
||||||
|
return new Promise(function (resolve, reject) {
|
||||||
|
i = 0
|
||||||
|
refreshIntervalId = setInterval(function () {
|
||||||
|
if (!!this.vmFunctions.asyncSnapshotFunction) {
|
||||||
|
resolve(this)
|
||||||
|
clearInterval(refreshIntervalId);
|
||||||
|
}
|
||||||
|
if (i >= 10000) {
|
||||||
|
reject("asyncSnapshotFunction is null even after 10 seconds")
|
||||||
|
clearInterval(refreshIntervalId);
|
||||||
|
}
|
||||||
|
i += 1;
|
||||||
|
}, 1);
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes a snapshot asynchronously.
|
||||||
|
* @returns The snapshot result.
|
||||||
|
* @example
|
||||||
|
* ```ts
|
||||||
|
* const result = await botguard.snapshot({
|
||||||
|
* contentBinding: {
|
||||||
|
* c: "a=6&a2=10&b=SZWDwKVIuixOp7Y4euGTgwckbJA&c=1729143849&d=1&t=7200&c1a=1&c6a=1&c6b=1&hh=HrMb5mRWTyxGJphDr0nW2Oxonh0_wl2BDqWuLHyeKLo",
|
||||||
|
* e: "ENGAGEMENT_TYPE_VIDEO_LIKE",
|
||||||
|
* encryptedVideoId: "P-vC09ZJcnM"
|
||||||
|
* }
|
||||||
|
* });
|
||||||
|
*
|
||||||
|
* console.log(result);
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
function snapshot(args) {
|
||||||
|
return new Promise(function (resolve, reject) {
|
||||||
|
if (!this.vmFunctions.asyncSnapshotFunction)
|
||||||
|
return reject(new Error('[BotGuardClient]: Async snapshot function not found'));
|
||||||
|
|
||||||
|
this.vmFunctions.asyncSnapshotFunction(function (response) { resolve(response) }, [
|
||||||
|
args.contentBinding,
|
||||||
|
args.signedTimestamp,
|
||||||
|
args.webPoSignalOutput,
|
||||||
|
args.skipPrivacyBuffer
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function runBotGuard(challengeData) {
|
||||||
|
const interpreterJavascript = challengeData.interpreterJavascript.privateDoNotAccessOrElseSafeScriptWrappedValue;
|
||||||
|
|
||||||
|
if (interpreterJavascript) {
|
||||||
|
new Function(interpreterJavascript)();
|
||||||
|
} else throw new Error('Could not load VM');
|
||||||
|
|
||||||
|
const webPoSignalOutput = [];
|
||||||
|
return loadBotGuard({
|
||||||
|
globalName: challengeData.globalName,
|
||||||
|
globalObj: this,
|
||||||
|
program: challengeData.program
|
||||||
|
}).then(function (botguard) {
|
||||||
|
return botguard.snapshot({ webPoSignalOutput: webPoSignalOutput })
|
||||||
|
}).then(function (botguardResponse) {
|
||||||
|
return { webPoSignalOutput: webPoSignalOutput, botguardResponse: botguardResponse }
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function obtainPoToken(webPoSignalOutput, integrityToken, identifier) {
|
||||||
|
const getMinter = webPoSignalOutput[0];
|
||||||
|
|
||||||
|
if (!getMinter)
|
||||||
|
throw new Error('PMD:Undefined');
|
||||||
|
|
||||||
|
const mintCallback = getMinter(integrityToken);
|
||||||
|
|
||||||
|
if (!(mintCallback instanceof Function))
|
||||||
|
throw new Error('APF:Failed');
|
||||||
|
|
||||||
|
const result = mintCallback(identifier);
|
||||||
|
|
||||||
|
if (!result)
|
||||||
|
throw new Error('YNJ:Undefined');
|
||||||
|
|
||||||
|
if (!(result instanceof Uint8Array))
|
||||||
|
throw new Error('ODM:Invalid');
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
</script></head><body></body></html>
|
||||||
@ -0,0 +1,131 @@
|
|||||||
|
package com.deniscerri.ytdl.util.extractors.newpipe.potoken
|
||||||
|
|
||||||
|
import kotlinx.serialization.json.Json
|
||||||
|
import kotlinx.serialization.json.JsonNull
|
||||||
|
import kotlinx.serialization.json.JsonObject
|
||||||
|
import kotlinx.serialization.json.JsonPrimitive
|
||||||
|
import kotlinx.serialization.json.jsonArray
|
||||||
|
import kotlinx.serialization.json.jsonPrimitive
|
||||||
|
import kotlinx.serialization.json.long
|
||||||
|
import okio.ByteString.Companion.decodeBase64
|
||||||
|
import okio.ByteString.Companion.toByteString
|
||||||
|
|
||||||
|
object JavascriptUtil {
|
||||||
|
/**
|
||||||
|
* Parses the raw challenge data obtained from the Create endpoint and returns an object that can be
|
||||||
|
* embedded in a JavaScript snippet.
|
||||||
|
*/
|
||||||
|
fun parseChallengeData(rawChallengeData: String): String {
|
||||||
|
val scrambled = Json.parseToJsonElement(rawChallengeData).jsonArray
|
||||||
|
|
||||||
|
val challengeData = if (scrambled.size > 1 && scrambled[1].jsonPrimitive.isString) {
|
||||||
|
val descrambled = descramble(scrambled[1].jsonPrimitive.content)
|
||||||
|
Json.parseToJsonElement(descrambled).jsonArray
|
||||||
|
} else {
|
||||||
|
scrambled[1].jsonArray
|
||||||
|
}
|
||||||
|
|
||||||
|
val messageId = challengeData[0].jsonPrimitive.content
|
||||||
|
val interpreterHash = challengeData[3].jsonPrimitive.content
|
||||||
|
val program = challengeData[4].jsonPrimitive.content
|
||||||
|
val globalName = challengeData[5].jsonPrimitive.content
|
||||||
|
val clientExperimentsStateBlob = challengeData[7].jsonPrimitive.content
|
||||||
|
|
||||||
|
val privateDoNotAccessOrElseSafeScriptWrappedValue = challengeData[1]
|
||||||
|
.takeIf { it !is JsonNull }
|
||||||
|
?.jsonArray
|
||||||
|
?.find { it.jsonPrimitive.isString }
|
||||||
|
val privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = challengeData[2]
|
||||||
|
.takeIf { it !is JsonNull }
|
||||||
|
?.jsonArray
|
||||||
|
?.find { it.jsonPrimitive.isString }
|
||||||
|
|
||||||
|
return Json.encodeToString(
|
||||||
|
JsonObject.serializer(), JsonObject(
|
||||||
|
mapOf(
|
||||||
|
"messageId" to JsonPrimitive(messageId),
|
||||||
|
"interpreterJavascript" to JsonObject(
|
||||||
|
mapOf(
|
||||||
|
"privateDoNotAccessOrElseSafeScriptWrappedValue" to (privateDoNotAccessOrElseSafeScriptWrappedValue
|
||||||
|
?: JsonNull),
|
||||||
|
"privateDoNotAccessOrElseTrustedResourceUrlWrappedValue" to (privateDoNotAccessOrElseTrustedResourceUrlWrappedValue
|
||||||
|
?: JsonNull)
|
||||||
|
)
|
||||||
|
),
|
||||||
|
"interpreterHash" to JsonPrimitive(interpreterHash),
|
||||||
|
"program" to JsonPrimitive(program),
|
||||||
|
"globalName" to JsonPrimitive(globalName),
|
||||||
|
"clientExperimentsStateBlob" to JsonPrimitive(clientExperimentsStateBlob)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses the raw integrity token data obtained from the GenerateIT endpoint to a JavaScript
|
||||||
|
* `Uint8Array` that can be embedded directly in JavaScript code, and an [Int] representing the
|
||||||
|
* duration of this token in seconds.
|
||||||
|
*/
|
||||||
|
fun parseIntegrityTokenData(rawIntegrityTokenData: String): Pair<String, Long> {
|
||||||
|
val integrityTokenData = Json.parseToJsonElement(rawIntegrityTokenData).jsonArray
|
||||||
|
return base64ToU8(integrityTokenData[0].jsonPrimitive.content) to integrityTokenData[1].jsonPrimitive.long
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Converts a string (usually the identifier used as input to `obtainPoToken`) to a JavaScript
|
||||||
|
* `Uint8Array` that can be embedded directly in JavaScript code.
|
||||||
|
*/
|
||||||
|
fun stringToU8(identifier: String): String {
|
||||||
|
return newUint8Array(identifier.toByteArray())
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes a poToken encoded as a sequence of bytes represented as integers separated by commas
|
||||||
|
* (e.g. "97,98,99" would be "abc"), which is the output of `Uint8Array::toString()` in JavaScript,
|
||||||
|
* and converts it to the specific base64 representation for poTokens.
|
||||||
|
*/
|
||||||
|
fun u8ToBase64(poToken: String): String {
|
||||||
|
return poToken.split(",")
|
||||||
|
.map { it.toUByte().toByte() }
|
||||||
|
.toByteArray()
|
||||||
|
.toByteString()
|
||||||
|
.base64()
|
||||||
|
.replace("+", "-")
|
||||||
|
.replace("/", "_")
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes the scrambled challenge, decodes it from base64, adds 97 to each byte.
|
||||||
|
*/
|
||||||
|
private fun descramble(scrambledChallenge: String): String {
|
||||||
|
return base64ToByteString(scrambledChallenge)
|
||||||
|
.map { (it + 97).toByte() }
|
||||||
|
.toByteArray()
|
||||||
|
.decodeToString()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decodes a base64 string encoded in the specific base64 representation used by YouTube, and
|
||||||
|
* returns a JavaScript `Uint8Array` that can be embedded directly in JavaScript code.
|
||||||
|
*/
|
||||||
|
private fun base64ToU8(base64: String): String {
|
||||||
|
return newUint8Array(base64ToByteString(base64))
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun newUint8Array(contents: ByteArray): String {
|
||||||
|
return "new Uint8Array([" + contents.joinToString(separator = ",") { it.toUByte().toString() } + "])"
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decodes a base64 string encoded in the specific base64 representation used by YouTube.
|
||||||
|
*/
|
||||||
|
private fun base64ToByteString(base64: String): ByteArray {
|
||||||
|
val base64Mod = base64
|
||||||
|
.replace('-', '+')
|
||||||
|
.replace('_', '/')
|
||||||
|
.replace('.', '=')
|
||||||
|
|
||||||
|
return (base64Mod.decodeBase64() ?: throw PoTokenException("Cannot base64 decode"))
|
||||||
|
.toByteArray()
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,142 @@
|
|||||||
|
package com.deniscerri.ytdl.util.extractors.newpipe.potoken
|
||||||
|
|
||||||
|
import android.os.Handler
|
||||||
|
import android.os.Looper
|
||||||
|
import android.util.Log
|
||||||
|
import android.webkit.CookieManager
|
||||||
|
import androidx.preference.PreferenceManager
|
||||||
|
import com.deniscerri.ytdl.App
|
||||||
|
import com.deniscerri.ytdl.BuildConfig
|
||||||
|
import com.deniscerri.ytdl.database.models.YoutubePlayerClientItem
|
||||||
|
import com.deniscerri.ytdl.database.models.YoutubePoTokenItem
|
||||||
|
import com.google.gson.Gson
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.runBlocking
|
||||||
|
import kotlinx.coroutines.sync.Mutex
|
||||||
|
import kotlinx.coroutines.sync.withLock
|
||||||
|
import kotlinx.coroutines.withContext
|
||||||
|
import org.schabi.newpipe.extractor.NewPipe
|
||||||
|
import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo
|
||||||
|
import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider
|
||||||
|
import org.schabi.newpipe.extractor.services.youtube.PoTokenResult
|
||||||
|
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper
|
||||||
|
|
||||||
|
class NewPipePoTokenGenerator : PoTokenProvider {
|
||||||
|
val TAG = NewPipePoTokenGenerator::class.simpleName
|
||||||
|
private val supportsWebView by lazy { runCatching { CookieManager.getInstance() }.isSuccess }
|
||||||
|
|
||||||
|
private object WebPoTokenGenLock
|
||||||
|
private var webPoTokenVisitorData: String? = null
|
||||||
|
private var webPoTokenStreamingPot: String? = null
|
||||||
|
private var webPoTokenGenerator: PoTokenWebView? = null
|
||||||
|
|
||||||
|
|
||||||
|
override fun getWebClientPoToken(videoId: String): PoTokenResult? {
|
||||||
|
if (!supportsWebView) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
val result = kotlin.runCatching {
|
||||||
|
getWebClientPoToken(videoId, false)
|
||||||
|
}
|
||||||
|
|
||||||
|
result.getOrNull()?.apply {
|
||||||
|
val preferences = PreferenceManager.getDefaultSharedPreferences(App.instance)
|
||||||
|
val editor = preferences.edit()
|
||||||
|
editor.putString("newpipe_gvs_potoken", this.streamingDataPoToken ?: "")
|
||||||
|
editor.putString("newpipe_player_potoken", this.playerRequestPoToken)
|
||||||
|
editor.putString("newpipe_visitordata", this.visitorData)
|
||||||
|
editor.apply()
|
||||||
|
}
|
||||||
|
return result.getOrNull()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param forceRecreate whether to force the recreation of [webPoTokenGenerator], to be used in
|
||||||
|
* case the current [webPoTokenGenerator] threw an error last time
|
||||||
|
* [PoTokenGenerator.generatePoToken] was called
|
||||||
|
*/
|
||||||
|
private fun getWebClientPoToken(videoId: String, forceRecreate: Boolean): PoTokenResult {
|
||||||
|
// just a helper class since Kotlin does not have builtin support for 4-tuples
|
||||||
|
data class Quadruple<T1, T2, T3, T4>(val t1: T1, val t2: T2, val t3: T3, val t4: T4)
|
||||||
|
|
||||||
|
val (poTokenGenerator, visitorData, streamingPot, hasBeenRecreated) =
|
||||||
|
synchronized(WebPoTokenGenLock) {
|
||||||
|
val shouldRecreate = webPoTokenGenerator == null || forceRecreate || webPoTokenGenerator!!.isExpired
|
||||||
|
|
||||||
|
if (shouldRecreate) {
|
||||||
|
val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient()
|
||||||
|
innertubeClientRequestInfo.clientInfo.clientVersion =
|
||||||
|
YoutubeParsingHelper.getClientVersion()
|
||||||
|
|
||||||
|
webPoTokenVisitorData = YoutubeParsingHelper.getVisitorDataFromInnertube(
|
||||||
|
innertubeClientRequestInfo,
|
||||||
|
NewPipe.getPreferredLocalization(),
|
||||||
|
NewPipe.getPreferredContentCountry(),
|
||||||
|
YoutubeParsingHelper.getYouTubeHeaders(),
|
||||||
|
YoutubeParsingHelper.YOUTUBEI_V1_URL,
|
||||||
|
null,
|
||||||
|
false
|
||||||
|
)
|
||||||
|
|
||||||
|
runBlocking {
|
||||||
|
// close the current webPoTokenGenerator on the main thread
|
||||||
|
webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } }
|
||||||
|
|
||||||
|
// create a new webPoTokenGenerator
|
||||||
|
webPoTokenGenerator = PoTokenWebView
|
||||||
|
.getNewPoTokenGenerator(App.instance)
|
||||||
|
|
||||||
|
// The streaming poToken needs to be generated exactly once before generating
|
||||||
|
// any other (player) tokens.
|
||||||
|
webPoTokenStreamingPot = webPoTokenGenerator!!.generatePoToken(webPoTokenVisitorData!!)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return@synchronized Quadruple(
|
||||||
|
webPoTokenGenerator!!,
|
||||||
|
webPoTokenVisitorData!!,
|
||||||
|
webPoTokenStreamingPot!!,
|
||||||
|
shouldRecreate
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
val playerPot = try {
|
||||||
|
// Not using synchronized here, since poTokenGenerator would be able to generate
|
||||||
|
// multiple poTokens in parallel if needed. The only important thing is for exactly one
|
||||||
|
// visitorData/streaming poToken to be generated before anything else.
|
||||||
|
runBlocking {
|
||||||
|
poTokenGenerator.generatePoToken(videoId)
|
||||||
|
}
|
||||||
|
} catch (throwable: Throwable) {
|
||||||
|
if (hasBeenRecreated) {
|
||||||
|
// the poTokenGenerator has just been recreated (and possibly this is already the
|
||||||
|
// second time we try), so there is likely nothing we can do
|
||||||
|
throw throwable
|
||||||
|
} else {
|
||||||
|
// retry, this time recreating the [webPoTokenGenerator] from scratch;
|
||||||
|
// this might happen for example if NewPipe goes in the background and the WebView
|
||||||
|
// content is lost
|
||||||
|
Log.e(TAG, "Failed to obtain poToken, retrying", throwable)
|
||||||
|
return getWebClientPoToken(videoId = videoId, forceRecreate = true)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
if (BuildConfig.DEBUG) {
|
||||||
|
Log.d(
|
||||||
|
TAG,
|
||||||
|
"poToken for $videoId: playerPot=$playerPot, " +
|
||||||
|
"streamingPot=$streamingPot, visitor_data=$visitorData"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
return PoTokenResult(visitorData, playerPot, streamingPot)
|
||||||
|
}
|
||||||
|
|
||||||
|
override fun getWebEmbedClientPoToken(videoId: String?): PoTokenResult? = null
|
||||||
|
|
||||||
|
override fun getAndroidClientPoToken(videoId: String?): PoTokenResult? = null
|
||||||
|
|
||||||
|
override fun getIosClientPoToken(videoId: String?): PoTokenResult? = null
|
||||||
|
}
|
||||||
@ -0,0 +1,13 @@
|
|||||||
|
package com.deniscerri.ytdl.util.extractors.newpipe.potoken
|
||||||
|
|
||||||
|
class PoTokenException(message: String) : Exception(message)
|
||||||
|
|
||||||
|
// to be thrown if the WebView provided by the system is broken
|
||||||
|
class BadWebViewException(message: String) : Exception(message)
|
||||||
|
|
||||||
|
fun buildExceptionForJsError(error: String): Exception {
|
||||||
|
return if (error.contains("SyntaxError"))
|
||||||
|
BadWebViewException(error)
|
||||||
|
else
|
||||||
|
PoTokenException(error)
|
||||||
|
}
|
||||||
@ -0,0 +1,353 @@
|
|||||||
|
package com.deniscerri.ytdl.util.extractors.newpipe.potoken
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.os.Build
|
||||||
|
import android.util.Log
|
||||||
|
import android.webkit.ConsoleMessage
|
||||||
|
import android.webkit.JavascriptInterface
|
||||||
|
import android.webkit.WebChromeClient
|
||||||
|
import android.webkit.WebView
|
||||||
|
import androidx.annotation.MainThread
|
||||||
|
import androidx.collection.ArrayMap
|
||||||
|
import androidx.preference.PreferenceManager
|
||||||
|
import kotlinx.coroutines.CoroutineExceptionHandler
|
||||||
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.MainScope
|
||||||
|
import kotlinx.coroutines.cancel
|
||||||
|
import kotlinx.coroutines.launch
|
||||||
|
import kotlinx.coroutines.suspendCancellableCoroutine
|
||||||
|
import kotlinx.coroutines.withContext
|
||||||
|
import com.deniscerri.ytdl.BuildConfig
|
||||||
|
import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseChallengeData
|
||||||
|
import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseIntegrityTokenData
|
||||||
|
import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.stringToU8
|
||||||
|
import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.u8ToBase64
|
||||||
|
import okhttp3.Headers.Companion.toHeaders
|
||||||
|
import okhttp3.OkHttpClient
|
||||||
|
import okhttp3.RequestBody.Companion.toRequestBody
|
||||||
|
import java.time.Instant
|
||||||
|
import java.time.temporal.ChronoUnit
|
||||||
|
import java.util.Collections
|
||||||
|
import kotlin.coroutines.Continuation
|
||||||
|
import kotlin.coroutines.resume
|
||||||
|
import kotlin.coroutines.resumeWithException
|
||||||
|
|
||||||
|
class PoTokenWebView private constructor(
|
||||||
|
context: Context,
|
||||||
|
// to be used exactly once only during initialization!
|
||||||
|
private val continuation: Continuation<PoTokenWebView>,
|
||||||
|
) {
|
||||||
|
private val webView = WebView(context)
|
||||||
|
private val scope = MainScope()
|
||||||
|
private val poTokenContinuations =
|
||||||
|
Collections.synchronizedMap(ArrayMap<String, Continuation<String>>())
|
||||||
|
private val exceptionHandler = CoroutineExceptionHandler { _, t ->
|
||||||
|
onInitializationErrorCloseAndCancel(t)
|
||||||
|
}
|
||||||
|
private lateinit var expirationInstant: Instant
|
||||||
|
|
||||||
|
//region Initialization
|
||||||
|
init {
|
||||||
|
val webViewSettings = webView.settings
|
||||||
|
//noinspection SetJavaScriptEnabled we want to use JavaScript!
|
||||||
|
webViewSettings.javaScriptEnabled = true
|
||||||
|
if (Build.VERSION.SDK_INT >= 26) {
|
||||||
|
webViewSettings.safeBrowsingEnabled = false
|
||||||
|
}
|
||||||
|
webViewSettings.userAgentString = USER_AGENT
|
||||||
|
webViewSettings.blockNetworkLoads = true // the WebView does not need internet access
|
||||||
|
|
||||||
|
// so that we can run async functions and get back the result
|
||||||
|
webView.addJavascriptInterface(this, JS_INTERFACE)
|
||||||
|
|
||||||
|
val preferences = PreferenceManager.getDefaultSharedPreferences(context)
|
||||||
|
preferences.getString("api_key", "")?.apply {
|
||||||
|
if (this.isNotBlank()) {
|
||||||
|
GOOGLE_API_KEY = this
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
webView.webChromeClient = object : WebChromeClient() {
|
||||||
|
override fun onConsoleMessage(m: ConsoleMessage): Boolean {
|
||||||
|
if (m.message().contains("Uncaught")) {
|
||||||
|
// There should not be any uncaught errors while executing the code, because
|
||||||
|
// everything that can fail is guarded by try-catch. Therefore, this likely
|
||||||
|
// indicates that there was a syntax error in the code, i.e. the WebView only
|
||||||
|
// supports a really old version of JS.
|
||||||
|
|
||||||
|
val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})"
|
||||||
|
val exception = BadWebViewException(fmt)
|
||||||
|
Log.e(TAG, "This WebView implementation is broken: $fmt")
|
||||||
|
|
||||||
|
onInitializationErrorCloseAndCancel(exception)
|
||||||
|
popAllPoTokenContinuations().forEach { (_, cont) -> cont.resumeWithException(exception) }
|
||||||
|
}
|
||||||
|
return super.onConsoleMessage(m)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Must be called right after instantiating [PoTokenWebView] to perform the actual
|
||||||
|
* initialization. This will asynchronously go through all the steps needed to load BotGuard,
|
||||||
|
* run it, and obtain an `integrityToken`.
|
||||||
|
*/
|
||||||
|
private fun loadHtmlAndObtainBotguard() {
|
||||||
|
Log.d(TAG, "loadHtmlAndObtainBotguard() called")
|
||||||
|
|
||||||
|
scope.launch(exceptionHandler) {
|
||||||
|
val html = withContext(Dispatchers.IO) {
|
||||||
|
webView.context.assets.open("po_token.html").bufferedReader().use { it.readText() }
|
||||||
|
}
|
||||||
|
|
||||||
|
// calls downloadAndRunBotguard() when the page has finished loading
|
||||||
|
val data = html.replaceFirst("</script>", "\n$JS_INTERFACE.downloadAndRunBotguard()</script>")
|
||||||
|
webView.loadDataWithBaseURL("https://www.youtube.com", data, "text/html", "utf-8", null)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called during initialization by the JavaScript snippet appended to the HTML page content in
|
||||||
|
* [loadHtmlAndObtainBotguard] after the WebView content has been loaded.
|
||||||
|
*/
|
||||||
|
@JavascriptInterface
|
||||||
|
fun downloadAndRunBotguard() {
|
||||||
|
Log.d(TAG, "downloadAndRunBotguard() called")
|
||||||
|
|
||||||
|
makeBotguardServiceRequest(
|
||||||
|
"https://www.youtube.com/api/jnn/v1/Create",
|
||||||
|
"[ \"$REQUEST_KEY\" ]",
|
||||||
|
) { responseBody ->
|
||||||
|
val parsedChallengeData = parseChallengeData(responseBody)
|
||||||
|
webView.evaluateJavascript(
|
||||||
|
"""try {
|
||||||
|
data = $parsedChallengeData
|
||||||
|
runBotGuard(data).then(function (result) {
|
||||||
|
this.webPoSignalOutput = result.webPoSignalOutput
|
||||||
|
$JS_INTERFACE.onRunBotguardResult(result.botguardResponse)
|
||||||
|
}, function (error) {
|
||||||
|
$JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
|
||||||
|
})
|
||||||
|
} catch (error) {
|
||||||
|
$JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack)
|
||||||
|
}""",
|
||||||
|
null
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called during initialization by the JavaScript snippets from either
|
||||||
|
* [downloadAndRunBotguard] or [onRunBotguardResult].
|
||||||
|
*/
|
||||||
|
@JavascriptInterface
|
||||||
|
fun onJsInitializationError(error: String) {
|
||||||
|
if (BuildConfig.DEBUG) {
|
||||||
|
Log.e(TAG, "Initialization error from JavaScript: $error")
|
||||||
|
}
|
||||||
|
onInitializationErrorCloseAndCancel(buildExceptionForJsError(error))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called during initialization by the JavaScript snippet from [downloadAndRunBotguard] after
|
||||||
|
* obtaining the BotGuard execution output [botguardResponse].
|
||||||
|
*/
|
||||||
|
@JavascriptInterface
|
||||||
|
fun onRunBotguardResult(botguardResponse: String) {
|
||||||
|
Log.d(TAG, "botguardResponse: $botguardResponse")
|
||||||
|
makeBotguardServiceRequest(
|
||||||
|
"https://www.youtube.com/api/jnn/v1/GenerateIT",
|
||||||
|
"[ \"$REQUEST_KEY\", \"$botguardResponse\" ]",
|
||||||
|
) { responseBody ->
|
||||||
|
Log.d(TAG, "GenerateIT response: $responseBody")
|
||||||
|
val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody)
|
||||||
|
|
||||||
|
// leave 10 minutes of margin just to be sure
|
||||||
|
expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds).minus(10, ChronoUnit.MINUTES)
|
||||||
|
|
||||||
|
webView.evaluateJavascript("this.integrityToken = $integrityToken") {
|
||||||
|
Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s")
|
||||||
|
continuation.resume(this)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
//endregion
|
||||||
|
|
||||||
|
//region Obtaining poTokens
|
||||||
|
suspend fun generatePoToken(identifier: String): String {
|
||||||
|
return withContext(Dispatchers.Main) {
|
||||||
|
suspendCancellableCoroutine { cont ->
|
||||||
|
Log.d(TAG, "generatePoToken() called with identifier $identifier")
|
||||||
|
addPoTokenEmitter(identifier, cont)
|
||||||
|
webView.evaluateJavascript(
|
||||||
|
"""try {
|
||||||
|
identifier = "$identifier"
|
||||||
|
u8Identifier = ${stringToU8(identifier)}
|
||||||
|
poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier)
|
||||||
|
poTokenU8String = poTokenU8.join(",")
|
||||||
|
$JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String)
|
||||||
|
} catch (error) {
|
||||||
|
$JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack)
|
||||||
|
}""",
|
||||||
|
null
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called by the JavaScript snippet from [generatePoToken] when an error occurs in calling the
|
||||||
|
* JavaScript `obtainPoToken()` function.
|
||||||
|
*/
|
||||||
|
@JavascriptInterface
|
||||||
|
fun onObtainPoTokenError(identifier: String, error: String) {
|
||||||
|
if (BuildConfig.DEBUG) {
|
||||||
|
Log.e(TAG, "obtainPoToken error from JavaScript: $error")
|
||||||
|
}
|
||||||
|
popPoTokenContinuation(identifier)?.resumeWithException(buildExceptionForJsError(error))
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called by the JavaScript snippet from [generatePoToken] with the original identifier and the
|
||||||
|
* result of the JavaScript `obtainPoToken()` function.
|
||||||
|
*/
|
||||||
|
@JavascriptInterface
|
||||||
|
fun onObtainPoTokenResult(identifier: String, poTokenU8: String) {
|
||||||
|
Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8")
|
||||||
|
val poToken = try {
|
||||||
|
u8ToBase64(poTokenU8)
|
||||||
|
} catch (t: Throwable) {
|
||||||
|
popPoTokenContinuation(identifier)?.resumeWithException(t)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken")
|
||||||
|
popPoTokenContinuation(identifier)?.resume(poToken)
|
||||||
|
}
|
||||||
|
|
||||||
|
val isExpired: Boolean
|
||||||
|
get() = Instant.now().isAfter(expirationInstant)
|
||||||
|
//endregion
|
||||||
|
|
||||||
|
//region Handling multiple emitters
|
||||||
|
/**
|
||||||
|
* Adds the ([identifier], [continuation]) pair to the [poTokenContinuations] list. This makes
|
||||||
|
* it so that multiple poToken requests can be generated in parallel, and the results will be
|
||||||
|
* notified to the right continuations.
|
||||||
|
*/
|
||||||
|
private fun addPoTokenEmitter(identifier: String, continuation: Continuation<String>) {
|
||||||
|
poTokenContinuations[identifier] = continuation
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extracts and removes from the [poTokenContinuations] list a [Continuation] based on its
|
||||||
|
* [identifier]. The continuation is supposed to be used immediately after to either signal a
|
||||||
|
* success or an error.
|
||||||
|
*/
|
||||||
|
private fun popPoTokenContinuation(identifier: String): Continuation<String>? {
|
||||||
|
return poTokenContinuations.remove(identifier)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Clears [poTokenContinuations] and returns its previous contents. The continuations are supposed
|
||||||
|
* to be used immediately after to either signal a success or an error.
|
||||||
|
*/
|
||||||
|
private fun popAllPoTokenContinuations(): Map<String, Continuation<String>> {
|
||||||
|
val result = poTokenContinuations.toMap()
|
||||||
|
poTokenContinuations.clear()
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
//endregion
|
||||||
|
|
||||||
|
//region Utils
|
||||||
|
/**
|
||||||
|
* Makes a POST request to [url] with the given [data] by setting the correct headers. Calls
|
||||||
|
* [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response
|
||||||
|
* does not have HTTP code 200, therefore this is supposed to be used only during
|
||||||
|
* initialization. Calls [handleResponseBody] with the response body if the response is
|
||||||
|
* successful. The request is performed in the background and a disposable is added to
|
||||||
|
* [disposables].
|
||||||
|
*/
|
||||||
|
private fun makeBotguardServiceRequest(
|
||||||
|
url: String,
|
||||||
|
data: String,
|
||||||
|
handleResponseBody: (String) -> Unit,
|
||||||
|
) {
|
||||||
|
scope.launch(exceptionHandler) {
|
||||||
|
val requestBuilder = okhttp3.Request.Builder()
|
||||||
|
.post(data.toRequestBody())
|
||||||
|
.headers(mapOf(
|
||||||
|
"User-Agent" to USER_AGENT,
|
||||||
|
"Accept" to "application/json",
|
||||||
|
"Content-Type" to "application/json+protobuf",
|
||||||
|
"x-goog-api-key" to GOOGLE_API_KEY,
|
||||||
|
"x-user-agent" to "grpc-web-javascript/0.1",
|
||||||
|
).toHeaders())
|
||||||
|
.url(url)
|
||||||
|
val response = withContext(Dispatchers.IO) {
|
||||||
|
httpClient.newCall(requestBuilder.build()).execute()
|
||||||
|
}
|
||||||
|
val httpCode = response.code
|
||||||
|
if (httpCode != 200) {
|
||||||
|
onInitializationErrorCloseAndCancel(PoTokenException("Invalid response code: $httpCode"))
|
||||||
|
} else {
|
||||||
|
val body = withContext(Dispatchers.IO) {
|
||||||
|
response.body!!.string()
|
||||||
|
}
|
||||||
|
handleResponseBody(body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handles any error happening during initialization, releasing resources and sending the error
|
||||||
|
* to [continuation].
|
||||||
|
*/
|
||||||
|
private fun onInitializationErrorCloseAndCancel(error: Throwable) {
|
||||||
|
close()
|
||||||
|
continuation.resumeWithException(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Releases all [webView] resources.
|
||||||
|
*/
|
||||||
|
@MainThread
|
||||||
|
fun close() {
|
||||||
|
scope.cancel()
|
||||||
|
|
||||||
|
webView.clearHistory()
|
||||||
|
// clears RAM cache and disk cache (globally for all WebViews)
|
||||||
|
webView.clearCache(true)
|
||||||
|
|
||||||
|
// ensures that the WebView isn't doing anything when destroying it
|
||||||
|
webView.loadUrl("about:blank")
|
||||||
|
|
||||||
|
webView.onPause()
|
||||||
|
webView.removeAllViews()
|
||||||
|
webView.destroy()
|
||||||
|
}
|
||||||
|
//endregion
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
private const val TAG = "PoTokenWebView"
|
||||||
|
//libretube api key, if user has his own api key his will be used
|
||||||
|
private var GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw"
|
||||||
|
private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo"
|
||||||
|
private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3"
|
||||||
|
private const val JS_INTERFACE = "PoTokenWebView"
|
||||||
|
|
||||||
|
private val httpClient = OkHttpClient.Builder()
|
||||||
|
//.proxy(YouTube.proxy)
|
||||||
|
.build()
|
||||||
|
|
||||||
|
suspend fun getNewPoTokenGenerator(context: Context): PoTokenWebView {
|
||||||
|
return withContext(Dispatchers.Main) {
|
||||||
|
suspendCancellableCoroutine { cont ->
|
||||||
|
val potWv = PoTokenWebView(context, cont)
|
||||||
|
potWv.loadHtmlAndObtainBotguard()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<com.google.android.material.floatingactionbutton.FloatingActionButton android:layout_width="wrap_content"
|
||||||
|
style="@style/Widget.MaterialComponents.Button.UnelevatedButton"
|
||||||
|
android:layout_height="wrap_content"
|
||||||
|
xmlns:app="http://schemas.android.com/apk/res-auto"
|
||||||
|
android:contentDescription="@string/more"
|
||||||
|
android:src="@drawable/ic_app_icon"
|
||||||
|
app:maxImageSize="70dp"
|
||||||
|
app:tint="@color/icon_fg"
|
||||||
|
android:backgroundTint="#00000000"
|
||||||
|
app:borderWidth="0dp"
|
||||||
|
xmlns:android="http://schemas.android.com/apk/res/android" />
|
||||||
@ -1,5 +1,4 @@
|
|||||||
<?xml version="1.0" encoding="utf-8"?>
|
|
||||||
<resources>
|
<resources>
|
||||||
<integer name="grid_size">3</integer>
|
<integer name="grid_size">2</integer>
|
||||||
<bool name="uses_side_nav">true</bool>
|
<bool name="isTablet">true</bool>
|
||||||
</resources>
|
</resources>
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
|
<resources>
|
||||||
|
<integer name="grid_size">3</integer>
|
||||||
|
<bool name="uses_side_nav">true</bool>
|
||||||
|
</resources>
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
# What's Changed
|
||||||
|
|
||||||
|
- Added ability to automatically generate Po Tokens and Visitor Data through NewPipe Extractor
|
||||||
|
- Fixed auto resume not working #727
|
||||||
|
- Applied "Prefer Container over Codec" preference on default format sorting
|
||||||
|
- Added automatic Backup
|
||||||
|
- Added milliseconds in the cut section
|
||||||
|
and more
|
||||||
Loading…
Reference in New Issue