diff --git a/CHANGELOG.md b/CHANGELOG.md index b8120909..1dc11dc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,30 @@ # YTDLnis Changelog +> # 1.8.3 (2025-03) + +# What's Changed + +## Automatically generate PO Tokens & Visitor Data with NewPipe Extractor + +Thanks to NewPipe Extractor for allowing the use of Po Token Generators, and some references from LibreTube, the app now has a po token generator using webview. Now all NewPipe actions like data fetching and format fetching will work. +Also at the same time these records are stored in the apps preferences. In advanced settings now there is a toggle to enable the use of the tokens in yt-dlp. +These tokens need visitor data to work and since visitor data and cookies cant be used at the same time, cookies will be disabled if you enable this function. +With it, the app will apply these settings as youtube extractor arguments +youtube:player_client=web;po_token=web.gvs+GVS_TOKEN,web.player+PLAYER_TOKEN;player-skip=webpage,configs;visitor_data=VISITOR_DATA +If you have po tokens set up with web client with po token, there might be duplication happening, so check that out. + + +Other stuff +- Cut Section has been reworked,thanks to madmini. Now you can cut down to milliseconds. +- Added safeBrowsingEnabled in WebView for generating cookies for devices of API 26 and above. Thought to generate incognito cookies that last longer +- Added feature to reset all settings belonging to a certain preference page +- Added option to turn off the code color highlighter +- Fixed app not applying prefer small formats. It shouldve been last in format sorting not first. +- Added ability to enable automatic backup when the app checks for new update and finds one +- Added write-subs and write-auto-subs and --compat-options no-keep-subs when the user embeds subs but doesnt want to save them so he can get more subtitles to embed +- Turned the changelog dialog to a separate screen for better visibility +- Some small bug fix here and there + > # 1.8.2.2 (2025-02) # What's Changed diff --git a/app/build.gradle b/app/build.gradle index 83123b46..bbe6227e 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -11,8 +11,8 @@ plugins { def properties = new Properties() def versionMajor = 1 def versionMinor = 8 -def versionPatch = 2 -def versionBuild = 2 // bump for dogfood builds, public betas, etc. +def versionPatch = 3 +def versionBuild = 0 // bump for dogfood builds, public betas, etc. def isBeta = false def versionExt = "" @@ -215,5 +215,5 @@ dependencies { implementation 'androidx.compose.material3:material3-android:1.3.1' implementation "io.noties.markwon:core:4.6.2" implementation("org.greenrobot:eventbus:3.3.1") - implementation("com.github.teamnewpipe:newpipeextractor:0.24.4") + implementation("com.github.teamnewpipe:newpipeextractor:0.24.5") } diff --git a/app/src/main/assets/po_token.html b/app/src/main/assets/po_token.html new file mode 100644 index 00000000..0dc4af28 --- /dev/null +++ b/app/src/main/assets/po_token.html @@ -0,0 +1,127 @@ + + \ No newline at end of file diff --git a/app/src/main/java/com/deniscerri/ytdl/database/models/YoutubePlayerClientItem.kt b/app/src/main/java/com/deniscerri/ytdl/database/models/YoutubePlayerClientItem.kt index a599e3cb..8e5d089b 100644 --- a/app/src/main/java/com/deniscerri/ytdl/database/models/YoutubePlayerClientItem.kt +++ b/app/src/main/java/com/deniscerri/ytdl/database/models/YoutubePlayerClientItem.kt @@ -5,8 +5,9 @@ data class YoutubePlayerClientItem( var poTokens: MutableList, var enabled: Boolean = true, var useOnlyPoToken: Boolean = false, - var urlRegex: MutableList = mutableListOf() - ) + var urlRegex: MutableList = mutableListOf(), + var autoGenerated: Boolean = false +) data class YoutubePoTokenItem( var context: String, diff --git a/app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt b/app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt index ea71aa0f..46d582f6 100644 --- a/app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt +++ b/app/src/main/java/com/deniscerri/ytdl/ui/adapter/YoutubePlayerClientAdapter.kt @@ -77,7 +77,7 @@ class YoutubePlayerClientAdapter(onItemClickListener: OnItemClickListener, activ } for (chip in chips.reversed()) { - content.addView(chip, 0) + content.addView(chip) } if (item.urlRegex.isNotEmpty()) { @@ -91,6 +91,8 @@ class YoutubePlayerClientAdapter(onItemClickListener: OnItemClickListener, activ title.alpha = if (item.enabled) 1f else 0.3f content.alpha = if (item.enabled) 1f else 0.3f + card.findViewById(R.id.autoGeneratedNewPipe).isVisible = item.autoGenerated + val switch = card.findViewById(R.id.materialSwitch) switch.isChecked = item.enabled switch.setOnClickListener { diff --git a/app/src/main/java/com/deniscerri/ytdl/ui/more/WebViewActivity.kt b/app/src/main/java/com/deniscerri/ytdl/ui/more/WebViewActivity.kt index 4fda23ec..088fef8f 100644 --- a/app/src/main/java/com/deniscerri/ytdl/ui/more/WebViewActivity.kt +++ b/app/src/main/java/com/deniscerri/ytdl/ui/more/WebViewActivity.kt @@ -2,6 +2,7 @@ package com.deniscerri.ytdl.ui.more import android.annotation.SuppressLint import android.content.SharedPreferences +import android.os.Build import android.os.Bundle import android.webkit.CookieManager import android.webkit.WebView @@ -132,6 +133,9 @@ class WebViewActivity : BaseActivity() { javaScriptEnabled = true domStorageEnabled = true javaScriptCanOpenWindowsAutomatically = true + if (Build.VERSION.SDK_INT >= 26) { + safeBrowsingEnabled = true + } preferences.edit().putString("useragent_header", userAgentString).apply() } cookieManager.setAcceptThirdPartyCookies(this, true) diff --git a/app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/AdvancedSettingsFragment.kt b/app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/AdvancedSettingsFragment.kt index 3d0fb385..75d23c8b 100644 --- a/app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/AdvancedSettingsFragment.kt +++ b/app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/AdvancedSettingsFragment.kt @@ -6,9 +6,11 @@ import android.graphics.Typeface import android.os.Bundle import android.widget.LinearLayout import android.widget.TextView +import androidx.lifecycle.lifecycleScope import androidx.navigation.fragment.findNavController import androidx.preference.Preference import androidx.preference.PreferenceManager +import androidx.preference.SwitchPreferenceCompat import androidx.recyclerview.widget.ItemTouchHelper import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView @@ -17,7 +19,11 @@ import com.deniscerri.ytdl.R import com.deniscerri.ytdl.ui.adapter.SortableTextItemAdapter import com.deniscerri.ytdl.ui.more.settings.BaseSettingsFragment import com.deniscerri.ytdl.util.UiUtil +import com.deniscerri.ytdl.util.extractors.newpipe.NewPipeUtil import com.google.android.material.dialog.MaterialAlertDialogBuilder +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext class AdvancedSettingsFragment : BaseSettingsFragment() { @@ -33,6 +39,53 @@ class AdvancedSettingsFragment : BaseSettingsFragment() { false } + val newPipeUtil = NewPipeUtil(requireContext()) + + findPreference("use_newpipe_potoken")?.apply { + fun getValues() : Triple { + val gvs = prefs.getString("newpipe_gvs_potoken", "")!! + val player = prefs.getString("newpipe_player_potoken", "")!! + val visitorData = prefs.getString("newpipe_visitordata", "")!! + + return Triple(gvs, player, visitorData) + } + + fun updateSummary() : String { + val values = getValues() + val gvs = values.first + val player = values.second + val visitorData = values.third + + return if (visitorData.isNotBlank()) { + "PO Token (GVS): ${gvs.take(10)}...\nPO Token (Player): ${player.take(10)}...\nVisitor Data: ${visitorData.take(10)}..." + }else { + "" + } + } + + summary = updateSummary() + + setOnPreferenceClickListener { + if (this.isChecked) { + this.isChecked = false + UiUtil.showGenericConfirmDialog(requireContext(), getString(R.string.use_newpipe_potoken), getString(R.string.use_newpipe_potoken_warning)) { + editor.putBoolean("use_cookies", false) + editor.putBoolean("use_newpipe_token", true) + editor.apply() + this.isChecked = true + lifecycleScope.launch { + summary = getString(R.string.loading) + withContext(Dispatchers.IO) { + newPipeUtil.testRun() + } + summary = updateSummary() + } + } + } + false + } + } + val formatImportanceAudio: Preference? = findPreference("format_importance_audio") val formatImportanceVideo: Preference? = findPreference("format_importance_video") diff --git a/app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/YoutubePlayerClientFragment.kt b/app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/YoutubePlayerClientFragment.kt index 4a00aeb5..ef8339f5 100644 --- a/app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/YoutubePlayerClientFragment.kt +++ b/app/src/main/java/com/deniscerri/ytdl/ui/more/settings/advanced/YoutubePlayerClientFragment.kt @@ -2,27 +2,44 @@ package com.deniscerri.ytdl.ui.more.settings.advanced import android.animation.AnimatorSet import android.annotation.SuppressLint +import android.app.Activity +import android.content.ClipboardManager import android.content.SharedPreferences import android.os.Bundle +import android.text.Editable +import android.text.TextWatcher import android.util.Log import android.view.LayoutInflater import android.view.View import android.view.ViewGroup +import android.view.Window import android.view.animation.AccelerateDecelerateInterpolator +import android.widget.Button +import android.widget.LinearLayout import android.widget.RelativeLayout +import androidx.appcompat.app.AppCompatActivity +import androidx.core.view.children import androidx.core.view.isVisible +import androidx.core.widget.doOnTextChanged import androidx.fragment.app.Fragment import androidx.preference.PreferenceManager import androidx.recyclerview.widget.ItemTouchHelper import androidx.recyclerview.widget.LinearLayoutManager import androidx.recyclerview.widget.RecyclerView +import com.afollestad.materialdialogs.utils.MDUtil.getStringArray import com.deniscerri.ytdl.R import com.deniscerri.ytdl.database.models.YoutubePlayerClientItem +import com.deniscerri.ytdl.database.models.YoutubePoTokenItem import com.deniscerri.ytdl.ui.adapter.YoutubePlayerClientAdapter import com.deniscerri.ytdl.ui.more.settings.SettingsActivity import com.deniscerri.ytdl.util.UiUtil +import com.google.android.material.bottomsheet.BottomSheetBehavior +import com.google.android.material.bottomsheet.BottomSheetDialog import com.google.android.material.card.MaterialCardView import com.google.android.material.chip.Chip +import com.google.android.material.chip.ChipGroup +import com.google.android.material.materialswitch.MaterialSwitch +import com.google.android.material.textfield.TextInputLayout import com.google.gson.Gson @@ -64,8 +81,8 @@ class YoutubePlayerClientFragment : Fragment(), YoutubePlayerClientAdapter.OnIte val newClient = view.findViewById(R.id.newClient) newClient.setOnClickListener { - UiUtil.showYoutubePlayerClientSheet( - settingsActivity, preferences, null, + showYoutubePlayerClientSheet( + null, newValue = { newItem -> currentList.add(newItem) currentListRaw = Gson().toJson(currentList).toString() @@ -97,9 +114,147 @@ class YoutubePlayerClientFragment : Fragment(), YoutubePlayerClientAdapter.OnIte } + private fun showYoutubePlayerClientSheet(currentValue: YoutubePlayerClientItem?, newValue: (item: YoutubePlayerClientItem) -> Unit, deleted: () -> Unit){ + val bottomSheet = BottomSheetDialog(requireContext()) + bottomSheet.requestWindowFeature(Window.FEATURE_NO_TITLE) + bottomSheet.setContentView(R.layout.youtube_player_client_create_bottom_sheet) + + val title : TextInputLayout = bottomSheet.findViewById(R.id.title)!! + val chipGroup : ChipGroup = bottomSheet.findViewById(R.id.chipGroup)!! + val suggestedLabel : View = bottomSheet.findViewById(R.id.suggestedLabel)!! + val okBtn : Button = bottomSheet.findViewById(R.id.client_create)!! + val deleteBtn : Button = bottomSheet.findViewById(R.id.client_delete)!! + deleteBtn.isVisible = currentValue != null + + val useOnlyPOToken : MaterialSwitch = bottomSheet.findViewById(R.id.use_only_po_token)!! + useOnlyPOToken.isChecked = currentValue?.useOnlyPoToken ?: false + + val contentLinear : LinearLayout = bottomSheet.findViewById(R.id.contentLinear)!! + + val defaultChips = requireContext().getStringArray(R.array.youtube_player_clients).toMutableSet() + + title.isEndIconVisible = false + title.editText!!.addTextChangedListener(object : TextWatcher { + override fun beforeTextChanged(p0: CharSequence?, p1: Int, p2: Int, p3: Int) {} + override fun onTextChanged(p0: CharSequence?, p1: Int, p2: Int, p3: Int) {} + override fun afterTextChanged(p0: Editable?) { + chipGroup.children.forEach { (it as Chip).isChecked = false } + chipGroup.children.firstOrNull { (it as Chip).text == p0.toString() }?.apply { + (this as Chip).isChecked = true + } + } + }) + + val existingConfigsRaw = preferences.getString("youtube_player_clients", "[]") + val existingConfigs = Gson().fromJson(existingConfigsRaw, Array::class.java).toMutableList() + + defaultChips.filter { it.isNotBlank() }.forEach { + if (!existingConfigs.any { it2 -> it2.playerClient == it }) { + val tmp = requireActivity().layoutInflater.inflate(R.layout.filter_chip, chipGroup, false) as Chip + tmp.text = it + tmp.setOnClickListener { + title.editText!!.setText(tmp.text.toString()) + } + chipGroup.addView(tmp) + } + } + + if (chipGroup.children.count() == 0) { + suggestedLabel.isVisible = false + } + + val poTokenInputs = contentLinear.children.filter { it is TextInputLayout }.map { it as TextInputLayout }.toList() + for(p in poTokenInputs) { + p.setEndIconOnClickListener { + val clipboard: ClipboardManager = requireActivity().getSystemService(AppCompatActivity.CLIPBOARD_SERVICE) as ClipboardManager + p.editText!!.setText(clipboard.primaryClip?.getItemAt(0)?.text) + } + } + + currentValue?.apply { + title.editText!!.setText(this.playerClient) + for (t in this.poTokens) { + poTokenInputs.firstOrNull { it.tag == "potoken_${t.context}" }?.editText?.setText(t.token) + } + okBtn.text = requireContext().getString(R.string.edit) + } + + val urlRegexInput = contentLinear.findViewById(R.id.url_regex) + urlRegexInput.isEndIconVisible = false + urlRegexInput.editText!!.doOnTextChanged { text, start, before, count -> + urlRegexInput.isEndIconVisible = urlRegexInput.editText!!.text.isNotBlank() + } + + val urlRegexChips = contentLinear.findViewById(R.id.urlRegexChipGroup) + currentValue?.apply { + for(chip in this.urlRegex) { + val tmp = requireActivity().layoutInflater.inflate(R.layout.input_chip, urlRegexChips, false) as Chip + tmp.text = chip + tmp.setOnClickListener { + urlRegexChips.removeView(tmp) + } + urlRegexChips.addView(tmp) + } + } + + urlRegexInput.setEndIconOnClickListener { + val text = urlRegexInput.editText!!.text + urlRegexInput.editText!!.setText("") + val tmp = requireActivity().layoutInflater.inflate(R.layout.input_chip, urlRegexChips, false) as Chip + tmp.text = text + tmp.setOnClickListener { + urlRegexChips.removeView(tmp) + } + urlRegexChips.addView(tmp) + } + + okBtn.setOnClickListener { + val titleVal = title.editText!!.text.toString() + if (titleVal.isBlank()) { + title.error = "Player Client tag shouldn't be empty" + return@setOnClickListener + } + + if(existingConfigs.any { it2 -> it2.playerClient == titleVal && it2.enabled } && ( currentValue == null || currentValue.playerClient != titleVal )) { + title.error = "Player Client is already created" + return@setOnClickListener + } + + if (useOnlyPOToken.isChecked && (poTokenInputs.filter { it.tag.toString().startsWith("potoken") }.all { it.editText!!.text.isBlank() })) { + poTokenInputs.first().error = "You need to write at least one PO Token" + return@setOnClickListener + } + + val obj = YoutubePlayerClientItem(titleVal, mutableListOf(), true, useOnlyPOToken.isChecked) + poTokenInputs.filter { it.editText!!.text.isNotBlank() && it.tag.toString().startsWith("potoken") }.forEach { + obj.poTokens.add(YoutubePoTokenItem(it.tag.toString().split("potoken_")[1], it.editText!!.text.toString())) + } + + val urlRegexes = urlRegexChips.children.map { (it as Chip).text.toString() } + obj.urlRegex.addAll(urlRegexes) + + bottomSheet.cancel() + newValue(obj) + } + + deleteBtn.setOnClickListener { + bottomSheet.cancel() + deleted() + } + + + bottomSheet.show() + bottomSheet.behavior.state = BottomSheetBehavior.STATE_EXPANDED + bottomSheet.window!!.setLayout( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT + ) + } + + override fun onItemClick(item: YoutubePlayerClientItem, index: Int) { - UiUtil.showYoutubePlayerClientSheet( - settingsActivity, preferences, item, + showYoutubePlayerClientSheet( + item, newValue = { newItem -> currentList.remove(item) currentList.add(newItem) diff --git a/app/src/main/java/com/deniscerri/ytdl/util/UiUtil.kt b/app/src/main/java/com/deniscerri/ytdl/util/UiUtil.kt index d6ca278e..8947f311 100644 --- a/app/src/main/java/com/deniscerri/ytdl/util/UiUtil.kt +++ b/app/src/main/java/com/deniscerri/ytdl/util/UiUtil.kt @@ -2367,143 +2367,6 @@ object UiUtil { ) } - fun showYoutubePlayerClientSheet(context: Activity, preferences: SharedPreferences, currentValue: YoutubePlayerClientItem?, newValue: (item: YoutubePlayerClientItem) -> Unit, deleted: () -> Unit){ - val bottomSheet = BottomSheetDialog(context) - bottomSheet.requestWindowFeature(Window.FEATURE_NO_TITLE) - bottomSheet.setContentView(R.layout.youtube_player_client_create_bottom_sheet) - - val title : TextInputLayout = bottomSheet.findViewById(R.id.title)!! - val chipGroup : ChipGroup = bottomSheet.findViewById(R.id.chipGroup)!! - val suggestedLabel : View = bottomSheet.findViewById(R.id.suggestedLabel)!! - val okBtn : Button = bottomSheet.findViewById(R.id.client_create)!! - val deleteBtn : Button = bottomSheet.findViewById(R.id.client_delete)!! - deleteBtn.isVisible = currentValue != null - - val useOnlyPOToken : MaterialSwitch = bottomSheet.findViewById(R.id.use_only_po_token)!! - useOnlyPOToken.isChecked = currentValue?.useOnlyPoToken ?: false - - val contentLinear : LinearLayout = bottomSheet.findViewById(R.id.contentLinear)!! - - val defaultChips = context.getStringArray(R.array.youtube_player_clients).toMutableSet() - - title.isEndIconVisible = false - title.editText!!.addTextChangedListener(object : TextWatcher { - override fun beforeTextChanged(p0: CharSequence?, p1: Int, p2: Int, p3: Int) {} - override fun onTextChanged(p0: CharSequence?, p1: Int, p2: Int, p3: Int) {} - override fun afterTextChanged(p0: Editable?) { - chipGroup.children.forEach { (it as Chip).isChecked = false } - chipGroup.children.firstOrNull { (it as Chip).text == p0.toString() }?.apply { - (this as Chip).isChecked = true - } - } - }) - - val existingConfigsRaw = preferences.getString("youtube_player_clients", "[]") - val existingConfigs = Gson().fromJson(existingConfigsRaw, Array::class.java).toMutableList() - - defaultChips.filter { it.isNotBlank() }.forEach { - if (!existingConfigs.any { it2 -> it2.playerClient == it }) { - val tmp = context.layoutInflater.inflate(R.layout.filter_chip, chipGroup, false) as Chip - tmp.text = it - tmp.setOnClickListener { - title.editText!!.setText(tmp.text.toString()) - } - chipGroup.addView(tmp) - } - } - - if (chipGroup.children.count() == 0) { - suggestedLabel.isVisible = false - } - - val poTokenInputs = contentLinear.children.filter { it is TextInputLayout }.map { it as TextInputLayout }.toList() - for(p in poTokenInputs) { - p.setEndIconOnClickListener { - val clipboard: ClipboardManager = context.getSystemService(AppCompatActivity.CLIPBOARD_SERVICE) as ClipboardManager - p.editText!!.setText(clipboard.primaryClip?.getItemAt(0)?.text) - } - } - - currentValue?.apply { - title.editText!!.setText(this.playerClient) - for (t in this.poTokens) { - poTokenInputs.firstOrNull { it.tag == "potoken_${t.context}" }?.editText?.setText(t.token) - } - okBtn.text = context.getString(R.string.edit) - } - - val urlRegexInput = contentLinear.findViewById(R.id.url_regex) - urlRegexInput.isEndIconVisible = false - urlRegexInput.editText!!.doOnTextChanged { text, start, before, count -> - urlRegexInput.isEndIconVisible = urlRegexInput.editText!!.text.isNotBlank() - } - - val urlRegexChips = contentLinear.findViewById(R.id.urlRegexChipGroup) - currentValue?.apply { - for(chip in this.urlRegex) { - val tmp = context.layoutInflater.inflate(R.layout.input_chip, urlRegexChips, false) as Chip - tmp.text = chip - tmp.setOnClickListener { - urlRegexChips.removeView(tmp) - } - urlRegexChips.addView(tmp) - } - } - - urlRegexInput.setEndIconOnClickListener { - val text = urlRegexInput.editText!!.text - urlRegexInput.editText!!.setText("") - val tmp = context.layoutInflater.inflate(R.layout.input_chip, urlRegexChips, false) as Chip - tmp.text = text - tmp.setOnClickListener { - urlRegexChips.removeView(tmp) - } - urlRegexChips.addView(tmp) - } - - okBtn.setOnClickListener { - val titleVal = title.editText!!.text.toString() - if (titleVal.isBlank()) { - title.error = "Player Client tag shouldn't be empty" - return@setOnClickListener - } - - if(existingConfigs.any { it2 -> it2.playerClient == titleVal } && ( currentValue == null || currentValue.playerClient != titleVal )) { - title.error = "Player Client is already created" - return@setOnClickListener - } - - if (useOnlyPOToken.isChecked && (poTokenInputs.filter { it.tag.toString().startsWith("potoken") }.all { it.editText!!.text.isBlank() })) { - poTokenInputs.first().error = "You need to write at least one PO Token" - return@setOnClickListener - } - - val obj = YoutubePlayerClientItem(titleVal, mutableListOf(), true, useOnlyPOToken.isChecked) - poTokenInputs.filter { it.editText!!.text.isNotBlank() && it.tag.toString().startsWith("potoken") }.forEach { - obj.poTokens.add(YoutubePoTokenItem(it.tag.toString().split("potoken_")[1], it.editText!!.text.toString())) - } - - val urlRegexes = urlRegexChips.children.map { (it as Chip).text.toString() } - obj.urlRegex.addAll(urlRegexes) - - bottomSheet.cancel() - newValue(obj) - } - - deleteBtn.setOnClickListener { - bottomSheet.cancel() - deleted() - } - - - bottomSheet.show() - bottomSheet.behavior.state = BottomSheetBehavior.STATE_EXPANDED - bottomSheet.window!!.setLayout( - ViewGroup.LayoutParams.MATCH_PARENT, - ViewGroup.LayoutParams.MATCH_PARENT - ) - } - @SuppressLint("UnspecifiedRegisterReceiverFlag") fun showNewAppUpdateDialog(v: GithubRelease, context: Activity, preferences: SharedPreferences) { val downloadManager = context.getSystemService(Context.DOWNLOAD_SERVICE) as DownloadManager diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt index 57e4c8e2..a116a5f3 100644 --- a/app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/YTDLPUtil.kt @@ -1271,7 +1271,7 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co private fun YoutubeDLRequest.setYoutubeExtractorArgs(url: String?) { val extractorArgs = mutableListOf() - val playerClients = mutableListOf() + val playerClients = mutableSetOf() val poTokens = mutableListOf() val configuredPlayerClientsRaw = sharedPreferences.getString("youtube_player_clients", "[]")!! @@ -1296,6 +1296,21 @@ class YTDLPUtil(private val context: Context, private val commandTemplateDao: Co } } + if (sharedPreferences.getBoolean("use_newpipe_potoken", false)) { + val visitorData = sharedPreferences.getString("newpipe_visitordata", "")!! + if (visitorData.isNotBlank()) { + playerClients.add("web") + sharedPreferences.getString("newpipe_gvs_potoken", "")?.apply { + if (this.isNotBlank()) poTokens.add("web.gvs+$this") + } + sharedPreferences.getString("newpipe_player_potoken", "")?.apply { + if (this.isNotBlank()) poTokens.add("web.player+$this") + } + extractorArgs.add("player-skip=webpage,configs") + extractorArgs.add("visitor_data=$visitorData") + } + } + if (playerClients.isNotEmpty()){ extractorArgs.add("player_client=${playerClients.joinToString(",")}") } diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/NewPipeUtil.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/NewPipeUtil.kt index d09dbf7f..69fea561 100644 --- a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/NewPipeUtil.kt +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/NewPipeUtil.kt @@ -9,6 +9,7 @@ import com.deniscerri.ytdl.database.models.Format import com.deniscerri.ytdl.database.models.ResultItem import com.deniscerri.ytdl.database.viewmodel.ResultViewModel import com.deniscerri.ytdl.util.Extensions.toStringDuration +import com.deniscerri.ytdl.util.extractors.newpipe.potoken.NewPipePoTokenGenerator import com.google.gson.Gson import okhttp3.OkHttpClient import org.json.JSONException @@ -22,6 +23,8 @@ import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler import org.schabi.newpipe.extractor.localization.Localization import org.schabi.newpipe.extractor.playlist.PlaylistInfo import org.schabi.newpipe.extractor.search.SearchInfo +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult +import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory import org.schabi.newpipe.extractor.stream.StreamInfo import org.schabi.newpipe.extractor.stream.StreamInfoItem @@ -32,13 +35,21 @@ class NewPipeUtil(context: Context) { private var sharedPreferences: SharedPreferences = PreferenceManager.getDefaultSharedPreferences(context) private val countryCode = sharedPreferences.getString("locale", "")!!.ifEmpty { "US" } private val language = sharedPreferences.getString("app_language", "")!!.ifEmpty { "en" } + private val testURL = "https://www.youtube.com/watch?v=aqz-KE-bpKQ" //bbb + init { NewPipe.init(NewPipeDownloaderImpl(OkHttpClient.Builder()), Localization(language, countryCode)) + YoutubeStreamExtractor.setPoTokenProvider(NewPipePoTokenGenerator()) + } + + fun testRun() { + getVideoData(testURL) + return } fun getVideoData(url : String) : Result> { try { - val streamInfo = StreamInfo.getInfo(NewPipe.getService(ServiceList.YouTube.serviceId), url) + val streamInfo = StreamInfo.getInfo(url) val vid = createVideoFromStream(streamInfo, url) ?: return Result.failure(Throwable()) return Result.success(listOf(vid)) }catch (e: Exception) { @@ -48,7 +59,7 @@ class NewPipeUtil(context: Context) { fun getFormats(url: String) : Result > { try { - val streamInfo = StreamInfo.getInfo(NewPipe.getService(ServiceList.YouTube.serviceId), url) + val streamInfo = StreamInfo.getInfo(url) val vid = createVideoFromStream(streamInfo, url, true) return Result.success(vid!!.formats) }catch(e: Exception) { @@ -62,7 +73,7 @@ class NewPipeUtil(context: Context) { return kotlin.runCatching { val formatCollection = mutableListOf>() urls.forEach { url -> - val streamInfo = StreamInfo.getInfo(NewPipe.getService(ServiceList.YouTube.serviceId), url) + val streamInfo = StreamInfo.getInfo(url) createVideoFromStream(streamInfo, url, true).apply { if (this!!.formats.isEmpty()) return Result.failure(Throwable()) formatCollection.add(this.formats) @@ -128,7 +139,7 @@ class NewPipeUtil(context: Context) { fun getStreamingUrlAndChapters(url: String) : Result, List?>> { try { - val streamInfo = StreamInfo.getInfo(NewPipe.getService(ServiceList.YouTube.serviceId), url) + val streamInfo = StreamInfo.getInfo(url) val item = createVideoFromStream(streamInfo, url) if (item!!.urls.isBlank()) return Result.failure(Throwable()) val urls = item.urls.split(",") @@ -426,4 +437,10 @@ class NewPipeUtil(context: Context) { query = el[0] return query!! } + + private fun generatePoToken(url: String) : PoTokenResult? { + val generator = NewPipePoTokenGenerator() + val id = getIDFromYoutubeURL(url) + return generator.getWebClientPoToken(id) + } } \ No newline at end of file diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/JavascriptUtil.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/JavascriptUtil.kt new file mode 100644 index 00000000..0d7e7a06 --- /dev/null +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/JavascriptUtil.kt @@ -0,0 +1,131 @@ +package com.deniscerri.ytdl.util.extractors.newpipe.potoken + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.long +import okio.ByteString.Companion.decodeBase64 +import okio.ByteString.Companion.toByteString + +object JavascriptUtil { + /** + * Parses the raw challenge data obtained from the Create endpoint and returns an object that can be + * embedded in a JavaScript snippet. + */ + fun parseChallengeData(rawChallengeData: String): String { + val scrambled = Json.parseToJsonElement(rawChallengeData).jsonArray + + val challengeData = if (scrambled.size > 1 && scrambled[1].jsonPrimitive.isString) { + val descrambled = descramble(scrambled[1].jsonPrimitive.content) + Json.parseToJsonElement(descrambled).jsonArray + } else { + scrambled[1].jsonArray + } + + val messageId = challengeData[0].jsonPrimitive.content + val interpreterHash = challengeData[3].jsonPrimitive.content + val program = challengeData[4].jsonPrimitive.content + val globalName = challengeData[5].jsonPrimitive.content + val clientExperimentsStateBlob = challengeData[7].jsonPrimitive.content + + val privateDoNotAccessOrElseSafeScriptWrappedValue = challengeData[1] + .takeIf { it !is JsonNull } + ?.jsonArray + ?.find { it.jsonPrimitive.isString } + val privateDoNotAccessOrElseTrustedResourceUrlWrappedValue = challengeData[2] + .takeIf { it !is JsonNull } + ?.jsonArray + ?.find { it.jsonPrimitive.isString } + + return Json.encodeToString( + JsonObject.serializer(), JsonObject( + mapOf( + "messageId" to JsonPrimitive(messageId), + "interpreterJavascript" to JsonObject( + mapOf( + "privateDoNotAccessOrElseSafeScriptWrappedValue" to (privateDoNotAccessOrElseSafeScriptWrappedValue + ?: JsonNull), + "privateDoNotAccessOrElseTrustedResourceUrlWrappedValue" to (privateDoNotAccessOrElseTrustedResourceUrlWrappedValue + ?: JsonNull) + ) + ), + "interpreterHash" to JsonPrimitive(interpreterHash), + "program" to JsonPrimitive(program), + "globalName" to JsonPrimitive(globalName), + "clientExperimentsStateBlob" to JsonPrimitive(clientExperimentsStateBlob) + ) + ) + ) + } + + /** + * Parses the raw integrity token data obtained from the GenerateIT endpoint to a JavaScript + * `Uint8Array` that can be embedded directly in JavaScript code, and an [Int] representing the + * duration of this token in seconds. + */ + fun parseIntegrityTokenData(rawIntegrityTokenData: String): Pair { + val integrityTokenData = Json.parseToJsonElement(rawIntegrityTokenData).jsonArray + return base64ToU8(integrityTokenData[0].jsonPrimitive.content) to integrityTokenData[1].jsonPrimitive.long + } + + /** + * Converts a string (usually the identifier used as input to `obtainPoToken`) to a JavaScript + * `Uint8Array` that can be embedded directly in JavaScript code. + */ + fun stringToU8(identifier: String): String { + return newUint8Array(identifier.toByteArray()) + } + + /** + * Takes a poToken encoded as a sequence of bytes represented as integers separated by commas + * (e.g. "97,98,99" would be "abc"), which is the output of `Uint8Array::toString()` in JavaScript, + * and converts it to the specific base64 representation for poTokens. + */ + fun u8ToBase64(poToken: String): String { + return poToken.split(",") + .map { it.toUByte().toByte() } + .toByteArray() + .toByteString() + .base64() + .replace("+", "-") + .replace("/", "_") + } + + /** + * Takes the scrambled challenge, decodes it from base64, adds 97 to each byte. + */ + private fun descramble(scrambledChallenge: String): String { + return base64ToByteString(scrambledChallenge) + .map { (it + 97).toByte() } + .toByteArray() + .decodeToString() + } + + /** + * Decodes a base64 string encoded in the specific base64 representation used by YouTube, and + * returns a JavaScript `Uint8Array` that can be embedded directly in JavaScript code. + */ + private fun base64ToU8(base64: String): String { + return newUint8Array(base64ToByteString(base64)) + } + + private fun newUint8Array(contents: ByteArray): String { + return "new Uint8Array([" + contents.joinToString(separator = ",") { it.toUByte().toString() } + "])" + } + + /** + * Decodes a base64 string encoded in the specific base64 representation used by YouTube. + */ + private fun base64ToByteString(base64: String): ByteArray { + val base64Mod = base64 + .replace('-', '+') + .replace('_', '/') + .replace('.', '=') + + return (base64Mod.decodeBase64() ?: throw PoTokenException("Cannot base64 decode")) + .toByteArray() + } +} \ No newline at end of file diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/NewPipePoTokenGenerator.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/NewPipePoTokenGenerator.kt new file mode 100644 index 00000000..4cde37a6 --- /dev/null +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/NewPipePoTokenGenerator.kt @@ -0,0 +1,142 @@ +package com.deniscerri.ytdl.util.extractors.newpipe.potoken + +import android.os.Handler +import android.os.Looper +import android.util.Log +import android.webkit.CookieManager +import androidx.preference.PreferenceManager +import com.deniscerri.ytdl.App +import com.deniscerri.ytdl.BuildConfig +import com.deniscerri.ytdl.database.models.YoutubePlayerClientItem +import com.deniscerri.ytdl.database.models.YoutubePoTokenItem +import com.google.gson.Gson +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.withContext +import org.schabi.newpipe.extractor.NewPipe +import org.schabi.newpipe.extractor.services.youtube.InnertubeClientRequestInfo +import org.schabi.newpipe.extractor.services.youtube.PoTokenProvider +import org.schabi.newpipe.extractor.services.youtube.PoTokenResult +import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper + +class NewPipePoTokenGenerator : PoTokenProvider { + val TAG = NewPipePoTokenGenerator::class.simpleName + private val supportsWebView by lazy { runCatching { CookieManager.getInstance() }.isSuccess } + + private object WebPoTokenGenLock + private var webPoTokenVisitorData: String? = null + private var webPoTokenStreamingPot: String? = null + private var webPoTokenGenerator: PoTokenWebView? = null + + + override fun getWebClientPoToken(videoId: String): PoTokenResult? { + if (!supportsWebView) { + return null + } + + val result = kotlin.runCatching { + getWebClientPoToken(videoId, false) + } + + result.getOrNull()?.apply { + val preferences = PreferenceManager.getDefaultSharedPreferences(App.instance) + val editor = preferences.edit() + editor.putString("newpipe_gvs_potoken", this.streamingDataPoToken ?: "") + editor.putString("newpipe_player_potoken", this.playerRequestPoToken) + editor.putString("newpipe_visitordata", this.visitorData) + editor.apply() + } + return result.getOrNull() + } + + /** + * @param forceRecreate whether to force the recreation of [webPoTokenGenerator], to be used in + * case the current [webPoTokenGenerator] threw an error last time + * [PoTokenGenerator.generatePoToken] was called + */ + private fun getWebClientPoToken(videoId: String, forceRecreate: Boolean): PoTokenResult { + // just a helper class since Kotlin does not have builtin support for 4-tuples + data class Quadruple(val t1: T1, val t2: T2, val t3: T3, val t4: T4) + + val (poTokenGenerator, visitorData, streamingPot, hasBeenRecreated) = + synchronized(WebPoTokenGenLock) { + val shouldRecreate = webPoTokenGenerator == null || forceRecreate || webPoTokenGenerator!!.isExpired + + if (shouldRecreate) { + val innertubeClientRequestInfo = InnertubeClientRequestInfo.ofWebClient() + innertubeClientRequestInfo.clientInfo.clientVersion = + YoutubeParsingHelper.getClientVersion() + + webPoTokenVisitorData = YoutubeParsingHelper.getVisitorDataFromInnertube( + innertubeClientRequestInfo, + NewPipe.getPreferredLocalization(), + NewPipe.getPreferredContentCountry(), + YoutubeParsingHelper.getYouTubeHeaders(), + YoutubeParsingHelper.YOUTUBEI_V1_URL, + null, + false + ) + + runBlocking { + // close the current webPoTokenGenerator on the main thread + webPoTokenGenerator?.let { Handler(Looper.getMainLooper()).post { it.close() } } + + // create a new webPoTokenGenerator + webPoTokenGenerator = PoTokenWebView + .getNewPoTokenGenerator(App.instance) + + // The streaming poToken needs to be generated exactly once before generating + // any other (player) tokens. + webPoTokenStreamingPot = webPoTokenGenerator!!.generatePoToken(webPoTokenVisitorData!!) + } + } + + return@synchronized Quadruple( + webPoTokenGenerator!!, + webPoTokenVisitorData!!, + webPoTokenStreamingPot!!, + shouldRecreate + ) + } + + val playerPot = try { + // Not using synchronized here, since poTokenGenerator would be able to generate + // multiple poTokens in parallel if needed. The only important thing is for exactly one + // visitorData/streaming poToken to be generated before anything else. + runBlocking { + poTokenGenerator.generatePoToken(videoId) + } + } catch (throwable: Throwable) { + if (hasBeenRecreated) { + // the poTokenGenerator has just been recreated (and possibly this is already the + // second time we try), so there is likely nothing we can do + throw throwable + } else { + // retry, this time recreating the [webPoTokenGenerator] from scratch; + // this might happen for example if NewPipe goes in the background and the WebView + // content is lost + Log.e(TAG, "Failed to obtain poToken, retrying", throwable) + return getWebClientPoToken(videoId = videoId, forceRecreate = true) + } + } + + + if (BuildConfig.DEBUG) { + Log.d( + TAG, + "poToken for $videoId: playerPot=$playerPot, " + + "streamingPot=$streamingPot, visitor_data=$visitorData" + ) + } + + return PoTokenResult(visitorData, playerPot, streamingPot) + } + + override fun getWebEmbedClientPoToken(videoId: String?): PoTokenResult? = null + + override fun getAndroidClientPoToken(videoId: String?): PoTokenResult? = null + + override fun getIosClientPoToken(videoId: String?): PoTokenResult? = null +} \ No newline at end of file diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenException.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenException.kt new file mode 100644 index 00000000..61c2f4ca --- /dev/null +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenException.kt @@ -0,0 +1,13 @@ +package com.deniscerri.ytdl.util.extractors.newpipe.potoken + +class PoTokenException(message: String) : Exception(message) + +// to be thrown if the WebView provided by the system is broken +class BadWebViewException(message: String) : Exception(message) + +fun buildExceptionForJsError(error: String): Exception { + return if (error.contains("SyntaxError")) + BadWebViewException(error) + else + PoTokenException(error) +} \ No newline at end of file diff --git a/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenWebView.kt b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenWebView.kt new file mode 100644 index 00000000..23fd72f0 --- /dev/null +++ b/app/src/main/java/com/deniscerri/ytdl/util/extractors/newpipe/potoken/PoTokenWebView.kt @@ -0,0 +1,353 @@ +package com.deniscerri.ytdl.util.extractors.newpipe.potoken + +import android.content.Context +import android.os.Build +import android.util.Log +import android.webkit.ConsoleMessage +import android.webkit.JavascriptInterface +import android.webkit.WebChromeClient +import android.webkit.WebView +import androidx.annotation.MainThread +import androidx.collection.ArrayMap +import androidx.preference.PreferenceManager +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.MainScope +import kotlinx.coroutines.cancel +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import com.deniscerri.ytdl.BuildConfig +import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseChallengeData +import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.parseIntegrityTokenData +import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.stringToU8 +import com.deniscerri.ytdl.util.extractors.newpipe.potoken.JavascriptUtil.u8ToBase64 +import okhttp3.Headers.Companion.toHeaders +import okhttp3.OkHttpClient +import okhttp3.RequestBody.Companion.toRequestBody +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.Collections +import kotlin.coroutines.Continuation +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException + +class PoTokenWebView private constructor( + context: Context, + // to be used exactly once only during initialization! + private val continuation: Continuation, +) { + private val webView = WebView(context) + private val scope = MainScope() + private val poTokenContinuations = + Collections.synchronizedMap(ArrayMap>()) + private val exceptionHandler = CoroutineExceptionHandler { _, t -> + onInitializationErrorCloseAndCancel(t) + } + private lateinit var expirationInstant: Instant + + //region Initialization + init { + val webViewSettings = webView.settings + //noinspection SetJavaScriptEnabled we want to use JavaScript! + webViewSettings.javaScriptEnabled = true + if (Build.VERSION.SDK_INT >= 26) { + webViewSettings.safeBrowsingEnabled = false + } + webViewSettings.userAgentString = USER_AGENT + webViewSettings.blockNetworkLoads = true // the WebView does not need internet access + + // so that we can run async functions and get back the result + webView.addJavascriptInterface(this, JS_INTERFACE) + + val preferences = PreferenceManager.getDefaultSharedPreferences(context) + preferences.getString("api_key", "")?.apply { + if (this.isNotBlank()) { + GOOGLE_API_KEY = this + } + } + + webView.webChromeClient = object : WebChromeClient() { + override fun onConsoleMessage(m: ConsoleMessage): Boolean { + if (m.message().contains("Uncaught")) { + // There should not be any uncaught errors while executing the code, because + // everything that can fail is guarded by try-catch. Therefore, this likely + // indicates that there was a syntax error in the code, i.e. the WebView only + // supports a really old version of JS. + + val fmt = "\"${m.message()}\", source: ${m.sourceId()} (${m.lineNumber()})" + val exception = BadWebViewException(fmt) + Log.e(TAG, "This WebView implementation is broken: $fmt") + + onInitializationErrorCloseAndCancel(exception) + popAllPoTokenContinuations().forEach { (_, cont) -> cont.resumeWithException(exception) } + } + return super.onConsoleMessage(m) + } + } + } + + /** + * Must be called right after instantiating [PoTokenWebView] to perform the actual + * initialization. This will asynchronously go through all the steps needed to load BotGuard, + * run it, and obtain an `integrityToken`. + */ + private fun loadHtmlAndObtainBotguard() { + Log.d(TAG, "loadHtmlAndObtainBotguard() called") + + scope.launch(exceptionHandler) { + val html = withContext(Dispatchers.IO) { + webView.context.assets.open("po_token.html").bufferedReader().use { it.readText() } + } + + // calls downloadAndRunBotguard() when the page has finished loading + val data = html.replaceFirst("", "\n$JS_INTERFACE.downloadAndRunBotguard()") + webView.loadDataWithBaseURL("https://www.youtube.com", data, "text/html", "utf-8", null) + } + } + + /** + * Called during initialization by the JavaScript snippet appended to the HTML page content in + * [loadHtmlAndObtainBotguard] after the WebView content has been loaded. + */ + @JavascriptInterface + fun downloadAndRunBotguard() { + Log.d(TAG, "downloadAndRunBotguard() called") + + makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/Create", + "[ \"$REQUEST_KEY\" ]", + ) { responseBody -> + val parsedChallengeData = parseChallengeData(responseBody) + webView.evaluateJavascript( + """try { + data = $parsedChallengeData + runBotGuard(data).then(function (result) { + this.webPoSignalOutput = result.webPoSignalOutput + $JS_INTERFACE.onRunBotguardResult(result.botguardResponse) + }, function (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }) + } catch (error) { + $JS_INTERFACE.onJsInitializationError(error + "\n" + error.stack) + }""", + null + ) + } + } + + /** + * Called during initialization by the JavaScript snippets from either + * [downloadAndRunBotguard] or [onRunBotguardResult]. + */ + @JavascriptInterface + fun onJsInitializationError(error: String) { + if (BuildConfig.DEBUG) { + Log.e(TAG, "Initialization error from JavaScript: $error") + } + onInitializationErrorCloseAndCancel(buildExceptionForJsError(error)) + } + + /** + * Called during initialization by the JavaScript snippet from [downloadAndRunBotguard] after + * obtaining the BotGuard execution output [botguardResponse]. + */ + @JavascriptInterface + fun onRunBotguardResult(botguardResponse: String) { + Log.d(TAG, "botguardResponse: $botguardResponse") + makeBotguardServiceRequest( + "https://www.youtube.com/api/jnn/v1/GenerateIT", + "[ \"$REQUEST_KEY\", \"$botguardResponse\" ]", + ) { responseBody -> + Log.d(TAG, "GenerateIT response: $responseBody") + val (integrityToken, expirationTimeInSeconds) = parseIntegrityTokenData(responseBody) + + // leave 10 minutes of margin just to be sure + expirationInstant = Instant.now().plusSeconds(expirationTimeInSeconds).minus(10, ChronoUnit.MINUTES) + + webView.evaluateJavascript("this.integrityToken = $integrityToken") { + Log.d(TAG, "initialization finished, expiration=${expirationTimeInSeconds}s") + continuation.resume(this) + } + } + } + //endregion + + //region Obtaining poTokens + suspend fun generatePoToken(identifier: String): String { + return withContext(Dispatchers.Main) { + suspendCancellableCoroutine { cont -> + Log.d(TAG, "generatePoToken() called with identifier $identifier") + addPoTokenEmitter(identifier, cont) + webView.evaluateJavascript( + """try { + identifier = "$identifier" + u8Identifier = ${stringToU8(identifier)} + poTokenU8 = obtainPoToken(webPoSignalOutput, integrityToken, u8Identifier) + poTokenU8String = poTokenU8.join(",") + $JS_INTERFACE.onObtainPoTokenResult(identifier, poTokenU8String) + } catch (error) { + $JS_INTERFACE.onObtainPoTokenError(identifier, error + "\n" + error.stack) + }""", + null + ) + } + } + } + + /** + * Called by the JavaScript snippet from [generatePoToken] when an error occurs in calling the + * JavaScript `obtainPoToken()` function. + */ + @JavascriptInterface + fun onObtainPoTokenError(identifier: String, error: String) { + if (BuildConfig.DEBUG) { + Log.e(TAG, "obtainPoToken error from JavaScript: $error") + } + popPoTokenContinuation(identifier)?.resumeWithException(buildExceptionForJsError(error)) + } + + /** + * Called by the JavaScript snippet from [generatePoToken] with the original identifier and the + * result of the JavaScript `obtainPoToken()` function. + */ + @JavascriptInterface + fun onObtainPoTokenResult(identifier: String, poTokenU8: String) { + Log.d(TAG, "Generated poToken (before decoding): identifier=$identifier poTokenU8=$poTokenU8") + val poToken = try { + u8ToBase64(poTokenU8) + } catch (t: Throwable) { + popPoTokenContinuation(identifier)?.resumeWithException(t) + return + } + + Log.d(TAG, "Generated poToken: identifier=$identifier poToken=$poToken") + popPoTokenContinuation(identifier)?.resume(poToken) + } + + val isExpired: Boolean + get() = Instant.now().isAfter(expirationInstant) + //endregion + + //region Handling multiple emitters + /** + * Adds the ([identifier], [continuation]) pair to the [poTokenContinuations] list. This makes + * it so that multiple poToken requests can be generated in parallel, and the results will be + * notified to the right continuations. + */ + private fun addPoTokenEmitter(identifier: String, continuation: Continuation) { + poTokenContinuations[identifier] = continuation + } + + /** + * Extracts and removes from the [poTokenContinuations] list a [Continuation] based on its + * [identifier]. The continuation is supposed to be used immediately after to either signal a + * success or an error. + */ + private fun popPoTokenContinuation(identifier: String): Continuation? { + return poTokenContinuations.remove(identifier) + } + + /** + * Clears [poTokenContinuations] and returns its previous contents. The continuations are supposed + * to be used immediately after to either signal a success or an error. + */ + private fun popAllPoTokenContinuations(): Map> { + val result = poTokenContinuations.toMap() + poTokenContinuations.clear() + return result + } + //endregion + + //region Utils + /** + * Makes a POST request to [url] with the given [data] by setting the correct headers. Calls + * [onInitializationErrorCloseAndCancel] in case of any network errors and also if the response + * does not have HTTP code 200, therefore this is supposed to be used only during + * initialization. Calls [handleResponseBody] with the response body if the response is + * successful. The request is performed in the background and a disposable is added to + * [disposables]. + */ + private fun makeBotguardServiceRequest( + url: String, + data: String, + handleResponseBody: (String) -> Unit, + ) { + scope.launch(exceptionHandler) { + val requestBuilder = okhttp3.Request.Builder() + .post(data.toRequestBody()) + .headers(mapOf( + "User-Agent" to USER_AGENT, + "Accept" to "application/json", + "Content-Type" to "application/json+protobuf", + "x-goog-api-key" to GOOGLE_API_KEY, + "x-user-agent" to "grpc-web-javascript/0.1", + ).toHeaders()) + .url(url) + val response = withContext(Dispatchers.IO) { + httpClient.newCall(requestBuilder.build()).execute() + } + val httpCode = response.code + if (httpCode != 200) { + onInitializationErrorCloseAndCancel(PoTokenException("Invalid response code: $httpCode")) + } else { + val body = withContext(Dispatchers.IO) { + response.body!!.string() + } + handleResponseBody(body) + } + } + } + + /** + * Handles any error happening during initialization, releasing resources and sending the error + * to [continuation]. + */ + private fun onInitializationErrorCloseAndCancel(error: Throwable) { + close() + continuation.resumeWithException(error) + } + + /** + * Releases all [webView] resources. + */ + @MainThread + fun close() { + scope.cancel() + + webView.clearHistory() + // clears RAM cache and disk cache (globally for all WebViews) + webView.clearCache(true) + + // ensures that the WebView isn't doing anything when destroying it + webView.loadUrl("about:blank") + + webView.onPause() + webView.removeAllViews() + webView.destroy() + } + //endregion + + companion object { + private const val TAG = "PoTokenWebView" + //libretube api key, if user has his own api key his will be used + private var GOOGLE_API_KEY = "AIzaSyDyT5W0Jh49F30Pqqtyfdf7pDLFKLJoAnw" + private const val REQUEST_KEY = "O43z0dpjhgX20SCx4KAo" + private const val USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3" + private const val JS_INTERFACE = "PoTokenWebView" + + private val httpClient = OkHttpClient.Builder() + //.proxy(YouTube.proxy) + .build() + + suspend fun getNewPoTokenGenerator(context: Context): PoTokenWebView { + return withContext(Dispatchers.Main) { + suspendCancellableCoroutine { cont -> + val potWv = PoTokenWebView(context, cont) + potWv.loadHtmlAndObtainBotguard() + } + } + } + } +} \ No newline at end of file diff --git a/app/src/main/res/layout-land/activity_main.xml b/app/src/main/res/layout-land/activity_main.xml index e7b550fc..91650764 100644 --- a/app/src/main/res/layout-land/activity_main.xml +++ b/app/src/main/res/layout-land/activity_main.xml @@ -50,9 +50,11 @@ app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintEnd_toEndOf="parent" app:labelVisibilityMode="selected" + android:paddingTop="8dp" app:layout_constraintTop_toTopOf="parent" app:layout_constraintHorizontal_bias="0.0" app:layout_constraintStart_toStartOf="parent" + app:headerLayout="@layout/app_logo_fab" app:menu="@menu/bottom_nav_menu" /> diff --git a/app/src/main/res/layout-sw600dp/activity_main.xml b/app/src/main/res/layout-sw1240dp-land/activity_main.xml similarity index 85% rename from app/src/main/res/layout-sw600dp/activity_main.xml rename to app/src/main/res/layout-sw1240dp-land/activity_main.xml index e7b550fc..c67d916c 100644 --- a/app/src/main/res/layout-sw600dp/activity_main.xml +++ b/app/src/main/res/layout-sw1240dp-land/activity_main.xml @@ -39,21 +39,22 @@ app:navGraph="@navigation/nav_graph" app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintEnd_toEndOf="parent" - app:layout_constraintStart_toEndOf="@+id/bottomNavigationView" + app:layout_constraintStart_toEndOf="@+id/navigationView" app:layout_constraintTop_toTopOf="parent" /> - + app:menu="@menu/nav_menu_largescreen" /> diff --git a/app/src/main/res/layout-sw600dp/fragment_home.xml b/app/src/main/res/layout-sw1240dp-land/fragment_home.xml similarity index 91% rename from app/src/main/res/layout-sw600dp/fragment_home.xml rename to app/src/main/res/layout-sw1240dp-land/fragment_home.xml index 5cdf1414..50528b7d 100644 --- a/app/src/main/res/layout-sw600dp/fragment_home.xml +++ b/app/src/main/res/layout-sw1240dp-land/fragment_home.xml @@ -91,19 +91,10 @@ android:layout_height="wrap_content"> - - + @@ -156,7 +147,6 @@ android:layout_width="wrap_content" android:scrollbars="none" android:visibility="gone" - app:layout_scrollFlags="scroll|enterAlways" android:layout_height="wrap_content"> @@ -220,7 +210,30 @@ + + + + + + + + + + diff --git a/app/src/main/res/layout-sw600dp-land/activity_main.xml b/app/src/main/res/layout-sw600dp-land/activity_main.xml index c67d916c..91650764 100644 --- a/app/src/main/res/layout-sw600dp-land/activity_main.xml +++ b/app/src/main/res/layout-sw600dp-land/activity_main.xml @@ -39,22 +39,23 @@ app:navGraph="@navigation/nav_graph" app:layout_constraintBottom_toBottomOf="parent" app:layout_constraintEnd_toEndOf="parent" - app:layout_constraintStart_toEndOf="@+id/navigationView" + app:layout_constraintStart_toEndOf="@+id/bottomNavigationView" app:layout_constraintTop_toTopOf="parent" /> - + app:headerLayout="@layout/app_logo_fab" + app:menu="@menu/bottom_nav_menu" /> diff --git a/app/src/main/res/layout-sw600dp-land/fragment_home.xml b/app/src/main/res/layout-sw600dp-land/fragment_home.xml index 50528b7d..7beab8ef 100644 --- a/app/src/main/res/layout-sw600dp-land/fragment_home.xml +++ b/app/src/main/res/layout-sw600dp-land/fragment_home.xml @@ -91,10 +91,19 @@ android:layout_height="wrap_content"> + + - @@ -147,6 +156,7 @@ android:layout_width="wrap_content" android:scrollbars="none" android:visibility="gone" + app:layout_scrollFlags="scroll|enterAlways" android:layout_height="wrap_content"> @@ -210,30 +220,7 @@ - - - - - - - - - - diff --git a/app/src/main/res/layout/app_logo_fab.xml b/app/src/main/res/layout/app_logo_fab.xml new file mode 100644 index 00000000..7d7fdee6 --- /dev/null +++ b/app/src/main/res/layout/app_logo_fab.xml @@ -0,0 +1,12 @@ + + \ No newline at end of file diff --git a/app/src/main/res/layout/header_navigation_drawer.xml b/app/src/main/res/layout/header_navigation_drawer.xml index 44b95273..c781eeac 100644 --- a/app/src/main/res/layout/header_navigation_drawer.xml +++ b/app/src/main/res/layout/header_navigation_drawer.xml @@ -3,6 +3,7 @@ xmlns:app="http://schemas.android.com/apk/res-auto" android:orientation="horizontal" android:layout_marginStart="15dp" + android:layout_marginTop="10dp" xmlns:android="http://schemas.android.com/apk/res/android"> + + diff --git a/app/src/main/res/values-sw600dp-land/attrs.xml b/app/src/main/res/values-sw600dp-land/attrs.xml index ec4c1a5c..ada7c233 100644 --- a/app/src/main/res/values-sw600dp-land/attrs.xml +++ b/app/src/main/res/values-sw600dp-land/attrs.xml @@ -1,5 +1,4 @@ - - 3 - true + 2 + true \ No newline at end of file diff --git a/app/src/main/res/values-sw800dp-land/attrs.xml b/app/src/main/res/values-sw800dp-land/attrs.xml new file mode 100644 index 00000000..ec4c1a5c --- /dev/null +++ b/app/src/main/res/values-sw800dp-land/attrs.xml @@ -0,0 +1,5 @@ + + + 3 + true + \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 65cd6835..b35fa88e 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -472,4 +472,6 @@ Automatically make a backup of everything when a new version the application is found Write subtitles when embedding This will apply Save Subtitles and Save Automatic Subtitles to download the sub files and then delete them after embedding + Use Autogenerated PO Tokens from NewPipe Extractor + By enabling this, the app will disable cookies and use the generated po tokens and visitor data as YouTube Extractor Arguments \ No newline at end of file diff --git a/app/src/main/res/xml/advanced_preferences.xml b/app/src/main/res/xml/advanced_preferences.xml index b37825a5..c2c21484 100644 --- a/app/src/main/res/xml/advanced_preferences.xml +++ b/app/src/main/res/xml/advanced_preferences.xml @@ -10,6 +10,13 @@ android:summary="@string/player_client_summary" android:title="@string/player_client" /> + +