From f85e44a82ac6a37c0c19cf62a15750dbb875b993 Mon Sep 17 00:00:00 2001 From: Devoxin Date: Sat, 30 Mar 2024 12:13:00 +0000 Subject: [PATCH] Move to WEB client, add 403/400 retry mechanism --- .../youtube/DefaultYoutubePlaylistLoader.java | 2 + .../DefaultYoutubeTrackDetailsLoader.java | 63 +++++++++++-------- .../youtube/YoutubeAccessTokenTracker.java | 2 +- .../source/youtube/YoutubeAudioTrack.java | 34 ++++++++-- .../source/youtube/YoutubeMixProvider.java | 2 + .../source/youtube/YoutubeSearchProvider.java | 2 +- .../youtube/YoutubeTrackDetailsLoader.java | 15 ++++- 7 files changed, 88 insertions(+), 32 deletions(-) diff --git a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubePlaylistLoader.java b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubePlaylistLoader.java index cde3bfa4..59818c19 100644 --- a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubePlaylistLoader.java +++ b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubePlaylistLoader.java @@ -36,6 +36,8 @@ public void setPlaylistPageCount(int playlistPageCount) { public AudioPlaylist load(HttpInterface httpInterface, String playlistId, String selectedVideoId, Function trackFactory) { HttpPost post = new HttpPost(BROWSE_URL); + // TODO: Keep an eye on this. + // Can't drop in WEB as it yields no results. YoutubeClientConfig clientConfig = YoutubeClientConfig.ANDROID.copy() .withRootField("browseId", "VL" + playlistId) .setAttribute(httpInterface); diff --git a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubeTrackDetailsLoader.java b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubeTrackDetailsLoader.java index 4f1195f7..14e0d98b 100644 --- a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubeTrackDetailsLoader.java +++ b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubeTrackDetailsLoader.java @@ -32,8 +32,17 @@ public class DefaultYoutubeTrackDetailsLoader implements YoutubeTrackDetailsLoad @Override public YoutubeTrackDetails loadDetails(HttpInterface httpInterface, String videoId, boolean requireFormats, YoutubeAudioSourceManager sourceManager) { + return loadDetails(httpInterface, videoId, requireFormats, sourceManager, null); + } + + @Override + public YoutubeTrackDetails loadDetails(HttpInterface httpInterface, + String videoId, + boolean requireFormats, + YoutubeAudioSourceManager sourceManager, + YoutubeClientConfig clientOverride) { try { - return load(httpInterface, videoId, requireFormats, sourceManager); + return load(httpInterface, videoId, requireFormats, sourceManager, clientOverride); } catch (IOException e) { throw ExceptionTools.toRuntimeException(e); } @@ -43,9 +52,10 @@ private YoutubeTrackDetails load( HttpInterface httpInterface, String videoId, boolean requireFormats, - YoutubeAudioSourceManager sourceManager + YoutubeAudioSourceManager sourceManager, + YoutubeClientConfig clientOverride ) throws IOException { - JsonBrowser mainInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, null); + JsonBrowser mainInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, null, clientOverride); try { YoutubeTrackJsonData initialData = loadBaseResponse(mainInfo, httpInterface, videoId, sourceManager); @@ -82,7 +92,7 @@ protected YoutubeTrackJsonData loadBaseResponse( } if (status == InfoStatus.PREMIERE_TRAILER) { - JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status); + JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status, null); data = YoutubeTrackJsonData.fromMainResult(trackInfo .get("playabilityStatus") .get("errorScreen") @@ -93,13 +103,13 @@ protected YoutubeTrackJsonData loadBaseResponse( } if (status == InfoStatus.REQUIRES_LOGIN) { - JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status); + JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status, null); data = YoutubeTrackJsonData.fromMainResult(trackInfo); status = checkPlayabilityStatus(data.playerResponse, true); } if (status == InfoStatus.NON_EMBEDDABLE) { - JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status); + JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status, null); data = YoutubeTrackJsonData.fromMainResult(trackInfo); checkPlayabilityStatus(data.playerResponse, true); } @@ -197,7 +207,8 @@ protected JsonBrowser loadTrackInfoFromInnertube( HttpInterface httpInterface, String videoId, YoutubeAudioSourceManager sourceManager, - InfoStatus infoStatus + InfoStatus infoStatus, + YoutubeClientConfig clientOverride ) throws IOException { if (cachedPlayerScript == null) fetchScript(videoId, httpInterface); @@ -206,24 +217,26 @@ protected JsonBrowser loadTrackInfoFromInnertube( cachedPlayerScript.playerScriptUrl ); HttpPost post = new HttpPost(PLAYER_URL); - YoutubeClientConfig clientConfig; - - if (infoStatus == InfoStatus.PREMIERE_TRAILER) { - // Android client gives encoded Base64 response to trailer which is also protobuf so we can't decode it - clientConfig = YoutubeClientConfig.WEB.copy(); - } else if (infoStatus == InfoStatus.NON_EMBEDDABLE) { - // Used when age restriction bypass failed, if we have valid auth then most likely this request will be successful - clientConfig = YoutubeClientConfig.ANDROID.copy() - .withRootField("params", PLAYER_PARAMS); - } else if (infoStatus == InfoStatus.REQUIRES_LOGIN) { - // Age restriction bypass - clientConfig = YoutubeClientConfig.TV_EMBEDDED.copy(); - } else { - // Default payload from what we start trying to get required data - clientConfig = YoutubeClientConfig.ANDROID.copy() - .withClientField("clientScreen", CLIENT_SCREEN_EMBED) - .withThirdPartyEmbedUrl(CLIENT_THIRD_PARTY_EMBED) - .withRootField("params", PLAYER_PARAMS); + YoutubeClientConfig clientConfig = clientOverride; + + if (clientConfig == null) { + if (infoStatus == InfoStatus.PREMIERE_TRAILER) { + // Android client gives encoded Base64 response to trailer which is also protobuf so we can't decode it + clientConfig = YoutubeClientConfig.WEB.copy(); + } else if (infoStatus == InfoStatus.NON_EMBEDDABLE) { + // Used when age restriction bypass failed, if we have valid auth then most likely this request will be successful + clientConfig = YoutubeClientConfig.WEB.copy() + .withRootField("params", PLAYER_PARAMS); + } else if (infoStatus == InfoStatus.REQUIRES_LOGIN) { + // Age restriction bypass + clientConfig = YoutubeClientConfig.TV_EMBEDDED.copy(); + } else { + // Default payload from what we start trying to get required data + clientConfig = YoutubeClientConfig.WEB.copy() + .withClientField("clientScreen", CLIENT_SCREEN_EMBED) + .withThirdPartyEmbedUrl(CLIENT_THIRD_PARTY_EMBED) + .withRootField("params", PLAYER_PARAMS); + } } clientConfig diff --git a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAccessTokenTracker.java b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAccessTokenTracker.java index 06f142cc..c85c2eca 100644 --- a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAccessTokenTracker.java +++ b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAccessTokenTracker.java @@ -229,7 +229,7 @@ private String fetchVisitorId() throws IOException { try (HttpInterface httpInterface = httpInterfaceManager.getInterface()) { httpInterface.getContext().setAttribute(TOKEN_FETCH_CONTEXT_ATTRIBUTE, true); - YoutubeClientConfig clientConfig = YoutubeClientConfig.ANDROID.copy().setAttribute(httpInterface); + YoutubeClientConfig clientConfig = YoutubeClientConfig.WEB.copy().setAttribute(httpInterface); HttpPost visitorIdPost = new HttpPost(VISITOR_ID_URL); StringEntity visitorIdPayload = new StringEntity(clientConfig.toJsonString(), "UTF-8"); visitorIdPost.setEntity(visitorIdPayload); diff --git a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAudioTrack.java b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAudioTrack.java index 992bb09c..8805f3a0 100644 --- a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAudioTrack.java +++ b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAudioTrack.java @@ -25,6 +25,8 @@ */ public class YoutubeAudioTrack extends DelegatedAudioTrack { private static final Logger log = LoggerFactory.getLogger(YoutubeAudioTrack.class); + private static final int MAX_RETRIES = 3; + private final YoutubeAudioSourceManager sourceManager; @@ -41,23 +43,47 @@ public YoutubeAudioTrack(AudioTrackInfo trackInfo, YoutubeAudioSourceManager sou @Override public void process(LocalAudioTrackExecutor localExecutor) throws Exception { try (HttpInterface httpInterface = sourceManager.getHttpInterface()) { - FormatWithUrl format = loadBestFormatWithUrl(httpInterface); + FormatWithUrl format = loadBestFormatWithUrl(httpInterface, null); log.debug("Starting track from URL: {}", format.signedUrl); if (trackInfo.isStream || format.details.getContentLength() == CONTENT_LENGTH_UNKNOWN) { processStream(localExecutor, format); } else { - processStatic(localExecutor, httpInterface, format); + processStaticWithRetry(localExecutor, httpInterface, format); } } } + @Override public boolean isSeekable() { return true; } + @SuppressWarnings("ConstantValue") + private void processStaticWithRetry(LocalAudioTrackExecutor localExecutor, HttpInterface httpInterface, FormatWithUrl initialFormat) throws Exception { + FormatWithUrl format = initialFormat; + + for (int i = 1; i <= MAX_RETRIES; i++) { + log.debug("Opening YouTube stream URL attempt {}/{}", i, MAX_RETRIES); + + try { + processStatic(localExecutor, httpInterface, format); + return; + } catch (RuntimeException e) { + String message = e.getMessage(); + + // Throw if it's not a message we're expecting, or this is the last retry. + if (!"Not success status code: 403".equals(message) && !"Invalid status code for video page response: 400".equals(message) || i == MAX_RETRIES) { + throw e; + } + + format = loadBestFormatWithUrl(httpInterface, null); + } + } + } + private void processStatic(LocalAudioTrackExecutor localExecutor, HttpInterface httpInterface, FormatWithUrl format) throws Exception { try (YoutubePersistentHttpStream stream = new YoutubePersistentHttpStream(httpInterface, format.signedUrl, format.details.getContentLength())) { if (format.details.getType().getMimeType().endsWith("/webm")) { @@ -78,9 +104,9 @@ private void processStream(LocalAudioTrackExecutor localExecutor, FormatWithUrl } } - private FormatWithUrl loadBestFormatWithUrl(HttpInterface httpInterface) throws Exception { + private FormatWithUrl loadBestFormatWithUrl(HttpInterface httpInterface, YoutubeClientConfig clientConfig) throws Exception { YoutubeTrackDetails details = sourceManager.getTrackDetailsLoader() - .loadDetails(httpInterface, getIdentifier(), true, sourceManager); + .loadDetails(httpInterface, getIdentifier(), true, sourceManager, clientConfig); // If the error reason is "Video unavailable" details will return null if (details == null) { diff --git a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeMixProvider.java b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeMixProvider.java index 8b5a4585..5cb1fd52 100644 --- a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeMixProvider.java +++ b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeMixProvider.java @@ -44,6 +44,8 @@ public AudioPlaylist load( List tracks = new ArrayList<>(); HttpPost post = new HttpPost(NEXT_URL); + // TODO: Keep an eye on this. + // Can't drop in WEB as it yields no results. YoutubeClientConfig clientConfig = YoutubeClientConfig.ANDROID.copy() .withRootField("videoId", selectedVideoId) .withRootField("playlistId", mixId) diff --git a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeSearchProvider.java b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeSearchProvider.java index fcbe33e1..3703c21a 100644 --- a/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeSearchProvider.java +++ b/main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeSearchProvider.java @@ -52,7 +52,7 @@ public AudioItem loadSearchResult(String query, Function