Move to WEB client, add 403/400 retry mechanism

main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubePlaylistLoader.java

@@ -36,6 +36,8 @@ public void setPlaylistPageCount(int playlistPageCount) {
     public AudioPlaylist load(HttpInterface httpInterface, String playlistId, String selectedVideoId,
                               Function<AudioTrackInfo, AudioTrack> trackFactory) {
         HttpPost post = new HttpPost(BROWSE_URL);
+        // TODO: Keep an eye on this.
+        // Can't drop in WEB as it yields no results.
         YoutubeClientConfig clientConfig = YoutubeClientConfig.ANDROID.copy()
             .withRootField("browseId", "VL" + playlistId)
             .setAttribute(httpInterface);

main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/DefaultYoutubeTrackDetailsLoader.java

@@ -32,8 +32,17 @@ public class DefaultYoutubeTrackDetailsLoader implements YoutubeTrackDetailsLoad
 
     @Override
     public YoutubeTrackDetails loadDetails(HttpInterface httpInterface, String videoId, boolean requireFormats, YoutubeAudioSourceManager sourceManager) {
+        return loadDetails(httpInterface, videoId, requireFormats, sourceManager, null);
+    }
+
+    @Override
+    public YoutubeTrackDetails loadDetails(HttpInterface httpInterface,
+                                           String videoId,
+                                           boolean requireFormats,
+                                           YoutubeAudioSourceManager sourceManager,
+                                           YoutubeClientConfig clientOverride) {
         try {
-            return load(httpInterface, videoId, requireFormats, sourceManager);
+            return load(httpInterface, videoId, requireFormats, sourceManager, clientOverride);
         } catch (IOException e) {
             throw ExceptionTools.toRuntimeException(e);
         }
@@ -43,9 +52,10 @@ private YoutubeTrackDetails load(
         HttpInterface httpInterface,
         String videoId,
         boolean requireFormats,
-        YoutubeAudioSourceManager sourceManager
+        YoutubeAudioSourceManager sourceManager,
+        YoutubeClientConfig clientOverride
     ) throws IOException {
-        JsonBrowser mainInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, null);
+        JsonBrowser mainInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, null, clientOverride);
 
         try {
             YoutubeTrackJsonData initialData = loadBaseResponse(mainInfo, httpInterface, videoId, sourceManager);
@@ -82,7 +92,7 @@ protected YoutubeTrackJsonData loadBaseResponse(
         }
 
         if (status == InfoStatus.PREMIERE_TRAILER) {
-            JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status);
+            JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status, null);
             data = YoutubeTrackJsonData.fromMainResult(trackInfo
                 .get("playabilityStatus")
                 .get("errorScreen")
@@ -93,13 +103,13 @@ protected YoutubeTrackJsonData loadBaseResponse(
         }
 
         if (status == InfoStatus.REQUIRES_LOGIN) {
-            JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status);
+            JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status, null);
             data = YoutubeTrackJsonData.fromMainResult(trackInfo);
             status = checkPlayabilityStatus(data.playerResponse, true);
         }
 
         if (status == InfoStatus.NON_EMBEDDABLE) {
-            JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status);
+            JsonBrowser trackInfo = loadTrackInfoFromInnertube(httpInterface, videoId, sourceManager, status, null);
             data = YoutubeTrackJsonData.fromMainResult(trackInfo);
             checkPlayabilityStatus(data.playerResponse, true);
         }
@@ -197,7 +207,8 @@ protected JsonBrowser loadTrackInfoFromInnertube(
         HttpInterface httpInterface,
         String videoId,
         YoutubeAudioSourceManager sourceManager,
-        InfoStatus infoStatus
+        InfoStatus infoStatus,
+        YoutubeClientConfig clientOverride
     ) throws IOException {
         if (cachedPlayerScript == null) fetchScript(videoId, httpInterface);
 
@@ -206,24 +217,26 @@ protected JsonBrowser loadTrackInfoFromInnertube(
             cachedPlayerScript.playerScriptUrl
         );
         HttpPost post = new HttpPost(PLAYER_URL);
-        YoutubeClientConfig clientConfig;
-
-        if (infoStatus == InfoStatus.PREMIERE_TRAILER) {
-            // Android client gives encoded Base64 response to trailer which is also protobuf so we can't decode it
-            clientConfig = YoutubeClientConfig.WEB.copy();
-        } else if (infoStatus == InfoStatus.NON_EMBEDDABLE) {
-            // Used when age restriction bypass failed, if we have valid auth then most likely this request will be successful
-            clientConfig = YoutubeClientConfig.ANDROID.copy()
-                .withRootField("params", PLAYER_PARAMS);
-        } else if (infoStatus == InfoStatus.REQUIRES_LOGIN) {
-            // Age restriction bypass
-            clientConfig = YoutubeClientConfig.TV_EMBEDDED.copy();
-        } else {
-            // Default payload from what we start trying to get required data
-            clientConfig = YoutubeClientConfig.ANDROID.copy()
-                .withClientField("clientScreen", CLIENT_SCREEN_EMBED)
-                .withThirdPartyEmbedUrl(CLIENT_THIRD_PARTY_EMBED)
-                .withRootField("params", PLAYER_PARAMS);
+        YoutubeClientConfig clientConfig = clientOverride;
+
+        if (clientConfig == null) {
+            if (infoStatus == InfoStatus.PREMIERE_TRAILER) {
+                // Android client gives encoded Base64 response to trailer which is also protobuf so we can't decode it
+                clientConfig = YoutubeClientConfig.WEB.copy();
+            } else if (infoStatus == InfoStatus.NON_EMBEDDABLE) {
+                // Used when age restriction bypass failed, if we have valid auth then most likely this request will be successful
+                clientConfig = YoutubeClientConfig.WEB.copy()
+                    .withRootField("params", PLAYER_PARAMS);
+            } else if (infoStatus == InfoStatus.REQUIRES_LOGIN) {
+                // Age restriction bypass
+                clientConfig = YoutubeClientConfig.TV_EMBEDDED.copy();
+            } else {
+                // Default payload from what we start trying to get required data
+                clientConfig = YoutubeClientConfig.WEB.copy()
+                    .withClientField("clientScreen", CLIENT_SCREEN_EMBED)
+                    .withThirdPartyEmbedUrl(CLIENT_THIRD_PARTY_EMBED)
+                    .withRootField("params", PLAYER_PARAMS);
+            }
         }
 
         clientConfig

main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAccessTokenTracker.java

@@ -229,7 +229,7 @@ private String fetchVisitorId() throws IOException {
         try (HttpInterface httpInterface = httpInterfaceManager.getInterface()) {
             httpInterface.getContext().setAttribute(TOKEN_FETCH_CONTEXT_ATTRIBUTE, true);
 
-            YoutubeClientConfig clientConfig = YoutubeClientConfig.ANDROID.copy().setAttribute(httpInterface);
+            YoutubeClientConfig clientConfig = YoutubeClientConfig.WEB.copy().setAttribute(httpInterface);
             HttpPost visitorIdPost = new HttpPost(VISITOR_ID_URL);
             StringEntity visitorIdPayload = new StringEntity(clientConfig.toJsonString(), "UTF-8");
             visitorIdPost.setEntity(visitorIdPayload);

main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeAudioTrack.java

@@ -25,6 +25,8 @@
  */
 public class YoutubeAudioTrack extends DelegatedAudioTrack {
     private static final Logger log = LoggerFactory.getLogger(YoutubeAudioTrack.class);
+    private static final int MAX_RETRIES = 3;
+
 
     private final YoutubeAudioSourceManager sourceManager;
 
@@ -41,23 +43,47 @@ public YoutubeAudioTrack(AudioTrackInfo trackInfo, YoutubeAudioSourceManager sou
     @Override
     public void process(LocalAudioTrackExecutor localExecutor) throws Exception {
         try (HttpInterface httpInterface = sourceManager.getHttpInterface()) {
-            FormatWithUrl format = loadBestFormatWithUrl(httpInterface);
+            FormatWithUrl format = loadBestFormatWithUrl(httpInterface, null);
 
             log.debug("Starting track from URL: {}", format.signedUrl);
 
             if (trackInfo.isStream || format.details.getContentLength() == CONTENT_LENGTH_UNKNOWN) {
                 processStream(localExecutor, format);
             } else {
-                processStatic(localExecutor, httpInterface, format);
+                processStaticWithRetry(localExecutor, httpInterface, format);
             }
         }
     }
 
+
     @Override
     public boolean isSeekable() {
         return true;
     }
 
+    @SuppressWarnings("ConstantValue")
+    private void processStaticWithRetry(LocalAudioTrackExecutor localExecutor, HttpInterface httpInterface, FormatWithUrl initialFormat) throws Exception {
+        FormatWithUrl format = initialFormat;
+
+        for (int i = 1; i <= MAX_RETRIES; i++) {
+            log.debug("Opening YouTube stream URL attempt {}/{}", i, MAX_RETRIES);
+
+            try {
+                processStatic(localExecutor, httpInterface, format);
+                return;
+            } catch (RuntimeException e) {
+                String message = e.getMessage();
+
+                // Throw if it's not a message we're expecting, or this is the last retry.
+                if (!"Not success status code: 403".equals(message) && !"Invalid status code for video page response: 400".equals(message) || i == MAX_RETRIES) {
+                    throw e;
+                }
+
+                format = loadBestFormatWithUrl(httpInterface, null);
+            }
+        }
+    }
+
     private void processStatic(LocalAudioTrackExecutor localExecutor, HttpInterface httpInterface, FormatWithUrl format) throws Exception {
         try (YoutubePersistentHttpStream stream = new YoutubePersistentHttpStream(httpInterface, format.signedUrl, format.details.getContentLength())) {
             if (format.details.getType().getMimeType().endsWith("/webm")) {
@@ -78,9 +104,9 @@ private void processStream(LocalAudioTrackExecutor localExecutor, FormatWithUrl
         }
     }
 
-    private FormatWithUrl loadBestFormatWithUrl(HttpInterface httpInterface) throws Exception {
+    private FormatWithUrl loadBestFormatWithUrl(HttpInterface httpInterface, YoutubeClientConfig clientConfig) throws Exception {
         YoutubeTrackDetails details = sourceManager.getTrackDetailsLoader()
-            .loadDetails(httpInterface, getIdentifier(), true, sourceManager);
+            .loadDetails(httpInterface, getIdentifier(), true, sourceManager, clientConfig);
 
         // If the error reason is "Video unavailable" details will return null
         if (details == null) {

main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeMixProvider.java

@@ -44,6 +44,8 @@ public AudioPlaylist load(
         List<AudioTrack> tracks = new ArrayList<>();
 
         HttpPost post = new HttpPost(NEXT_URL);
+        // TODO: Keep an eye on this.
+        // Can't drop in WEB as it yields no results.
         YoutubeClientConfig clientConfig = YoutubeClientConfig.ANDROID.copy()
             .withRootField("videoId", selectedVideoId)
             .withRootField("playlistId", mixId)

main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeSearchProvider.java

@@ -52,7 +52,7 @@ public AudioItem loadSearchResult(String query, Function<AudioTrackInfo, AudioTr
 
         try (HttpInterface httpInterface = httpInterfaceManager.getInterface()) {
             HttpPost post = new HttpPost(SEARCH_URL);
-            YoutubeClientConfig clientConfig = YoutubeClientConfig.ANDROID.copy()
+            YoutubeClientConfig clientConfig = YoutubeClientConfig.WEB.copy()
                 .withRootField("query", query)
                 .withRootField("params", SEARCH_PARAMS)
                 .setAttribute(httpInterface);

main/src/main/java/com/sedmelluq/discord/lavaplayer/source/youtube/YoutubeTrackDetailsLoader.java

@@ -3,5 +3,18 @@
 import com.sedmelluq.discord.lavaplayer.tools.io.HttpInterface;
 
 public interface YoutubeTrackDetailsLoader {
-    YoutubeTrackDetails loadDetails(HttpInterface httpInterface, String videoId, boolean requireFormats, YoutubeAudioSourceManager sourceManager);
+    YoutubeTrackDetails loadDetails(HttpInterface httpInterface,
+                                    String videoId,
+                                    boolean requireFormats,
+                                    YoutubeAudioSourceManager sourceManager);
+
+    default YoutubeTrackDetails loadDetails(HttpInterface httpInterface,
+                                    String videoId,
+                                    boolean requireFormats,
+                                    YoutubeAudioSourceManager sourceManager,
+                                    YoutubeClientConfig clientOverride) {
+        // Implemented this way to be non-breaking. clientOverride will be ignored for implementations
+        // that don't implement this override.
+        return loadDetails(httpInterface, videoId, requireFormats, sourceManager);
+    }
 }